def tokenize()

in utils/trees.py [0:0]

• 11 lines of code
• 5 McCabe index (conditional complexity)

    def tokenize(flat_string):
        '''
        Tokenize EXR string
        Example input string:
        "(ORDER (PIZZAORDER (NUMBER 1) (TOPPING HAM) (COMPLEX_TOPPING (TOPPING ONIONS) (QUANTITY EXTRA))))"

        :param flat_string: (str) EXR-format input flat string to construct a tree

        :return: (list) A list of tokens obtained after tokenizing `flat_string`.
        '''
        special_characters = [',', '\n']
        return [t for t in re.split('([^a-zA-Z0-9._-])',flat_string) \
            if t and not(t.isspace()) and t not in special_characters]