def cql_massage_tokens()

in cqlsh-expansion/pylib/cqlshlib/cqlhandling.py [0:0]

• 28 lines of code
• 5 McCabe index (conditional complexity)

    def cql_massage_tokens(self, toklist):
        curstmt = []
        output = []

        term_on_nl = False

        for t in toklist:
            if t[0] == 'endline':
                if term_on_nl:
                    t = ('endtoken',) + t[1:]
                else:
                    # don't put any 'endline' tokens in output
                    continue

            # Convert all unicode tokens to ascii, where possible.  This
            # helps avoid problems with performing unicode-incompatible
            # operations on tokens (like .lower()).  See CASSANDRA-9083
            # for one example of this.
            str_token = t[1]
            if isinstance(str_token, unicode):
                try:
                    str_token = str_token.encode('ascii')
                    t = (t[0], str_token) + t[2:]
                except UnicodeEncodeError:
                    pass

            curstmt.append(t)
            if t[0] == 'endtoken':
                term_on_nl = False
                output.extend(curstmt)
                curstmt = []
            else:
                if len(curstmt) == 1:
                    # first token in statement; command word
                    cmd = t[1].lower()
                    term_on_nl = bool(cmd in self.commands_end_with_newline)

        output.extend(curstmt)
        return output