def added_tokens()

in src/engine/step4/model_dev/utils/model.py [0:0]

• 40 lines of code
• 2 McCabe index (conditional complexity)

    def added_tokens(self):
        """Tokens to be added to the pretrained tokenizer/vocab."""
        added_tokens = [
            "[ARG-DRUG]",
            "[ARG-CONDITION]",
            "[ARG-GENDER]",
            "[ARG-RACE]",
            "[ARG-ETHNICITY]",
            "[ARG-STATE]",
            "[ARG-AGE]",
            "[ARG-TIMEDAYS]",
            "[ARG-TIMEYEARS]",
            "[GENDER-TEMPLATE]",
            "[RACE-TEMPLATE]",
            "[ETHNICITY-TEMPLATE]",
            "[STATEID-TEMPLATE]",
            "[CONDITION-TEMPLATE]",
            "[DRUG-TEMPLATE]",
            "[ARG-CONDITION]",
            "[STATENAME-TEMPLATE]",
            "[ARG-DRUG]",
            "[ARG-DAYS]",
            "DATEDIFF",
            "DISTINCT",
            "GREATEST",
            "[SCHEMA]",
            "SELECT",
            "GROUP",
            "LEAST",
            "UNION",
            "COUNT",
            "WHERE",
            "JOIN",
            "FROM",
            "AND",
            "AS",
            "OR",
            "BY",
            "ON",
        ] + [f"[{i}]" for i in range(10)]

        return added_tokens