public Token nextToken()

in enterprise/jsp.lexer/src/org/netbeans/lib/jsp/lexer/JspLexer.java [307:1272]


    public Token<JspTokenId> nextToken() {
        int actChar;
        while (true) {
            actChar = input.read();
            
            if (actChar == EOF) {
                if(input.readLengthEOF() == 1) {
                    return null; //just EOL is read
                } else {
                    //there is something else in the buffer except EOL
                    //we will return last token now
                    input.backup(1); //backup the EOL, we will return null in next nextToken() call
                    break;
                }
            }
            
            switch (lexerState) {
                case INIT:
                    switch (actChar) {
                        //                        case '\n':
                        //                            return token(JspTokenId.EOL);
                        case '<':
                            lexerState = ISA_LT;
                            break;
                            //                        default:
                            //                            state = ISI_ERROR;
                            //                            break;
                        case '\\':
                            lexerState = ISA_BS;
                            break;
                        case '$':
                            lexerStateBeforeEL = lexerState; //remember main state
                            lexerState = ISA_EL_DELIM_DOLLAR;
                            break;
                        case '#': //maybe expression language
                            lexerStateBeforeEL = lexerState; //remember main state
                            lexerState = ISA_EL_DELIM_HASH;
                            break;
                    }
                    break;
                    
                case ISA_BS:
                    if(actChar != '\\') {
                        lexerState = INIT; //prevent scaped EL in text being recognized
                    }
                    break;
                    
                case ISA_EL_DELIM_DOLLAR:
                    if(isELIgnored()) {
                        //reset to previous state - do not recognize EL
                        lexerState = lexerStateBeforeEL;
                        lexerStateBeforeEL = INIT;
                        break;
                    }
                case ISA_EL_DELIM_HASH:
                    switch (actChar) {
                        case '{':
                            if (input.readLength() > 2) {
                                //we have something read except the '${' or '#{' => it's content language
                                input.backup(2); //backup the '$/#{'
                                lexerState = lexerStateBeforeEL; //we will read the '$/#{' again
                                lexerStateBeforeEL = INIT;
                                return token(JspTokenId.TEXT); //return the content language token
                            }
                            lexerState = ISI_EL;
                            break;
                        default:
                            input.backup(1); //put the read char back
                            lexerState = lexerStateBeforeEL;
                            lexerStateBeforeEL = INIT;
                    }
                    break;
                    
                case ISI_EL:
                    switch (actChar) {
                        case '\'':
                            lexerState = ISI_EL_SINGLE_QUOTE;
                            break;
                        case '"':
                            lexerState = ISI_EL_DOUBLE_QUOTE;
                            break;
                        case '{':
                            lexerStateCurlyNestedLevel++;
                            break;
                        case '}':
                            if (lexerStateCurlyNestedLevel > 0) {
                                lexerStateCurlyNestedLevel--;
                            } else {
                                //return EL token
                                lexerState = lexerStateBeforeEL;
                                lexerStateBeforeEL = INIT;
                                return token(JspTokenId.EL);
                            }
                    }
                    break;

                case ISI_EL_SINGLE_QUOTE:
                    switch (actChar) {
                        case '\\':
                            lexerState = ISA_EL_SINGLE_QUOTE_ESCAPE;
                            break;
                        case '\'':
                            lexerState = ISI_EL;
                            break;
                    }
                    break;

                case ISI_EL_DOUBLE_QUOTE:
                    switch (actChar) {
                        case '\\':
                            if (lexerStateBeforeEL != 14) {
                                lexerState = ISA_EL_DOUBLE_QUOTE_ESCAPE;
                            }
                            break;
                        case '"':
                            lexerState = ISI_EL;
                            break;
                    }
                    break;

                case ISA_EL_DOUBLE_QUOTE_ESCAPE:
                    //just skip back qouted string
                    lexerState = ISI_EL_DOUBLE_QUOTE;
                    break;

                case ISA_EL_SINGLE_QUOTE_ESCAPE:
                    //just skip back qouted string
                    lexerState = ISI_EL_SINGLE_QUOTE;
                    break;
                    
                case ISA_LT:
                    if (Character.isLetter(actChar) ||
                            (actChar == '_')
                            ) { // possible tag begining
                        input.backup(1); //backup the read letter
                        CharSequence tagName = getPossibleTagName();
                        if(isJspTag(tagName)) { //test if a jsp tag follows
                            if(input.readLength() > 1) {
                                //we have something read except the '<' => it's content language
                                input.backup(1); //backup the '<'
                                lexerState = INIT; //we will read the '<' again
                                return token(JspTokenId.TEXT); //return the content language token
                            }
                            //possibly switch to scriptlet when <jsp:scriptlet> found
                            
                            if(CharSequenceUtilities.equals("jsp:scriptlet",tagName)) { //NOI18N
                                lexerStateJspScriptlet = JAVA_SCRITPLET_DOCUMENT;
                            } else if(CharSequenceUtilities.equals("jsp:declaration", tagName)) { //NOI18N
                                lexerStateJspScriptlet = JAVA_DECLARATION_DOCUMENT;
                            } else if(CharSequenceUtilities.equals("jsp:expression", tagName)) { //NOI18N
                                lexerStateJspScriptlet = JAVA_EXPRESSION_DOCUMENT;
                            }
                            
                            lexerState = ISI_TAGNAME;
                            return token(JspTokenId.SYMBOL); //return the read '<' symbol first
                        } else {
                            //just a content language
                            lexerState = INIT;
                            break;
                        }
                    }
                    
                    switch (actChar) {
                        case '/':
                            lexerState = ISA_LT_SLASH;
                            break;
                            //                        case '\n':
                            //                            state = ISI_TAG_ERROR;
                            //                            input.backup(1);
                            //                            return token(JspTokenId.SYMBOL);
                        case '%':
                            lexerState = ISA_LT_PC;
                            break;
                        default:
                            input.backup(1);
                            lexerState = INIT; //just content
                            //                            state = ISI_TAG_ERROR;
                            //                            break;
                    }
                    break;
                    
                case ISA_LT_SLASH:
                    if (Character.isLetter(actChar) ||
                            (actChar == '_')) {
                        //possible end tag beginning
                        input.backup(1); //backup the first letter
                        if(followsJspTag()) {
                            if(input.readLength() > 2) {
                                //we have something read except the '</' symbol
                                input.backup(2);
                                lexerState = INIT;
                                return token(JspTokenId.TEXT);
                            } else {
                                lexerState = ISI_ENDTAG;
                                return token(JspTokenId.SYMBOL);  //return the read '</' symbol first
                            }
                            //break;
                        } else {
                            //just a content language
                            lexerState = INIT;
                            break;
                        }
                    }
                    
                    //not jsp end tag -> just content -> switch to init state
                    lexerState = INIT;
                    break;
                    
                case ISI_TAGNAME:
                case ISI_DIRNAME:
                    
                    if (!(Character.isLetter(actChar) ||
                            Character.isDigit(actChar) ||
                            (actChar == '_') ||
                            (actChar == '-') ||
                            (actChar == ':') ||
                            (actChar == '.'))) { // not alpha
                        switch(actChar) {
                            case '<':
                                lexerState = INIT;
                                input.backup(1);
                                break;
                            case '/':
                                input.backup(1);
                                lexerState = ((lexerState == ISI_TAGNAME) ? ISP_TAG : ISP_DIR);
                                break;
                            case '>':
                                input.backup(1); //backup the '<' char
                                if(lexerStateJspScriptlet != INIT) {
                                    //switch to java scriptlet
                                    lexerState = ISP_GT_SCRIPTLET;
                                } else {
                                    lexerState = ((lexerState == ISI_TAGNAME) ? ISP_TAG : ISP_DIR);
                                }
                                break;
                            case ' ':
                                input.backup(1);
                                lexerState = ((lexerState == ISI_TAGNAME) ? ISP_TAG : ISP_DIR);
                                break;
                            case '\n':
                                lexerState = ISP_TAG;
                                input.backup(1); //backup the eof
                                return token(JspTokenId.TAG);
                            default:
                                lexerState = ((lexerState == ISI_TAGNAME) ? ISP_TAG : ISP_DIR);
                        }
                        return token(JspTokenId.TAG);
                    }
                    break;

                //internal state for > symbol after jsp:expression/scriptlet/declaration tag
                case ISP_GT_SCRIPTLET:
                    assert actChar == '>';
                    lexerState = ISI_SCRIPTLET;
                    return token(JspTokenId.SYMBOL);

                case ISP_TAG:
                case ISP_DIR:
                    if (Character.isLetter(actChar) ||
                            (actChar == '_')
                            ) {
                        lexerState = ((lexerState == ISP_TAG) ? ISI_TAG_ATTR : ISI_DIR_ATTR);
                        break;
                    }
                    switch (actChar) {
                        case '\n':
                            //                            if (input.readLength() == 1) { // no char
                            return token(JspTokenId.EOL);
                            //                            } else { // return string first
                            //                                input.backup(1);
                            //                                return decide_jsp_tag_token();
                            //                            }
                        case '>': // for tags
                            if (lexerState == ISP_TAG) {
                                //                                if (input.readLength() == 1) {  // no char
                                //                                    state = ISA_END_JSP;
                                lexerState = INIT;
                                return token(JspTokenId.SYMBOL);
                                //                                } else { // return string first
                                //                                    input.backup(1);
                                //                                    return decide_jsp_tag_token();
                                //                                }
                            } else { // directive
                                lexerState = ISI_DIR_ERROR;
                                break;
                            }
                        case '/': // for tags
                            if (lexerState == ISP_TAG) {
                                //                                if (input.readLength() == 1) {  // no char
                                lexerState = ISA_ENDSLASH;
                                break;
                                //                                } else { // return string first
                                //                                    input.backup(1);
                                //                                    return decide_jsp_tag_token();
                                //                                }
                            } else { // directive
                                lexerState = ISI_DIR_ERROR;
                                break;
                            }
                        case '%': // for directives
                            if (lexerState == ISP_DIR) {
                                //                                if (input.readLength() == 1) {  // no char
                                lexerState = ISA_ENDPC;
                                break;
                                //                                } else { // return string first
                                //                                    input.backup(1);
                                //                                    return decide_jsp_tag_token();
                                //                                }
                            } else { // tag
                                lexerState = ISI_TAG_ERROR;
                                break;
                            }
                        case '=':
                            lexerState = ((lexerState == ISP_TAG) ? ISP_TAG_EQ : ISP_DIR_EQ);
                            return token(JspTokenId.SYMBOL);
                        case ' ':
                        case '\t':
                            lexerState = ((lexerState == ISP_TAG) ? ISI_TAG_I_WS : ISI_DIR_I_WS);
                            break;
                        case '<':
                            // assume that this is the start of the next tag
                            //we shouldn't have anything else than then the < char in buffer
                            assert input.readLength() == 1 : "There is something more than '<' char in the read text: '" + input.readText() + "'"; //NOI18N
                            input.backup(1);
                            lexerState = INIT;
                        default: //numbers or illegal symbols
                            lexerState = ((lexerState == ISP_TAG) ? ISI_TAG_ERROR : ISI_DIR_ERROR);
                            break;
                    }
                    break;
                    
                case ISI_TAG_I_WS:
                case ISI_DIR_I_WS:
                    switch (actChar) {
                        case ' ':
                        case '\t':
                            break;
                        case '<': //start of the next tag
                            //                            state = ISA_END_JSP;
                            lexerState = INIT;
                            input.backup(1);
                            return token(JspTokenId.TAG);
                        default:
                            lexerState = ((lexerState == ISI_TAG_I_WS) ? ISP_TAG : ISP_DIR);
                            input.backup(1);
                            return token(JspTokenId.WHITESPACE);
                    }
                    break;
                    
                case ISI_ENDTAG:
                    if (!(Character.isLetter(actChar) ||
                            Character.isDigit(actChar) ||
                            (actChar == '_') ||
                            (actChar == '.') ||
                            (actChar == '-') ||
                            (actChar == ':'))
                            ) { // not alpha
                        lexerState = ISP_TAG;
                        input.backup(1);
                        return token(JspTokenId.ENDTAG);
                    }
                    break;
                    
                case ISI_TAG_ATTR:
                case ISI_DIR_ATTR:
                    if (!(Character.isLetter(actChar) ||
                            Character.isDigit(actChar) ||
                            (actChar == '_') ||
                            (actChar == ':') ||
                            (actChar == '.') ||
                            (actChar == '-'))
                            ) { // not alpha or '-' (http-equiv)
                        lexerState = ((lexerState == ISI_TAG_ATTR) ? ISP_TAG : ISP_DIR);
                        input.backup(1);
                        return token(JspTokenId.ATTRIBUTE);
                    }
                    break;
                    
                case ISP_TAG_EQ:
                case ISP_DIR_EQ:
                    switch (actChar) {
                        case '\n':
                            //                            if (input.readLength() == 1) { // no char
                            return token(JspTokenId.EOL);
                            //                            } else { // return string first
                            //                                input.backup(1);
                            //                                return token(JspTokenId.ATTR_VALUE);
                            //                            }
                        case '"':
                            lexerState = ((lexerState == ISP_TAG_EQ) ? ISI_TAG_STRING : ISI_DIR_STRING);
                            break;
                        case '\'':
                            lexerState = ((lexerState == ISP_TAG_EQ) ? ISI_TAG_STRING2 : ISI_DIR_STRING2);
                            break;
                        case ' ':
                        case '\t':
                            // don't change the state
                            break;
                        default:
                            //invalid value - lets backup it and swith to tag content
                            lexerState = ((lexerState == ISP_TAG_EQ) ? ISP_TAG : ISP_DIR);
                            input.backup(input.readLength());
                            break;
                    }
                    break;
                    
                case ISI_TAG_STRING:
                case ISI_DIR_STRING:
                case ISI_TAG_STRING2:
                case ISI_DIR_STRING2:
                    if ((actChar == '"') && ((lexerState == ISI_TAG_STRING) || (lexerState == ISI_DIR_STRING))) {
                        lexerState = ((lexerState == ISI_TAG_STRING) ? ISP_TAG : ISP_DIR);
                        return token(JspTokenId.ATTR_VALUE);
                    }
                    
                    if ((actChar == '\'') && ((lexerState == ISI_TAG_STRING2) || (lexerState == ISI_DIR_STRING2))) {
                        lexerState = ((lexerState == ISI_TAG_STRING2) ? ISP_TAG : ISP_DIR);
                        return token(JspTokenId.ATTR_VALUE);
                    }

                    if(followsScriptletExpressionDelimiter(actChar)) {
                        if(input.readLength() == 3) {
                            //just the delimiter in buffer, we may make the token

                            //remember where we jumped into the scriptlet,
                            //we'll return to the state once we reach end of the
                            //scriptlet
                            lexerStateBeforeScriptlet = lexerState;
                            lexerStateJspScriptlet = JAVA_EXPRESSION;
                            lexerState = ISI_SCRIPTLET;
                            
                            return token(JspTokenId.SYMBOL2);
                            
                        } else {
                            //first tokenize the text before
                            input.backup(3); //backup <%=
                            //keep the state
                            return token(JspTokenId.ATTR_VALUE);
                        }
                    }

                    switch (actChar) {
                        case '\\':
                            switch (lexerState) {
                                case ISI_TAG_STRING:
                                    lexerState = ISI_TAG_STRING_B;
                                    break;
                                case ISI_DIR_STRING:
                                    lexerState = ISI_DIR_STRING_B;
                                    break;
                                case ISI_TAG_STRING2:
                                    lexerState = ISI_TAG_STRING2_B;
                                    break;
                                case ISI_DIR_STRING2:
                                    lexerState = ISI_DIR_STRING2_B;
                                    break;
                            }
                            break;
                        case '\n':
                            if (input.readLength() == 1) { // no char
                                return token(JspTokenId.EOL);
                            } else { // return string first
                                input.backup(1);
                                return token(JspTokenId.ATTR_VALUE);
                            }
                        case '$':
                            if(input.readLength() > 1) {
                                //return part of the attribute value before EL
                                input.backup(1); //backup $ or #
                                return token(JspTokenId.ATTR_VALUE);
                            } else {
                                lexerStateBeforeEL = lexerState; //remember main state
                                lexerState = ISA_EL_DELIM_DOLLAR;
                            }
                            break;
                        case '#':
                            if(input.readLength() > 1) {
                                //return part of the attribute value before EL
                                input.backup(1); //backup $ or #
                                return token(JspTokenId.ATTR_VALUE);
                            } else {
                                lexerStateBeforeEL = lexerState; //remember main state
                                lexerState = ISA_EL_DELIM_HASH;
                            }
                            break;
                            
                        default:
                            break;//stay in ISI_TAG_STRING/2;
                            
                    }
                    break;
                    
                case ISI_TAG_STRING_B:
                case ISI_DIR_STRING_B:
                case ISI_TAG_STRING2_B:
                case ISI_DIR_STRING2_B:
                    switch (actChar) {
                        case '"':
                        case '\'':
                        case '\\':
                        case '$':
                        case '#':
                            break;
                        default:
                            input.backup(1);
                            break;
                    }
                    switch (lexerState) {
                        case ISI_TAG_STRING_B:
                            lexerState = ISI_TAG_STRING;
                            break;
                        case ISI_DIR_STRING_B:
                            lexerState = ISI_DIR_STRING;
                            break;
                        case ISI_TAG_STRING2_B:
                            lexerState = ISI_TAG_STRING2;
                            break;
                        case ISI_DIR_STRING2_B:
                            lexerState = ISI_DIR_STRING2;
                            break;
                    }
                    break;
                    
                case ISA_ENDSLASH:
                    switch (actChar) {
                        case '>':
                            //                            state = ISA_END_JSP;
                            lexerState = INIT;
                            return token(JspTokenId.SYMBOL);
                        case '\n':
                            lexerState = ISI_TAG_ERROR;
                            input.backup(1);
                            return token(JspTokenId.SYMBOL);
                        default:
                            lexerState = ISP_TAG;
                            input.backup(1);
                            return token(JspTokenId.SYMBOL);
                    }
                    //break; not reached
                    
                case ISA_ENDPC:
                    switch (actChar) {
                        case '>':
                            //                            state = ISA_END_JSP;
                            lexerState = INIT;
                            return token(JspTokenId.SYMBOL);
                        case '\n':
                            lexerState = ISI_DIR_ERROR;
                            input.backup(1);
                            return token(JspTokenId.SYMBOL);
                        default:
                            lexerState = ISP_DIR;
                            input.backup(1);
                            return token(JspTokenId.SYMBOL);
                    }
                    //break; not reached
                    
                case ISA_LT_PC:
                    switch (actChar) {
                        case '@':
                            if(input.readLength() == 3) {
                                // just <%@ read
                                lexerState = ISA_LT_PC_AT;
                                return token(JspTokenId.SYMBOL);
                            } else {
                                //jsp symbol, but we also have content language in the buffer
                                input.backup(3); //backup <%@
                                lexerState = INIT;
                                return token(JspTokenId.TEXT); //return CL token
                            }
                        case '-': //may be JSP comment
                            lexerState = ISA_LT_PC_DASH;
                            break;
                        case '!': // java declaration
                        case '=': // java expression
                            if(input.readLength() == 3) {
                                // just <%! or <%= read
                                lexerStateJspScriptlet = actChar == '!' ? JAVA_DECLARATION : JAVA_EXPRESSION;
                                lexerState = ISI_SCRIPTLET;
                                return token(JspTokenId.SYMBOL2);
                            } else {
                                //jsp symbol, but we also have content language in the buffer
                                input.backup(3); //backup <%! or <%=
                                lexerState = INIT;
                                return token(JspTokenId.TEXT); //return CL token
                            }
                        default:  //java scriptlet delimiter '<%'
                            if(input.readLength() == 3) {
                                // just <% + something != [-,!,=,@] read
                                lexerStateJspScriptlet = JAVA_SCRITPLET;
                                lexerState = ISI_SCRIPTLET;
                                input.backup(1); //backup the third character, it is a part of the java scriptlet
                                return token(JspTokenId.SYMBOL2);
                            } else {
                                //jsp symbol, but we also have content language in the buffer
                                input.backup(3); //backup <%@
                                lexerState = INIT;
                                return token(JspTokenId.TEXT); //return CL token
                            }
                    }
                    break;
                    
                case ISI_SCRIPTLET:
                    switch(actChar) {
                        case '%':
                                lexerState = ISP_SCRIPTLET_PC;
                                break;
                        case '<':
                            //may be end of scriptlet section in JSP document
                            CharSequence tagName = getPossibleTagName();
                            if(CharSequenceUtilities.equals("/jsp:scriptlet", tagName) || //NOI18N
                                    CharSequenceUtilities.equals("/jsp:declaration", tagName) || //NOI18N
                                    CharSequenceUtilities.equals("/jsp:expression", tagName)) { //NOI18N
                                if(input.readLength() == 1) {
                                    //just the '<' symbol read
                                    input.backup(1);
                                    lexerState = INIT;
                                } else {
                                    //return the scriptlet content
                                    input.backup(1); // backup '<' we will read it again
                                    int lxs = lexerStateJspScriptlet;
                                    lexerStateJspScriptlet = INIT;
                                    return scriptletToken(JspTokenId.SCRIPTLET, lxs);
                                }
                            }
                    }
                    break;
                    
                case ISP_SCRIPTLET_PC:
                    switch(actChar) {
                        case '>':
                            if(input.readLength() == 2) {
                                //just the '%>' symbol read
                                lexerState = lexerStateBeforeScriptlet;
                                lexerStateBeforeScriptlet = INIT;
                                lexerStateJspScriptlet = INIT;
                                return token(JspTokenId.SYMBOL2);
                            } else {
                                //return the scriptlet content
                                input.backup(2); // backup '%>' we will read JUST them again
                                lexerState = ISI_SCRIPTLET;
                                int lxs = lexerStateJspScriptlet;
                                lexerStateJspScriptlet = INIT;
                                return scriptletToken(JspTokenId.SCRIPTLET, lxs);
                            }
                        default:
                            lexerState = ISI_SCRIPTLET;
                            break;
                    }
                    break;
                    
                case ISA_LT_PC_DASH:
                    switch(actChar) {
                        case '-':
                            if(input.readLength() == 4) {
                                //just the '<%--' symbol read
                                lexerState = ISI_JSP_COMMENT;
                            } else {
                                //return the scriptlet content
                                input.backup(4); // backup '<%--', we will read it again
                                lexerState = INIT;
                                return token(JspTokenId.TEXT);
                            }
                            break;
                        default:
                            //                            state = ISA_END_JSP;
                            lexerState = INIT; //XXX how to handle content language?
                            return token(JspTokenId.TEXT); //marek: should I token here????
                    }
                    
                    // JSP states
                case ISI_JSP_COMMENT:
                    switch (actChar) {
                        case '\n':
                            if (input.readLength() == 1) { // no char
                                return token(JspTokenId.EOL);
                            } else { // return block comment first
                                input.backup(1);
                                return token(JspTokenId.COMMENT);
                            }
                        case '-':
                            lexerState = ISI_JSP_COMMENT_M;
                            break;
                    }
                    break;
                    
                case ISI_JSP_COMMENT_M:
                    switch (actChar) {
                        case '\n':
                            lexerState = ISI_JSP_COMMENT;
                            if (input.readLength() == 1) { // no char
                                return token(JspTokenId.EOL);
                            } else { // return block comment first
                                input.backup(1);
                                return token(JspTokenId.COMMENT);
                            }
                        case '-':
                            lexerState = ISI_JSP_COMMENT_MM;
                            break;
                        default:
                            lexerState = ISI_JSP_COMMENT;
                            break;
                    }
                    break;
                    
                case ISI_JSP_COMMENT_MM:
                    switch (actChar) {
                        case '\n':
                            lexerState = ISI_JSP_COMMENT;
                            if (input.readLength() == 1) { // no char
                                return token(JspTokenId.EOL);
                            } else { // return block comment first
                                input.backup(1);
                                return token(JspTokenId.COMMENT);
                            }
                        case '%':
                            lexerState = ISI_JSP_COMMENT_MMP;
                            break;
                        case '-':
                            lexerState = ISI_JSP_COMMENT_MM;
                            break;
                        default:
                            lexerState = ISI_JSP_COMMENT;
                            break;
                    }
                    break;
                    
                case ISI_JSP_COMMENT_MMP:
                    switch (actChar) {
                        case '\n':
                            lexerState = ISI_JSP_COMMENT;
                            if (input.readLength() == 1) { // no char
                                return token(JspTokenId.EOL);
                            } else { // return block comment first
                                input.backup(1);
                                return token(JspTokenId.COMMENT);
                            }
                        case '>':
                            //                            state = ISA_END_JSP;
                            lexerState = INIT;
                            return token(JspTokenId.COMMENT);
                        default:
                            lexerState = ISI_JSP_COMMENT;
                            break;
                    }
                    break;
                    
                case ISI_ERROR:
                    switch (actChar) {
                        case '\n':
                            lexerState = INIT;
                            input.backup(1);
                            return token(JspTokenId.ERROR);
                        case '<':
                            lexerState = ISA_LT;
                            input.backup(1);
                            return token(JspTokenId.ERROR);
                    }
                    break;
                    
                case ISI_TAG_ERROR:
                    switch (actChar) {
                        case '\n':
                            if (input.readLength() == 1) { // no char
                                lexerState = ISP_TAG;
                                return token(JspTokenId.EOL);
                            } else { // return error first
                                input.backup(1);
                                return token(JspTokenId.ERROR);
                            }
                        case '>':
                        case ' ':
                        case '\t':
                            lexerState = ISP_TAG;
                            input.backup(1);
                            return token(JspTokenId.ERROR);
                        default:
                            break;
                    }
                    break;
                    
                case ISI_DIR_ERROR:
                    switch (actChar) {
                        case '\n':
                            if (input.readLength() == 1) { // no char
                                lexerState = ISP_DIR;
                                return token(JspTokenId.EOL);
                            } else { // return error first
                                input.backup(1);
                                return token(JspTokenId.ERROR);
                            }
                            //                        case '%':
                        case '\t':
                        case ' ':
                            lexerState = ISP_DIR;
                            if(input.readLength() > 1) {
                                input.backup(1);
                                return token(JspTokenId.ERROR);
                            }
                        default:
                            break;
                    }
                    break;
                    
                case ISI_DIR_ERROR_P:
                    switch (actChar) {
                        case '\n':
                            if (input.readLength() == 1) { // no char
                                lexerState = ISI_DIR_I_WS;
                                return token(JspTokenId.EOL);
                            } else { // return error first
                                input.backup(1);
                                return token(JspTokenId.ERROR);
                            }
                        case '>':
                            input.backup(2);
                            lexerState = ISI_DIR_I_WS;
                            return token(JspTokenId.ERROR);
                        default:
                            break;
                    }
                    break;
                    
                    //                case ISA_END_JSP:
                    //                    if (input.readLength() == 1) {
                    //                        offset++;
                    //                        return JspTokenId.AFTER_UNEXPECTED_LT;
                    //                    }
                    //                    else {
                    //                        return JspTokenId.TEXT;
                    //                    }
                    //                    //break;
                    
                    // added states
                case ISA_LT_PC_AT:
                    if (Character.isLetter(actChar) ||
                            (actChar == '_')
                            ) {
                        // the directive starts
                        lexerState = ISI_DIRNAME;
                        break;
                    }
                    
                    switch (actChar) {
                        case '\n':
                            if (input.readLength() == 1) { // no char
                                return token(JspTokenId.EOL);
                            } else {
                                input.backup(1);
                                return token(JspTokenId.SYMBOL);
                            }
                        case ' ':
                        case '\t':
                            break;
                        case '%':
                            lexerState = ISA_ENDPC;
                            break;
                        default:
                            //error
                            lexerState = ISI_DIR_ERROR;
                            if(input.readLength() > 1) {
                                input.backup(1); //backup the error char if there is something more in the buffer
                                return token(JspTokenId.SYMBOL);
                            }
                            break;
                    }
                    break;
                    
            }
            
        }
        
        // At this stage there's no more text in the scanned buffer.
        // Scanner first checks whether this is completely the last
        // available buffer.
        
        switch(lexerState) {
            case INIT:
            case ISA_BS:
            case ISA_LT:
            case ISA_LT_SLASH:
                if (input.readLength() == 0) {
                    return null;
                } else {
                    return token(JspTokenId.TEXT);
                }
            case ISI_ERROR:
            case ISI_TAG_ERROR:
                lexerState = INIT;
                return token(JspTokenId.ERROR);
            case ISI_DIR_ERROR:
            case ISI_DIR_ERROR_P:
                lexerState = INIT;
                return token(JspTokenId.ERROR);
            case ISA_ENDSLASH:
            case ISP_TAG_EQ:
                lexerState = INIT;
                return token(JspTokenId.SYMBOL);
            case ISA_LT_PC:
            case ISA_LT_PC_DASH:
            case ISA_ENDPC:
            case ISP_DIR_EQ:
                lexerState = INIT;
                return token(JspTokenId.SYMBOL);
            case ISI_TAGNAME:
            case ISI_ENDTAG:
                lexerState = INIT;
                return token(JspTokenId.TAG);
            case ISI_DIRNAME:
                lexerState = INIT;
                return token(JspTokenId.TAG);
            case ISP_TAG:
            case ISI_TAG_I_WS:
                lexerState = INIT;
                return token(JspTokenId.TAG);
            case ISP_DIR:
            case ISI_DIR_I_WS:
            case ISA_LT_PC_AT:
                lexerState = INIT;
                return token(JspTokenId.TAG);
            case ISI_TAG_ATTR:
                lexerState = INIT;
                return token(JspTokenId.ATTRIBUTE);
            case ISI_DIR_ATTR:
                lexerState = INIT;
                return token(JspTokenId.ATTRIBUTE);
            case ISI_TAG_STRING:
            case ISI_TAG_STRING_B:
            case ISI_TAG_STRING2:
            case ISI_TAG_STRING2_B:
                lexerState = INIT;
                return token(JspTokenId.ATTR_VALUE);
            case ISI_DIR_STRING:
            case ISI_DIR_STRING_B:
            case ISI_DIR_STRING2:
            case ISI_DIR_STRING2_B:
                lexerState = INIT;
                return token(JspTokenId.ATTR_VALUE);
            case ISI_JSP_COMMENT:
            case ISI_JSP_COMMENT_M:
            case ISI_JSP_COMMENT_MM:
            case ISI_JSP_COMMENT_MMP:
                lexerState = INIT;
                return token(JspTokenId.COMMENT);
            case ISA_EL_DELIM_DOLLAR:
            case ISA_EL_DELIM_HASH:
                lexerState = INIT;
                return token(JspTokenId.TEXT);
            case ISI_EL:
            case ISI_EL_DOUBLE_QUOTE:
            case ISI_EL_SINGLE_QUOTE:
            case ISA_EL_DOUBLE_QUOTE_ESCAPE:
            case ISA_EL_SINGLE_QUOTE_ESCAPE:                
                lexerState = INIT;
                return token(JspTokenId.EL);
            case ISI_SCRIPTLET:
            case ISP_SCRIPTLET_PC:
                lexerState = INIT;
                return scriptletToken(JspTokenId.SCRIPTLET, lexerStateJspScriptlet);
            default:
                break;
        }
        
        return null;
        
    }