protected TokenID parseToken()

in enterprise/web.core.syntax/src/org/netbeans/modules/web/core/syntax/deprecated/JspMultiSyntax.java [220:1249]


    protected TokenID parseToken() {
        //<editor-fold defaultstate="collapsed" desc="setting states">
        if (state != ISI_LANGUAGE) {
            char actChar;
            while(offset < stopOffset) {
                actChar = buffer[offset];
                switch (state) {
                    case ISI_HOST_JSPTAG: // switch to JspTagSyntax
                        nestMode = MODE_HOST_JSPTAG;
                        state = ISI_LANGUAGE;
                        transferMasterToSlave(jspTagSyntax, null);
                        if (jspTagStateInfo == null) {
                            jspTagStateInfo = jspTagSyntax.createStateInfo();
                        }
                        jspTagSyntax.storeState(jspTagStateInfo);
                        break;

                    case ISI_HOST_JAVA: // switch from hostSyntax to JavaSyntax
                        switch (actChar) {
                            case '<':
                                state = ISI_HOST_JAVA_LT;
                                break;
                            default:  
                                Logger.getLogger("global").log(Level.INFO, null, new Exception("bad hostsyntax analyzer"));  // NOI18N
                        } // switch (actChar)
                        break;

                    case ISI_HOST_JAVA_LT: 
                        switch (actChar) {
                            case '%':
                                state = ISI_HOST_JAVA_LT_PC;
                                break;
                            default:  
                                Logger.getLogger("global").log(Level.INFO, null, new Exception("bad hostsyntax analyzer"));  // NOI18N
                        } // switch (actChar)
                        break;

                    case ISI_HOST_JAVA_LT_PC: 
                        switch (actChar) {
                            case '!': // declaration
                                javaNestMode = JAVA_DECLARATION;
                                state = ISI_HOST_JAVA_JUMP;
                                offset++;
                                tokenContextPath = JspMultiTokenContext.jspTagContextPath;
                                return JspTagTokenContext.SYMBOL2;
                            case '=': // expression
                                javaNestMode = JAVA_EXPRESSION;
                                state = ISI_HOST_JAVA_JUMP;
                                offset++;
                                tokenContextPath = JspMultiTokenContext.jspTagContextPath;
                                return JspTagTokenContext.SYMBOL2;
                            default: // assume this is a scriptlet
                                javaNestMode = JAVA_SCRIPTLET;
                                state = ISI_HOST_JAVA_JUMP;
                                tokenContextPath = JspMultiTokenContext.jspTagContextPath;
                                return JspTagTokenContext.SYMBOL2;
                        } // switch (actChar)
                        // break; - not reached

                    case ISI_HOST_JAVA_JUMP: 
                        nestMode = MODE_HOST_JAVA;
                        state = ISI_LANGUAGE;
                        transferMasterToSlave(javaSyntax, null);
                        //javaSyntax.load(null, buffer, offset, stopOffset - offset, lastBuffer);
                        if (javaStateInfo == null) {
                            javaStateInfo = javaSyntax.createStateInfo();
                        }
                        javaSyntax.storeState(javaStateInfo);
                        break;

                    case ISI_HOST_EL: // switch from hostSyntax to ELSyntax
                        switch (actChar) {
                            case '$':
                            case '#':    
                                state = ISI_HOST_EL_D;
                                break;
                            default:  
                                Logger.getLogger("global").log(Level.INFO, null, new Exception("bad hostsyntax analyzer"));  // NOI18N
                        } // switch (actChar)
                        break;

                    case ISI_HOST_EL_D: 
                        switch (actChar) {
                            case '{': // EL expression
                                state = ISI_HOST_EL_JUMP;
                                offset++;
                                tokenContextPath = JspMultiTokenContext.elContextPath;
                                return ELTokenContext.EL_DELIM;
                            default:  
                                Logger.getLogger("global").log(Level.INFO, null, new Exception("bad hostsyntax analyzer"));  // NOI18N
                        } // switch (actChar)
                        break;

                    case ISI_HOST_EL_JUMP: 
                        nestMode = MODE_HOST_EL;
                        state = ISI_LANGUAGE;
                        transferMasterToSlave(elSyntax, null);
                        //elSyntax.load(null, buffer, offset, stopOffset - offset, lastBuffer);
                        if (elStateInfo == null) {
                            elStateInfo = elSyntax.createStateInfo();
                        }
                        elSyntax.storeState(elStateInfo);
                        break;

                    case ISI_JSPTAG_JAVA: // switch from JSP tag to JavaSyntax
                        switch (actChar) {
                            case '<':
                                state = ISI_JSPTAG_JAVA_LT;
                                break;
                            default:  
                                Logger.getLogger("global").log(Level.INFO, null, new Exception("bad hostsyntax analyzer"));  // NOI18N
                        } // switch (actChar)
                        break;

                    case ISI_JSPTAG_JAVA_LT: 
                        switch (actChar) {
                            case '%':
                                state = ISI_JSPTAG_JAVA_LT_PC;
                                break;
                            default:  
                                Logger.getLogger("global").log(Level.INFO, null, new Exception("bad hostsyntax analyzer"));  // NOI18N
                        } // switch (actChar)
                        break;

                    case ISI_JSPTAG_JAVA_LT_PC: 
                        switch (actChar) {
                            case '!': // declaration
                                javaNestMode = JAVA_DECLARATION;
                                state = ISI_JSPTAG_JAVA_JUMP;
                                offset++;
                                tokenContextPath = JspMultiTokenContext.jspTagContextPath;
                                return JspTagTokenContext.SYMBOL2;
                            case '=': // expression
                                javaNestMode = JAVA_EXPRESSION;
                                state = ISI_JSPTAG_JAVA_JUMP;
                                offset++;
                                tokenContextPath = JspMultiTokenContext.jspTagContextPath;
                                return JspTagTokenContext.SYMBOL2;
                            default: // assume this is a scriptlet
                                javaNestMode = JAVA_SCRIPTLET;
                                state = ISI_JSPTAG_JAVA_JUMP;
                                tokenContextPath = JspMultiTokenContext.jspTagContextPath;
                                return JspTagTokenContext.SYMBOL2;
                        } // switch (actChar)
                        // break; - not reached

                    case ISI_JSPTAG_JAVA_JUMP: 
                        nestMode = MODE_HOST_JSPTAG_JAVA;
                        state = ISI_LANGUAGE;
                        transferMasterToSlave(javaSyntax, null);
                        if (javaStateInfo == null) {
                            javaStateInfo = javaSyntax.createStateInfo();
                        }
                        javaSyntax.storeState(javaStateInfo);
                        break;
            
                    case ISI_JSPTAG_EL: // switch from JspTagSyntax to ELSyntax
                        switch (actChar) {
                            case '$':
                            case '#':    
                                state = ISI_JSPTAG_EL_D;
                                break;
                            default:  
                                Logger.getLogger("global").log(Level.INFO, null, new Exception("bad hostsyntax analyzer"));  // NOI18N
                        } // switch (actChar)
                        break;

                    case ISI_JSPTAG_EL_D: 
                        switch (actChar) {
                            case '{': // EL expression
                                state = ISI_JSPTAG_EL_JUMP;
                                offset++;
                                tokenContextPath = JspMultiTokenContext.elContextPath;
                                return ELTokenContext.EL_DELIM;
                            default:  
                                Logger.getLogger("global").log(Level.INFO, null, new Exception("bad hostsyntax analyzer"));  // NOI18N
                        } // switch (actChar)
                        break;

                    case ISI_JSPTAG_EL_JUMP: 
                        nestMode = MODE_HOST_JSPTAG_EL;
                        state = ISI_LANGUAGE;
                        transferMasterToSlave(elSyntax, null);
                        //elSyntax.load(null, buffer, offset, stopOffset - offset, lastBuffer);
                        if (elStateInfo == null) {
                            elStateInfo = elSyntax.createStateInfo();
                        }
                        elSyntax.storeState(elStateInfo);
                        break;

                    // switching from Java back to JSPTAG
                    case ISI_JAVA1_SWITCH:
                        switch (actChar) {
                            case '%':
                                state = ISI_JAVA1_PC;
                                break;
                            default:  
                                Logger.getLogger("global").log(Level.INFO, null, new Exception("bad hostsyntax analyzer"));  // NOI18N
                        } // switch (actChar)
                        break;                  
            
                    case ISI_JAVA1_PC:
                        switch (actChar) {
                            case '>':
                                state = ISI_JAVA1_JUMP;
                                offset++;
                                tokenContextPath = JspMultiTokenContext.jspTagContextPath;
                                return JspTagTokenContext.SYMBOL2;
                            default:  
                                Logger.getLogger("global").log(Level.INFO, null, new Exception("bad hostsyntax analyzer"));  // NOI18N
                        }
          
                    case ISI_JAVA1_JUMP:
                        nestMode = MODE_HOST_JSPTAG;
                        jspTagStateInfo.setPreScan(0);
                        state = ISI_LANGUAGE;
                        javaStateInfo = null;
                        break;
                                               
                    // switching from Java back to host                                     
                    case ISI_JAVA2_SWITCH:
                        switch (actChar) {
                            case '%':
                                state = ISI_JAVA2_PC;
                                break;
                            default:  
                                Logger.getLogger("global").log(Level.INFO, null, new Exception("bad hostsyntax analyzer"));  // NOI18N
                        } // switch (actChar)
                        break;                  
            
                    case ISI_JAVA2_PC:

                        switch (actChar) {
                            case '>':
                                state = ISI_JAVA2_JUMP;
                                offset++;
                                tokenContextPath = JspMultiTokenContext.jspTagContextPath;
                                return JspTagTokenContext.SYMBOL2;
                            default:  
                                Logger.getLogger("global").log(Level.INFO, null, new Exception("bad hostsyntax analyzer"));  // NOI18N
                        }
          
                    case ISI_JAVA2_JUMP:
                        nestMode = MODE_HOST;
                        hostStateInfo.setPreScan(0);
                        state = ISI_LANGUAGE;
                        javaStateInfo = null;
                        break;

                    // switching from EL back to JSPTAG
                    case ISI_EL1_SWITCH:
                        switch (actChar) {
                            case '}':
                                state = ISI_EL1_JUMP;
                                offset++;
                                tokenContextPath = JspMultiTokenContext.elContextPath;
                                return ELTokenContext.EL_DELIM;
                            default:  
                                Logger.getLogger("global").log(Level.INFO, null, new Exception("bad hostsyntax analyzer"));  // NOI18N
                        } // switch (actChar)
                        break;                  
            
                    case ISI_EL1_JUMP:
                        nestMode = MODE_HOST_JSPTAG;
                        jspTagStateInfo.setPreScan(0);
                        state = ISI_LANGUAGE;
                        elStateInfo = null;
                        break;
                                               
                    // switching from EL back to host
                    case ISI_EL2_SWITCH:
                        switch (actChar) {
                            case '}':
                                state = ISI_EL2_JUMP;
                                offset++;
                                tokenContextPath = JspMultiTokenContext.elContextPath;
                                return ELTokenContext.EL_DELIM;
                            default:  
                                Logger.getLogger("global").log(Level.INFO, null, new Exception("bad hostsyntax analyzer"));  // NOI18N
                        } // switch (actChar)
                        break;                  
            
                    case ISI_EL2_JUMP:
                        nestMode = MODE_HOST;
                        hostStateInfo.setPreScan(0);
                        state = ISI_LANGUAGE;
                        elStateInfo = null;
                        break;
                                               
                }

                if (state == ISI_LANGUAGE)
                    break;

                offset++;
            } // end of while(offset...)

            if (state != ISI_LANGUAGE) {
                /** At this stage there's no more text in the scanned buffer.
                * Scanner first checks whether this is completely the last
                * available buffer.
                */
                if (lastBuffer) {
                    switch(state) {
                        case ISI_HOST_JSPTAG:
                        case ISI_HOST_JAVA:
                        case ISI_HOST_JAVA_LT:
                        case ISI_HOST_JAVA_LT_PC:
                        case ISI_HOST_JAVA_JUMP:
                        case ISI_JSPTAG_JAVA:
                        case ISI_JSPTAG_JAVA_LT:
                        case ISI_JSPTAG_JAVA_LT_PC:
                        case ISI_JSPTAG_JAVA_JUMP:
                        case ISI_JAVA1_SWITCH:
                        case ISI_JAVA1_PC:
                        case ISI_JAVA1_JUMP:
                        case ISI_JAVA2_SWITCH:
                        case ISI_JAVA2_PC:
                        case ISI_JAVA2_JUMP:
                            tokenContextPath = JspMultiTokenContext.jspTagContextPath;
                            return JspTagTokenContext.SYMBOL2;
                            
                        case ISI_HOST_EL:
                        case ISI_HOST_EL_D:
                        case ISI_HOST_EL_JUMP:
                        case ISI_JSPTAG_EL:
                        case ISI_JSPTAG_EL_D:
                        case ISI_JSPTAG_EL_JUMP:
                        case ISI_EL1_SWITCH:
                        case ISI_EL1_JUMP:
                        case ISI_EL2_SWITCH:
                        case ISI_EL2_JUMP:
                            tokenContextPath = JspMultiTokenContext.elContextPath;
                            return ELTokenContext.EL_DELIM;
                    } // switch (state)
                } // if lastBuffer
                return null;
            }  // if state != ISI_LANGUAGE - inner
        } // if state != ISI_LANGUAGE - outer
    
        //</editor-fold>
        
        // now state is ISI_LANGUAGE
        TokenID slaveTokenID = null;
        TokenID returnedTokenID;

        int slaveOffset;
        int canBe;
        boolean firstTokenNotRead = ((tokenOffset == offset) || (firstTokenID == null));
        boolean equalPositions = (tokenOffset == offset);

        switch (nestMode) {
            // BIG BRANCH - we are in the HOST mode
            //<editor-fold defaultstate="collapsed" desc="MODE_HOST & MODE_JSPTAG">
            case MODE_HOST: 
                if (hostStateInfo == null) {
                    hostStateInfo = hostSyntax.createStateInfo();
                    hostSyntax.reset();
                    hostSyntax.storeState(hostStateInfo);
                }
                    
/*if (debug) {
System.out.print("NOT EQUAL tokenOffset=" + tokenOffset + ", offset=" + offset + ", tokenPart='");   
for (int i = tokenOffset; i<offset;i++) System.out.print(buffer[i]);
System.out.println("', firstTokenID=" + firstTokenID + ", firstTokenLength=" + firstTokenLength);
System.out.println("hoststate " + hostStateInfo.getState() + ", prescan=" + hostStateInfo.getPreScan());
System.out.println("helpstate " + helpHostStateInfo.getState() + ", prescan=" + helpHostStateInfo.getPreScan());
}         */
        
//if (equalPositions && (hostStateInfo.getPreScan() != 0))
//new Exception("prescan should be 0 !!").printStackTrace();
//if (debug)
//System.out.println("html state at offset " + ((BaseStateInfo)hostStateInfo).toString(this));
                if (firstTokenNotRead) {
                    // the first step - parse the first token of the slave
                    transferMasterToSlave(hostSyntax, hostStateInfo);
                    returnedTokenID = hostSyntax.nextToken();
                    slaveTokenID = returnedTokenID;
                    tokenContextPath = JspMultiTokenContext.context.getContextPath(
                        hostSyntax.getTokenContextPath());
                    slaveOffset = hostSyntax.getOffset();
                    firstTokenID = slaveTokenID;
                    firstTokenLength = hostSyntax.getTokenLength();
                    if (slaveTokenID == null) {
                        offset = slaveOffset;
                        firstTokenLength = -1;
                        // need to property transfer states
                        if (equalPositions) {
                            helpHostStateInfo = hostStateInfo;
                            hostStateInfo = hostSyntax.createStateInfo();
                            hostSyntax.storeState(hostStateInfo);
                        }
                        else {
                            if (hostStateInfo == null) {
                                hostStateInfo = hostSyntax.createStateInfo();
                            }
                            hostSyntax.storeState(hostStateInfo);
                        }
//if (debug)
//System.out.println("returnuju (1) " + null + " at " + offset);            
                        return null;
                    }  
                    if (returnedTokenID.getNumericID() == HtmlTokenContext.BLOCK_COMMENT_ID && isXMLSyntax() )
                        canBe = DELIMCHECK_NO;
                    else{
                        // find out if the token could contain a starting symbol for JspTag or Java
                        canBe = canBeHostDelimiter(tokenOffset, slaveOffset, slaveOffset, false);
                    }
                    if (canBe == DELIMCHECK_NO) { // do not switch 
                        offset = slaveOffset;
                        if (hostStateInfo == null) {
                            hostStateInfo = hostSyntax.createStateInfo();
                        }
                        hostSyntax.storeState(hostStateInfo);
//if (debug)
//System.out.println("returnuju (2) " + slaveTokenID + " at " + offset);            
                        return slaveTokenID;
                    }
                    // store the state
                    hostSyntax.storeState(firstHostStateInfo);
//if (firstHostStateInfo == hostStateInfo)
//new Exception("stateinfo instance conflict").printStackTrace();
                }
                else { // first position read - offsets different and firstTokenID is a valid token
                    transferMasterToSlave(hostSyntax, hostStateInfo);
                    canBe = DELIMCHECK_PART;
                }

                // we have successfully read the first token, the following statements hold:
                // - canBe is not DELIMCHECK_NO
                // - firstTokenID and firstTokenLength are meaningful
                // - if (equalPositions) then firstHostStateInfo is meaningful
//if (firstTokenID == null) {
//new Exception("invalid firstTokenID !!!!!!!").printStackTrace();
//}  
                while (canBe == DELIMCHECK_PART) { // need another token
                    // now get the new token
                    returnedTokenID = hostSyntax.nextToken();
                    slaveTokenID = returnedTokenID;
                    tokenContextPath = JspMultiTokenContext.context.getContextPath(
                        hostSyntax.getTokenContextPath());
                    slaveOffset = hostSyntax.getOffset();

                    if ((slaveTokenID == null) && lastBuffer) {
                        // ask about the delimiter, but with lastPart=true
                        canBe = canBeHostDelimiter(tokenOffset, slaveOffset, tokenOffset + firstTokenLength, true);
                        if (canBe != DELIMCHECK_PART)
                            break;
                    }

                    if (slaveTokenID == null) {
                        if (lastBuffer) {
                            canBe = DELIMCHECK_NO;
                            break;
                        }
                        offset = slaveOffset;
                        if (equalPositions) {
                            helpHostStateInfo = hostStateInfo;
                            hostStateInfo = hostSyntax.createStateInfo();
                            hostSyntax.storeState(hostStateInfo);
                        }
                        else {
                            if (hostStateInfo == null) {
                                hostStateInfo = hostSyntax.createStateInfo();
                            }
                            hostSyntax.storeState(hostStateInfo);
                        }
//if (debug)
//System.out.println("returnuju (3) " + null + " at " + offset);            
                        return null;
                    }
                    canBe = canBeHostDelimiter(tokenOffset, slaveOffset, tokenOffset + firstTokenLength, false);
                }
        
                // now canBe is not DELIMCHECK_PART
                // now we have read possibly more tokens and know whether to switch or not
                if (canBe == DELIMCHECK_NO) { // do not switch 
                    offset = tokenOffset + firstTokenLength;
                    if (equalPositions) {
                        hostStateInfo = firstHostStateInfo;
                        firstHostStateInfo = hostSyntax.createStateInfo();
                    }
                    else {
//if (debug)
//System.out.println("= imagine - rescan called !!");          
//if (helpHostStateInfo.getPreScan() != 0)
//new Exception("help prescan should be 0 !!").printStackTrace();
                        // we need to rescan the first token to find out the state
                        // now helpHostStateInfo is useful
                        hostSyntax.load(helpHostStateInfo, buffer, tokenOffset, stopOffset - tokenOffset, lastBuffer, -1);
                        returnedTokenID = hostSyntax.nextToken();
//if (tokenOffset != hostSyntax.getTokenOffset())
//new Exception("starts of tokens do not match").printStackTrace();            
                        slaveTokenID = returnedTokenID;
                        tokenContextPath = JspMultiTokenContext.context.getContextPath(
                            hostSyntax.getTokenContextPath());
                        if (hostStateInfo == null) {
                            hostStateInfo = hostSyntax.createStateInfo();
                        }
                        hostSyntax.storeState(hostStateInfo);

                    }
//if (debug)
//System.out.println("returnuju (4) " + firstTokenID + " at " + offset);            

                    return firstTokenID;
                }
                else { // we found a delimiter
//if (canBe >= tokenOffset + firstTokenLength)        
//new Exception("value of canBe is invalid !!!!!!!").printStackTrace();
                    // now use the saved state
                    if (equalPositions) {
                        hostSyntax.load(hostStateInfo, buffer, tokenOffset, canBe - tokenOffset, true, -1);
                    }
                    else {
                        hostSyntax.load(helpHostStateInfo, buffer, tokenOffset, canBe - tokenOffset, true, -1);
                    }
                    returnedTokenID = hostSyntax.nextToken();
                    tokenContextPath = JspMultiTokenContext.context.getContextPath(
                                    hostSyntax.getTokenContextPath());
                    // we got the StateInfo, which is why we did all this
                    if (hostStateInfo == null) {
                        hostStateInfo = hostSyntax.createStateInfo();
                    }
//if (debug)
//System.out.println("html state before saving back " + ((BaseStateInfo)hostStateInfo).toString(this));
                    hostSyntax.storeState(hostStateInfo);
                    hostStateInfo.setPreScan(0);
//if (hostSyntax.getOffset() != canBe)
//new Exception("bad number of characters parsed !!!").printStackTrace();          
                    offset = canBe;
/*if (debug) {
System.out.println("switching from HOST to JSPTAG at offset " + offset + ", hostState " + ((BaseStateInfo)hostStateInfo).toString(this));
System.out.println("offset of the returned (a)" + hostSyntax.getOffset());          
System.out.println("found delimiter at " + offset);          
System.out.println("returnuju (5) " + firstTokenID + " at " + offset);            
}*/
                    return firstTokenID;
                }
                //break; //- not reached
        
        
            // BIG BRANCH - we are in the HOST_JSPTAG mode
            case MODE_HOST_JSPTAG:
                // check if the JSP tag hasn't finished on its own will
                if ((jspTagStateInfo != null) && (jspTagStateInfo.getState() == JspTagSyntax.ISA_END_JSP)) {
                    // give up control
                    jspTagStateInfo = null;
                    nestMode = MODE_HOST;
//if (debug) {
//System.out.println("switching back to HOST from JSPTAG at offset " + offset + ", hostState " + ((BaseStateInfo)hostStateInfo).toString(this));
//System.out.println("returnuju (6) " /*+ JspTagSyntax.TEXT + jspTagSyntaxInfo.tokenIDShift */+ " at " + offset);            
//}
                    tokenContextPath = JspMultiTokenContext.jspTagContextPath;
                    return JspTagTokenContext.TEXT;
                }

                if (jspTagStateInfo == null) {
                    jspTagStateInfo = jspTagSyntax.createStateInfo();
                    jspTagSyntax.reset();
                    jspTagSyntax.storeState(jspTagStateInfo);
                }
                if (firstTokenNotRead) {
                    // the first step - parse the first token of the slave
                    transferMasterToSlave(jspTagSyntax, jspTagStateInfo);
                    returnedTokenID = jspTagSyntax.nextToken();
                    if (returnedTokenID == JspTagTokenContext.AFTER_UNEXPECTED_LT) {
                        // give up control
                        jspTagStateInfo = null;
                        nestMode = MODE_HOST;
                        hostStateInfo.setPreScan(0);
                        tokenContextPath = JspMultiTokenContext.jspTagContextPath;
//System.out.println("switch to host " + returnedTokenID + " at " + offset);            
                        return returnedTokenID;
                    }   
                    slaveTokenID = returnedTokenID;
                    tokenContextPath = JspMultiTokenContext.jspTagContextPath;
//if (debug)
//System.out.println("first JSPtoken returned '" + getToken(jspTagSyntax) + "' id " + slaveTokenID);
                    slaveOffset = jspTagSyntax.getOffset();
                    firstTokenID = slaveTokenID;
                    firstTokenLength = jspTagSyntax.getTokenLength();
                    if (slaveTokenID == null) {
                        offset = slaveOffset;
                        firstTokenLength = -1;
                        // need to properly transfer states
                        if (equalPositions) {
                            helpJspTagStateInfo = jspTagStateInfo;
                            jspTagStateInfo = jspTagSyntax.createStateInfo();
                            jspTagSyntax.storeState(jspTagStateInfo);
                        }
                        else {
                            if (jspTagStateInfo == null) {
                                jspTagStateInfo = jspTagSyntax.createStateInfo();
                            }
                            jspTagSyntax.storeState(jspTagStateInfo);
                        }
//if (debug)
//System.out.println("returnuju (7) " + null + " at " + offset);            
                        return null;
                    }  
                    // find out if the token could contain a starting symbol for Java
                    canBe = canBeJspTagDelimiter(tokenOffset, slaveOffset, slaveOffset, false, returnedTokenID == JspTagTokenContext.COMMENT);
                    if (canBe == DELIMCHECK_NO) { // do not switch 
                        offset = slaveOffset;
                        if (jspTagStateInfo == null) {
                            jspTagStateInfo = jspTagSyntax.createStateInfo();
                        }
                        jspTagSyntax.storeState(jspTagStateInfo);
//if (debug)
//System.out.println("returnuju (8) " + slaveTokenID + " at " + offset);            
                        return slaveTokenID;
                    }
                    // store the state
                    jspTagSyntax.storeState(firstJspTagStateInfo);
                }
                else { // first position read - offsets different and firstTokenID is a valid token
                    transferMasterToSlave(jspTagSyntax, jspTagStateInfo);
                    canBe = DELIMCHECK_PART;
                }

                // we have successfully read the first token, the following statements hold:
                // - canBe is not DELIMCHECK_NO
                // - firstTokenID and firstTokenLength are meaningful
                // - if (equalPositions) then firstJspTagStateInfo is meaningful
//if (firstTokenID == null) {
//new Exception("invalid firstTokenID !!!!!!!").printStackTrace();
//}  
                while (canBe == DELIMCHECK_PART) { // need another token
                    // now get the new token
                    returnedTokenID = jspTagSyntax.nextToken();
                    slaveTokenID = returnedTokenID;
                    tokenContextPath = JspMultiTokenContext.jspTagContextPath;
                    slaveOffset = jspTagSyntax.getOffset();

                    if ((slaveTokenID == null) && lastBuffer) {
                        // ask about the delimiter, but with lastPart=true
                        canBe = canBeJspTagDelimiter(tokenOffset, slaveOffset, tokenOffset + firstTokenLength, true, returnedTokenID == JspTagTokenContext.COMMENT);
                        if (canBe != DELIMCHECK_PART)
                            break;
                    }

                    if (slaveTokenID == null) {
                        if (lastBuffer) {
                            canBe = DELIMCHECK_NO;
                            break;
                        }
                        offset = slaveOffset;
                        if (equalPositions) {
                            helpJspTagStateInfo = jspTagStateInfo;
                            jspTagStateInfo = jspTagSyntax.createStateInfo();
                            jspTagSyntax.storeState(jspTagStateInfo);
                        }
                        else {
                            if (jspTagStateInfo == null) {
                                jspTagStateInfo = jspTagSyntax.createStateInfo();
                            }
                            jspTagSyntax.storeState(jspTagStateInfo);
                        }
//if (debug)
//System.out.println("returnuju (9) " + null + " at " + offset);            
                        return null;
                    }
                    canBe = canBeJspTagDelimiter(tokenOffset, slaveOffset, tokenOffset + firstTokenLength, false, returnedTokenID == JspTagTokenContext.COMMENT);
                }
        
                // now canBe is not DELIMCHECK_PART
                // now we have read possibly more tokens and know whether to switch or not
                if (canBe == DELIMCHECK_NO) { // do not switch 
                    offset = tokenOffset + firstTokenLength;
                    if (equalPositions) {
                        jspTagStateInfo = firstJspTagStateInfo;
                        firstJspTagStateInfo = jspTagSyntax.createStateInfo();
                    }
                    else {
//if (debug)
//System.out.println("= imagine - rescan called !!");          
                        // we need to rescan the first token to find out the state
                        // now helpJspTagStateInfo is useful
                        jspTagSyntax.load(helpJspTagStateInfo, buffer, tokenOffset, stopOffset - tokenOffset, lastBuffer, -1);
                        returnedTokenID = jspTagSyntax.nextToken();
                        tokenContextPath = JspMultiTokenContext.jspTagContextPath;
                        slaveTokenID = returnedTokenID;
                        if (jspTagStateInfo == null) {
                            jspTagStateInfo = jspTagSyntax.createStateInfo();
                        }
                        jspTagSyntax.storeState(jspTagStateInfo);
//if (slaveTokenID != firstTokenID)
//new Exception("token ID does not match !!!!!!!").printStackTrace();
//if (offset != jspTagSyntax.getOffset())
//new Exception("offset does not match !!!!!!!").printStackTrace();
                    }
//if (debug)
//System.out.println("returnuju (10) " + firstTokenID + " at " + offset);            
                    return firstTokenID;
                }
                else { // we found a delimiter
//if (canBe >= tokenOffset + firstTokenLength)        
//new Exception("value of canBe is invalid !!!!!!!").printStackTrace();
                    // now use the saved state
                    if (equalPositions) {
                        jspTagSyntax.load(jspTagStateInfo, buffer, tokenOffset, canBe - tokenOffset, true, -1);
                    }
                    else {
                        jspTagSyntax.load(helpJspTagStateInfo, buffer, tokenOffset, canBe - tokenOffset, true, -1);
                    }
                    returnedTokenID = jspTagSyntax.nextToken();
                    tokenContextPath = JspMultiTokenContext.jspTagContextPath;
                    // we got the StateInfo, which is why we did all this
                    if (jspTagStateInfo == null) {
                        jspTagStateInfo = jspTagSyntax.createStateInfo();
                    }
                    jspTagSyntax.storeState(jspTagStateInfo);
                    jspTagStateInfo.setPreScan(0);
//if (jspTagSyntax.getOffset() != canBe)
//new Exception("bad number of characters parsed !!!").printStackTrace();          
                    offset = canBe;
//if (debug) {
//System.out.println("offset of the returned (a)" + jspTagSyntax.getOffset());          
//System.out.println("found delimiter at " + offset);          
//System.out.println("returnuju (11) " + firstTokenID + " at " + offset);            
//}
                    return firstTokenID;
                }
                //break; //- not reached
        
                //</editor-fold>
            case MODE_HOST_JSPTAG_JAVA:
            case MODE_HOST_JAVA: 
                if (javaStateInfo == null) {
                    javaStateInfo = javaSyntax.createStateInfo();
                    javaSyntax.reset();
                    javaSyntax.storeState(javaStateInfo);
                }
                if (firstTokenNotRead) {
                    // the first step - parse the first token of the slave
                    transferMasterToSlave(javaSyntax, javaStateInfo);
                    returnedTokenID = javaSyntax.nextToken();
                    tokenContextPath = getJavaTokenContextPath();
                    slaveTokenID = returnedTokenID;
                    slaveOffset = javaSyntax.getOffset();
                    firstTokenID = slaveTokenID;
                    firstTokenLength = javaSyntax.getTokenLength();
                    if (slaveTokenID == null) {
                        offset = slaveOffset;
                        firstTokenLength = -1;
                        // need to property transfer states
                        if (equalPositions) {
                            helpJavaStateInfo = javaStateInfo;
                            javaStateInfo = javaSyntax.createStateInfo();
                            javaSyntax.storeState(javaStateInfo);
                        }
                        else {
                            if (javaStateInfo == null) {
                                javaStateInfo = javaSyntax.createStateInfo();
                            }
                            javaSyntax.storeState(javaStateInfo);
                        }
                        return null;
                    }  
                    // find out if the token could contain an ending symbol for a Java block
                    canBe = canBeJavaDelimiter(tokenOffset, slaveOffset, slaveOffset, false, nestMode);
                    if (canBe == DELIMCHECK_NO) { // do not switch 
                        offset = slaveOffset;
                        if (javaStateInfo == null) {
                            javaStateInfo = javaSyntax.createStateInfo();
                        }
                        javaSyntax.storeState(javaStateInfo);
                        return slaveTokenID;
                    }
                    // store the state
                    javaSyntax.storeState(firstJavaStateInfo);
                }
                else { // first position read - offsets different and firstTokenID is a valid token
                    transferMasterToSlave(javaSyntax, javaStateInfo);
                    canBe = DELIMCHECK_PART;
                }

                // we have successfully read the first token, the following statements hold:
                // - canBe is not DELIMCHECK_NO
                // - firstTokenID and firstTokenLength are meaningful
                // - if (equalPositions) then firstJavaStateInfo is meaningful
                while (canBe == DELIMCHECK_PART) { // need another token
                    // now get the new token
                    returnedTokenID = javaSyntax.nextToken();
                    slaveTokenID = returnedTokenID;
                    tokenContextPath = getJavaTokenContextPath();
                    slaveOffset = javaSyntax.getOffset();

                    if ((slaveTokenID == null) && lastBuffer) {
                        // ask about the delimiter, but with lastPart=true
                        canBe = canBeJavaDelimiter(tokenOffset, slaveOffset, tokenOffset + firstTokenLength, true, nestMode);
                        if (canBe != DELIMCHECK_PART)
                            break;
                    }

                    if (slaveTokenID == null) {
                        if (lastBuffer) {
                            canBe = DELIMCHECK_NO;
                            break;
                        }
                        offset = slaveOffset;
                        if (equalPositions) {
                            helpJavaStateInfo = javaStateInfo;
                            javaStateInfo = javaSyntax.createStateInfo();
                            javaSyntax.storeState(javaStateInfo);
                        }
                        else {
                            if (javaStateInfo == null) {
                                javaStateInfo = javaSyntax.createStateInfo();
                            }
                            javaSyntax.storeState(javaStateInfo);
                        }
                        return null;
                    }
                    canBe = canBeJavaDelimiter(tokenOffset, slaveOffset, tokenOffset + firstTokenLength, false, nestMode);
                }
        
                // now canBe is not DELIMCHECK_PART
                // now we have read possibly more tokens and know whether to switch or not
                if (canBe == DELIMCHECK_NO) { // do not switch 
                    offset = tokenOffset + firstTokenLength;
                    if (equalPositions) {
                        javaStateInfo = firstJavaStateInfo;
                        firstJavaStateInfo = javaSyntax.createStateInfo();
                    }
                    else {
                        // we need to rescan the first token to find out the state
                        // now helpJavaStateInfo is useful
                        javaSyntax.load(helpJavaStateInfo, buffer, tokenOffset, stopOffset - tokenOffset, lastBuffer, -1);
                        returnedTokenID = javaSyntax.nextToken();
                        slaveTokenID = returnedTokenID;
                        tokenContextPath = getJavaTokenContextPath();
                        if (javaStateInfo == null) {
                            javaStateInfo = javaSyntax.createStateInfo();
                        }
                        javaSyntax.storeState(javaStateInfo);
                    }
                    return firstTokenID;
                }
                else { // we found a delimiter
                    // now use the saved state
                    if (equalPositions) {
                        javaSyntax.load(javaStateInfo, buffer, tokenOffset, canBe - tokenOffset, true, -1);
                    }
                    else {
                        javaSyntax.load(helpJavaStateInfo, buffer, tokenOffset, canBe - tokenOffset, true, -1);
                    }
                    returnedTokenID = javaSyntax.nextToken();
                    tokenContextPath = getJavaTokenContextPath();
                    // we got the StateInfo, which is why we did all this
                    if (javaStateInfo == null) {
                        javaStateInfo = javaSyntax.createStateInfo();
                    }
                    javaSyntax.storeState(javaStateInfo);
                    javaStateInfo.setPreScan(0);
                    offset = canBe;
                    
                    return firstTokenID;
                }
                // break; //- not reached
                
                //<editor-fold defaultstate="collapsed" desc="MODE_HOST_EL">
            case MODE_HOST_JSPTAG_EL:
            case MODE_HOST_EL: 
                if (elStateInfo == null) {
                    elStateInfo = elSyntax.createStateInfo();
                    elSyntax.reset();
                    elSyntax.storeState(elStateInfo);
                }
                if (firstTokenNotRead) {
                    // the first step - parse the first token of the slave
                    transferMasterToSlave(elSyntax, elStateInfo);
                    returnedTokenID = elSyntax.nextToken();
                    tokenContextPath = JspMultiTokenContext.elContextPath;
                    slaveTokenID = returnedTokenID;
                    slaveOffset = elSyntax.getOffset();
                    firstTokenID = slaveTokenID;
                    firstTokenLength = elSyntax.getTokenLength();
                    if (slaveTokenID == null) {
                        offset = slaveOffset;
                        firstTokenLength = -1;
                        // need to property transfer states
                        if (equalPositions) {
                            helpELStateInfo = elStateInfo;
                            elStateInfo = elSyntax.createStateInfo();
                            elSyntax.storeState(elStateInfo);
                        }
                        else {
                            if (elStateInfo == null) {
                                elStateInfo = elSyntax.createStateInfo();
                            }
                            elSyntax.storeState(elStateInfo);
                        }
//if (debug)
//System.out.println("returnuju (12.5) " + null + " at " + offset);            
                        return null;
                    }  
                    // find out if the token could contain an ending symbol for a EL block
                    canBe = canBeELDelimiter(tokenOffset, slaveOffset, slaveOffset, false, nestMode);
                    if (canBe == DELIMCHECK_NO) { // do not switch 
                        offset = slaveOffset;
                        if (elStateInfo == null) {
                            elStateInfo = elSyntax.createStateInfo();
                        }
                        elSyntax.storeState(elStateInfo);
//if (debug)
//System.out.println("returnuju (13.5) " + slaveTokenID + " at " + offset);            
                        return slaveTokenID;
                    }
                    // store the state
                    elSyntax.storeState(firstELStateInfo);
                }
                else { // first position read - offsets different and firstTokenID is a valid token
                    transferMasterToSlave(elSyntax, elStateInfo);
                    canBe = DELIMCHECK_PART;
                }

                // we have successfully read the first token, the following statements hold:
                // - canBe is not DELIMCHECK_NO
                // - firstTokenID and firstTokenLength are meaningful
                // - if (equalPositions) then firstELStateInfo is meaningful
//if (firstTokenID == null) {
//new Exception("invalid firstTokenID !!!!!!!").printStackTrace();
//}  
                while (canBe == DELIMCHECK_PART) { // need another token
                    // now get the new token
                    returnedTokenID = elSyntax.nextToken();
                    slaveTokenID = returnedTokenID;
                    tokenContextPath = JspMultiTokenContext.elContextPath;
                    slaveOffset = elSyntax.getOffset();

                    if ((slaveTokenID == null) && lastBuffer) {
                        // ask about the delimiter, but with lastPart=true
                        canBe = canBeELDelimiter(tokenOffset, slaveOffset, tokenOffset + firstTokenLength, true, nestMode);
                        if (canBe != DELIMCHECK_PART)
                            break;
                    }

                    if (slaveTokenID == null) {
                        if (lastBuffer) {
                            canBe = DELIMCHECK_NO;
                            break;
                        }
                        offset = slaveOffset;
                        if (equalPositions) {
                            helpELStateInfo = elStateInfo;
                            elStateInfo = elSyntax.createStateInfo();
                            elSyntax.storeState(elStateInfo);
                        }
                        else {
                            if (elStateInfo == null) {
                                elStateInfo = elSyntax.createStateInfo();
                            }
                            elSyntax.storeState(elStateInfo);
                        }
//if (debug)
//System.out.println("returnuju (14) " + null + " at " + offset);            
                        return null;
                    }
                    canBe = canBeELDelimiter(tokenOffset, slaveOffset, tokenOffset + firstTokenLength, false, nestMode);
                }
        
                // now canBe is not DELIMCHECK_PART
                // now we have read possibly more tokens and know whether to switch or not
                if (canBe == DELIMCHECK_NO) { // do not switch 
                    offset = tokenOffset + firstTokenLength;
                    if (equalPositions) {
                        elStateInfo = firstELStateInfo;
                        firstELStateInfo = elSyntax.createStateInfo();
                    }
                    else {
//if (debug)
//System.out.println("= imagine - rescan called !!");          
                        // we need to rescan the first token to find out the state
                        // now helpELStateInfo is useful
                        elSyntax.load(helpELStateInfo, buffer, tokenOffset, stopOffset - tokenOffset, lastBuffer, -1);
                        returnedTokenID = elSyntax.nextToken();
                        slaveTokenID = returnedTokenID;
                        tokenContextPath = JspMultiTokenContext.elContextPath;
                        if (elStateInfo == null) {
                            elStateInfo = elSyntax.createStateInfo();
                        }
                        elSyntax.storeState(elStateInfo);
//if (slaveTokenID != firstTokenID)
//new Exception("token ID does not match !!!!!!!").printStackTrace();
//if (offset != elSyntax.getOffset())
//new Exception("offset does not match !!!!!!!").printStackTrace();
                    }
//if (debug)
//System.out.println("returnuju (15.5) " + firstTokenID + " at " + offset);            
                    return firstTokenID;
                }
                else { // we found a delimiter
//if (canBe >= tokenOffset + firstTokenLength)        
//new Exception("value of canBe is invalid !!!!!!!").printStackTrace();
                    // now use the saved state
                    if (equalPositions) {
                        elSyntax.load(elStateInfo, buffer, tokenOffset, canBe - tokenOffset, true, -1);
                    }
                    else {
                        elSyntax.load(helpELStateInfo, buffer, tokenOffset, canBe - tokenOffset, true, -1);
                    }
                    returnedTokenID = elSyntax.nextToken();
                    tokenContextPath = JspMultiTokenContext.elContextPath;
                    // we got the StateInfo, which is why we did all this
                    if (elStateInfo == null) {
                        elStateInfo = elSyntax.createStateInfo();
                    }
                    elSyntax.storeState(elStateInfo);
                    elStateInfo.setPreScan(0);
//if (elSyntax.getOffset() != canBe)
//new Exception("bad number of characters parsed !!!").printStackTrace();          
                    offset = canBe;
/*if (debug) {
System.out.println("offset of the returned (a)" + elSyntax.getOffset());          
System.out.println("found delimiter at " + offset);          
System.out.println("returnuju (16.5) " + firstTokenID + " at " + offset);            
}*/
                    return firstTokenID;
                }
                // break; //- not reached
                
                //</editor-fold>
            default:  
                Logger.getLogger("global").log(Level.INFO, null, new Exception("bad nestmode"));  // NOI18N
                tokenContextPath = JspMultiTokenContext.contextPath;
                return JspMultiTokenContext.ERROR; // !!! don't know what to return
        }
    }