in java/java.editor.lib/src/org/netbeans/editor/ext/java/JavaSyntax.java [93:788]
protected TokenID parseToken() {
char actChar;
while(offset < stopOffset) {
actChar = buffer[offset];
switch (state) {
case INIT:
switch (actChar) {
case '"': // NOI18N
state = ISI_STRING;
break;
case '\'':
state = ISI_CHAR;
break;
case '/':
state = ISA_SLASH;
break;
case '=':
state = ISA_EQ;
break;
case '>':
state = ISA_GT;
break;
case '<':
state = ISA_LT;
break;
case '+':
state = ISA_PLUS;
break;
case '-':
state = ISA_MINUS;
break;
case '*':
state = ISA_STAR;
break;
case '|':
state = ISA_PIPE;
break;
case '%':
state = ISA_PERCENT;
break;
case '&':
state = ISA_AND;
break;
case '^':
state = ISA_XOR;
break;
case '~':
offset++;
return JavaTokenContext.NEG;
case '!':
state = ISA_EXCLAMATION;
break;
case '0':
state = ISA_ZERO;
break;
case '.':
state = ISA_DOT;
break;
case ',':
offset++;
return JavaTokenContext.COMMA;
case ';':
offset++;
return JavaTokenContext.SEMICOLON;
case ':':
offset++;
return JavaTokenContext.COLON;
case '?':
offset++;
return JavaTokenContext.QUESTION;
case '(':
offset++;
return JavaTokenContext.LPAREN;
case ')':
offset++;
return JavaTokenContext.RPAREN;
case '[':
offset++;
return JavaTokenContext.LBRACKET;
case ']':
offset++;
return JavaTokenContext.RBRACKET;
case '{':
offset++;
return JavaTokenContext.LBRACE;
case '}':
offset++;
return JavaTokenContext.RBRACE;
case '@': // 1.5 "@ident" annotation // NOI18N
offset++;
return JavaTokenContext.ANNOTATION;
default:
// Check for whitespace
if (Character.isWhitespace(actChar)) {
state = ISI_WHITESPACE;
break;
}
// Check for digit
if (Character.isDigit(actChar)) {
state = ISI_INT;
break;
}
// Check for identifier
if (Character.isJavaIdentifierStart(actChar)) {
state = ISI_IDENTIFIER;
break;
}
offset++;
return JavaTokenContext.INVALID_CHAR;
}
break;
case ISI_WHITESPACE: // white space
if (!Character.isWhitespace(actChar)) {
state = INIT;
return JavaTokenContext.WHITESPACE;
}
break;
case ISI_LINE_COMMENT:
switch (actChar) {
case '\n':
state = INIT;
return JavaTokenContext.LINE_COMMENT;
}
break;
case ISI_BLOCK_COMMENT:
switch (actChar) {
case '*':
state = ISA_STAR_I_BLOCK_COMMENT;
break;
//create a block comment token for each line of the comment - a performance fix for #55628
case '\n':
if(useInJsp) {
//leave the some state - we are still in the block comment,
//we just need to create a token for each line.
offset++;
return JavaTokenContext.BLOCK_COMMENT;
}
}
break;
case ISI_STRING:
switch (actChar) {
case '\\':
state = ISI_STRING_A_BSLASH;
break;
case '\n':
state = INIT;
supposedTokenID = JavaTokenContext.STRING_LITERAL;
//!!! return JavaTokenContext.INCOMPLETE_STRING_LITERAL;
return supposedTokenID;
case '"': // NOI18N
offset++;
state = INIT;
return JavaTokenContext.STRING_LITERAL;
}
break;
case ISI_STRING_A_BSLASH:
switch (actChar) {
case '"': // NOI18N
case '\\':
break;
default:
offset--;
break;
}
state = ISI_STRING;
break;
case ISI_CHAR:
switch (actChar) {
case '\\':
state = ISI_CHAR_A_BSLASH;
break;
case '\n':
state = INIT;
supposedTokenID = JavaTokenContext.CHAR_LITERAL;
// !!! return JavaTokenContext.INCOMPLETE_CHAR_LITERAL;
return supposedTokenID;
case '\'':
offset++;
state = INIT;
return JavaTokenContext.CHAR_LITERAL;
}
break;
case ISI_CHAR_A_BSLASH:
switch (actChar) {
case '\'':
case '\\':
break;
default:
offset--;
break;
}
state = ISI_CHAR;
break;
case ISI_IDENTIFIER:
if (!(Character.isJavaIdentifierPart(actChar))) {
state = INIT;
TokenID tid = matchKeyword(buffer, tokenOffset, offset - tokenOffset);
return (tid != null) ? tid : JavaTokenContext.IDENTIFIER;
}
break;
case ISA_SLASH:
switch (actChar) {
case '=':
offset++;
state = INIT;
return JavaTokenContext.DIV_EQ;
case '/':
state = ISI_LINE_COMMENT;
break;
case '*':
state = ISI_BLOCK_COMMENT;
break;
default:
state = INIT;
return JavaTokenContext.DIV;
}
break;
case ISA_EQ:
switch (actChar) {
case '=':
offset++;
return JavaTokenContext.EQ_EQ;
default:
state = INIT;
return JavaTokenContext.EQ;
}
// break;
case ISA_GT:
switch (actChar) {
case '>':
state = ISA_GTGT;
break;
case '=':
offset++;
return JavaTokenContext.GT_EQ;
default:
state = INIT;
return JavaTokenContext.GT;
}
break;
case ISA_GTGT:
switch (actChar) {
case '>':
state = ISA_GTGTGT;
break;
case '=':
offset++;
return JavaTokenContext.RSSHIFT_EQ;
default:
state = INIT;
return JavaTokenContext.RSSHIFT;
}
break;
case ISA_GTGTGT:
switch (actChar) {
case '=':
offset++;
return JavaTokenContext.RUSHIFT_EQ;
default:
state = INIT;
return JavaTokenContext.RUSHIFT;
}
// break;
case ISA_LT:
switch (actChar) {
case '<':
state = ISA_LTLT;
break;
case '=':
offset++;
return JavaTokenContext.LT_EQ;
default:
state = INIT;
return JavaTokenContext.LT;
}
break;
case ISA_LTLT:
switch (actChar) {
case '<':
state = INIT;
offset++;
return JavaTokenContext.INVALID_OPERATOR;
case '=':
offset++;
return JavaTokenContext.LSHIFT_EQ;
default:
state = INIT;
return JavaTokenContext.LSHIFT;
}
case ISA_PLUS:
switch (actChar) {
case '+':
offset++;
return JavaTokenContext.PLUS_PLUS;
case '=':
offset++;
return JavaTokenContext.PLUS_EQ;
default:
state = INIT;
return JavaTokenContext.PLUS;
}
case ISA_MINUS:
switch (actChar) {
case '-':
offset++;
return JavaTokenContext.MINUS_MINUS;
case '=':
offset++;
return JavaTokenContext.MINUS_EQ;
default:
state = INIT;
return JavaTokenContext.MINUS;
}
case ISA_STAR:
switch (actChar) {
case '=':
offset++;
return JavaTokenContext.MUL_EQ;
case '/':
offset++;
state = INIT;
return JavaTokenContext.INVALID_COMMENT_END; // '*/' outside comment
default:
state = INIT;
return JavaTokenContext.MUL;
}
case ISA_STAR_I_BLOCK_COMMENT:
switch (actChar) {
case '/':
offset++;
state = INIT;
return JavaTokenContext.BLOCK_COMMENT;
default:
offset--;
state = ISI_BLOCK_COMMENT;
break;
}
break;
case ISA_PIPE:
switch (actChar) {
case '=':
offset++;
state = INIT;
return JavaTokenContext.OR_EQ;
case '|':
offset++;
state = INIT;
return JavaTokenContext.OR_OR;
default:
state = INIT;
return JavaTokenContext.OR;
}
// break;
case ISA_PERCENT:
switch (actChar) {
case '=':
offset++;
state = INIT;
return JavaTokenContext.MOD_EQ;
default:
state = INIT;
return JavaTokenContext.MOD;
}
// break;
case ISA_AND:
switch (actChar) {
case '=':
offset++;
state = INIT;
return JavaTokenContext.AND_EQ;
case '&':
offset++;
state = INIT;
return JavaTokenContext.AND_AND;
default:
state = INIT;
return JavaTokenContext.AND;
}
// break;
case ISA_XOR:
switch (actChar) {
case '=':
offset++;
state = INIT;
return JavaTokenContext.XOR_EQ;
default:
state = INIT;
return JavaTokenContext.XOR;
}
// break;
case ISA_EXCLAMATION:
switch (actChar) {
case '=':
offset++;
state = INIT;
return JavaTokenContext.NOT_EQ;
default:
state = INIT;
return JavaTokenContext.NOT;
}
// break;
case ISA_ZERO:
switch (actChar) {
case '.':
state = ISI_DOUBLE;
break;
case 'x':
case 'X':
state = ISI_HEX;
break;
case 'l':
case 'L':
offset++;
state = INIT;
return JavaTokenContext.LONG_LITERAL;
case 'f':
case 'F':
offset++;
state = INIT;
return JavaTokenContext.FLOAT_LITERAL;
case 'd':
case 'D':
offset++;
state = INIT;
return JavaTokenContext.DOUBLE_LITERAL;
case '8': // it's error to have '8' and '9' in octal number
case '9':
state = INIT;
offset++;
return JavaTokenContext.INVALID_OCTAL_LITERAL;
case 'e':
case 'E':
state = ISI_DOUBLE_EXP;
break;
default:
if (Character.isDigit(actChar)) { // '8' and '9' already handled
state = ISI_OCTAL;
break;
}
state = INIT;
return JavaTokenContext.INT_LITERAL;
}
break;
case ISI_INT:
switch (actChar) {
case 'l':
case 'L':
offset++;
state = INIT;
return JavaTokenContext.LONG_LITERAL;
case '.':
state = ISI_DOUBLE;
break;
case 'f':
case 'F':
offset++;
state = INIT;
return JavaTokenContext.FLOAT_LITERAL;
case 'd':
case 'D':
offset++;
state = INIT;
return JavaTokenContext.DOUBLE_LITERAL;
case 'e':
case 'E':
state = ISI_DOUBLE_EXP;
break;
default:
if (!(actChar >= '0' && actChar <= '9')) {
state = INIT;
return JavaTokenContext.INT_LITERAL;
}
}
break;
case ISI_OCTAL:
if (!(actChar >= '0' && actChar <= '7')) {
state = INIT;
return JavaTokenContext.OCTAL_LITERAL;
}
break;
case ISI_DOUBLE:
switch (actChar) {
case 'f':
case 'F':
offset++;
state = INIT;
return JavaTokenContext.FLOAT_LITERAL;
case 'd':
case 'D':
offset++;
state = INIT;
return JavaTokenContext.DOUBLE_LITERAL;
case 'e':
case 'E':
state = ISI_DOUBLE_EXP;
break;
default:
if (!((actChar >= '0' && actChar <= '9')
|| actChar == '.')) {
state = INIT;
return JavaTokenContext.DOUBLE_LITERAL;
}
}
break;
case ISI_DOUBLE_EXP:
switch (actChar) {
case 'f':
case 'F':
offset++;
state = INIT;
return JavaTokenContext.FLOAT_LITERAL;
case 'd':
case 'D':
offset++;
state = INIT;
return JavaTokenContext.DOUBLE_LITERAL;
default:
if (!(Character.isDigit(actChar)
|| actChar == '-' || actChar == '+')) {
state = INIT;
return JavaTokenContext.DOUBLE_LITERAL;
}
}
break;
case ISI_HEX:
if (!((actChar >= 'a' && actChar <= 'f')
|| (actChar >= 'A' && actChar <= 'F')
|| Character.isDigit(actChar))
) {
state = INIT;
return JavaTokenContext.HEX_LITERAL;
}
break;
case ISA_DOT:
if (Character.isDigit(actChar)) {
state = ISI_DOUBLE;
} else if (actChar == '.' && offset + 1 < stopOffset && buffer[offset + 1] == '.') {
offset += 2;
state = INIT;
return JavaTokenContext.ELLIPSIS;
} else { // only single dot
state = INIT;
return JavaTokenContext.DOT;
}
break;
} // end of switch(state)
offset++;
} // end of while(offset...)
/** At this stage there's no more text in the scanned buffer.
* Scanner first checks whether this is completely the last
* available buffer.
*/
if (lastBuffer) {
switch(state) {
case ISI_WHITESPACE:
state = INIT;
return JavaTokenContext.WHITESPACE;
case ISI_IDENTIFIER:
state = INIT;
TokenID kwd = matchKeyword(buffer, tokenOffset, offset - tokenOffset);
return (kwd != null) ? kwd : JavaTokenContext.IDENTIFIER;
case ISI_LINE_COMMENT:
return JavaTokenContext.LINE_COMMENT; // stay in line-comment state
case ISI_BLOCK_COMMENT:
case ISA_STAR_I_BLOCK_COMMENT:
return JavaTokenContext.BLOCK_COMMENT; // stay in block-comment state
case ISI_STRING:
case ISI_STRING_A_BSLASH:
return JavaTokenContext.STRING_LITERAL; // hold the state
case ISI_CHAR:
case ISI_CHAR_A_BSLASH:
return JavaTokenContext.CHAR_LITERAL; // hold the state
case ISA_ZERO:
case ISI_INT:
state = INIT;
return JavaTokenContext.INT_LITERAL;
case ISI_OCTAL:
state = INIT;
return JavaTokenContext.OCTAL_LITERAL;
case ISI_DOUBLE:
case ISI_DOUBLE_EXP:
state = INIT;
return JavaTokenContext.DOUBLE_LITERAL;
case ISI_HEX:
state = INIT;
return JavaTokenContext.HEX_LITERAL;
case ISA_DOT:
state = INIT;
return JavaTokenContext.DOT;
case ISA_SLASH:
state = INIT;
return JavaTokenContext.DIV;
case ISA_EQ:
state = INIT;
return JavaTokenContext.EQ;
case ISA_GT:
state = INIT;
return JavaTokenContext.GT;
case ISA_GTGT:
state = INIT;
return JavaTokenContext.RSSHIFT;
case ISA_GTGTGT:
state = INIT;
return JavaTokenContext.RUSHIFT;
case ISA_LT:
state = INIT;
return JavaTokenContext.LT;
case ISA_LTLT:
state = INIT;
return JavaTokenContext.LSHIFT;
case ISA_PLUS:
state = INIT;
return JavaTokenContext.PLUS;
case ISA_MINUS:
state = INIT;
return JavaTokenContext.MINUS;
case ISA_STAR:
state = INIT;
return JavaTokenContext.MUL;
case ISA_PIPE:
state = INIT;
return JavaTokenContext.OR;
case ISA_PERCENT:
state = INIT;
return JavaTokenContext.MOD;
case ISA_AND:
state = INIT;
return JavaTokenContext.AND;
case ISA_XOR:
state = INIT;
return JavaTokenContext.XOR;
case ISA_EXCLAMATION:
state = INIT;
return JavaTokenContext.NOT;
}
}
/* At this stage there's no more text in the scanned buffer, but
* this buffer is not the last so the scan will continue on another buffer.
* The scanner tries to minimize the amount of characters
* that will be prescanned in the next buffer by returning the token
* where possible.
*/
switch (state) {
case ISI_WHITESPACE:
return JavaTokenContext.WHITESPACE;
}
return null; // nothing found
}