in compiler/src/main/java/org/apache/royale/compiler/internal/parsing/as/StreamingASTokenizer.java [775:1356]
public final ASToken next()
{
ASToken retVal = null;
// If the lexer for the included file is open, read from the included tokenizer.
boolean consumeSemi = false;
try
{
// Return token from the main file.
if (forkIncludeTokenizer != null)
{
retVal = forkIncludeTokenizer.next();
// Check if the forked tokenizer reached EOF.
if (retVal == null)
{
closeIncludeTokenizer();
// We should consume the next semicolon we find.
// Most include statements are terminated with a semicolon,
// and because we read the contents of the included file,
// this could cause problems with a semicolon in a place
// we don't want it.
consumeSemi = true;
}
else
return retVal;
}
if (bufferSize > 0)
{
retVal = lookAheadBuffer.remove(0);
bufferSize--;
}
else
{
retVal = nextTokenFromReader();
}
if (retVal == null)
return null;
final int tokenType = retVal.getType();
switch (tokenType)
{
// if we're seeing each in this part of the loop, it's not a
// syntactic keyword
// since we do lookahead when we see "for", checking for "each"
case TOKEN_RESERVED_WORD_EACH:
treatKeywordAsIdentifier(retVal);
processUserDefinedNamespace(retVal, 0);
return retVal;
case TOKEN_KEYWORD_INCLUDE:
{
if (!config.strictIdentifierNames && lastToken != null)
{
int lastTokenType = lastToken.getType();
switch (lastTokenType)
{
case TOKEN_KEYWORD_VAR:
case TOKEN_KEYWORD_FUNCTION:
case TOKEN_RESERVED_WORD_GET:
case TOKEN_RESERVED_WORD_SET:
case TOKEN_OPERATOR_MEMBER_ACCESS:
{
retVal.setType(TOKEN_IDENTIFIER);
return retVal;
}
}
}
// "followIncludes=false" is usually used for code model
// partitioner. They want the "include" token.
if (!config.followIncludes)
return retVal;
final ASToken token = LT(1);
// "include" at EOF is always a keyword
if (token == null)
return retVal;
if (!matches(token, TOKEN_LITERAL_STRING))
{
treatKeywordAsIdentifier(retVal); // it's an identifier
processUserDefinedNamespace(retVal, 0);
}
else
{
hasEncounteredIncludeStatements = true;
// Consume the file path after the include token.
consume(1);
final String filenameTokenText = token.getText();
final String includeString = filenameTokenText.substring(1, filenameTokenText.length() - 1);
if (sourcePath == null)
throw new NullPointerException("Source file is needed for resolving included file path.");
IFileSpecification includedFileSpec = null;
//respond to problems from our file handler
includedFileSpec = includeHandler.getFileSpecificationForInclude(sourcePath, includeString);
//
if (includedFileSpec == null)
{
ICompilerProblem problem = new FileNotFoundProblem(token, filenameTokenText); //the text will be the path not found
problems.add(problem);
retVal = next();
return retVal;
}
if (includeHandler.isCyclicInclude(includedFileSpec.getPath()))
{
ICompilerProblem problem = new CyclicalIncludesProblem(token);
problems.add(problem);
retVal = next();
return retVal;
}
else
{
// Fork a tokenizer for the included file
try
{
forkIncludeTokenizer = createForIncludeFile(this, includedFileSpec, includeHandler);
retVal = forkIncludeTokenizer.next();
}
catch (FileNotFoundException fnfe)
{
includeHandler.handleFileNotFound(includedFileSpec);
ICompilerProblem problem = new FileNotFoundProblem(token, includedFileSpec.getPath());
problems.add(problem);
retVal = next();
return retVal;
}
}
}
// Recover from compiler problems and continue.
if (retVal == null)
{
// Included file is empty.
closeIncludeTokenizer();
// Fall back to main source.
retVal = this.next();
}
return retVal;
}
case TOKEN_RESERVED_WORD_CONFIG:
if (matches(LT(1), TOKEN_RESERVED_WORD_NAMESPACE))
{ //we config namespace
retVal.setType(TOKEN_RESERVED_WORD_CONFIG);
return retVal;
}
treatKeywordAsIdentifier(retVal); //identifier
processUserDefinedNamespace(retVal, 0);
return retVal;
case HIDDEN_TOKEN_BUILTIN_NS:
if (matches(LT(1), TOKEN_OPERATOR_NS_QUALIFIER))
{ //we have public:: and this structure is not an annotation but a name ref
retVal.setType(TOKEN_NAMESPACE_NAME);
return retVal;
}
retVal.setType(TOKEN_NAMESPACE_ANNOTATION);
return retVal;
case TOKEN_MODIFIER_DYNAMIC:
case TOKEN_MODIFIER_FINAL:
case TOKEN_MODIFIER_NATIVE:
case TOKEN_MODIFIER_OVERRIDE:
case TOKEN_MODIFIER_STATIC:
case TOKEN_MODIFIER_VIRTUAL:
case TOKEN_MODIFIER_ABSTRACT:
{
// previous token is either a modifier or a namespace, or if
// null, assume keyword
// next token is from a definition or a modifier or a namespace
final ASToken nextToken = LT(1);
if (nextToken != null)
{
switch (nextToken.getType())
{
case TOKEN_KEYWORD_CLASS:
case TOKEN_KEYWORD_FUNCTION:
case TOKEN_KEYWORD_INTERFACE:
case TOKEN_RESERVED_WORD_NAMESPACE:
case TOKEN_KEYWORD_VAR:
case TOKEN_KEYWORD_CONST:
case TOKEN_MODIFIER_DYNAMIC:
case TOKEN_MODIFIER_FINAL:
case TOKEN_MODIFIER_NATIVE:
case TOKEN_MODIFIER_OVERRIDE:
case TOKEN_MODIFIER_STATIC:
case TOKEN_MODIFIER_VIRTUAL:
case TOKEN_MODIFIER_ABSTRACT:
case TOKEN_NAMESPACE_ANNOTATION:
case TOKEN_NAMESPACE_NAME:
case HIDDEN_TOKEN_BUILTIN_NS:
return retVal;
case TOKEN_IDENTIFIER:
if (isUserDefinedNamespace(nextToken, 1)) // we're already looking ahead one so make sure we look ahead one further
return retVal;
default:
// Not applicable to other token types.
break;
}
}
treatKeywordAsIdentifier(retVal);
processUserDefinedNamespace(retVal, 0);
return retVal;
}
//we combine +/- for numeric literals here
case TOKEN_OPERATOR_MINUS:
case TOKEN_OPERATOR_PLUS:
{
if (lastTokenNotRegularCommentOrASDocComment == null
|| !lastTokenNotRegularCommentOrASDocComment.canPreceedSignedOperator())
{
final ASToken nextToken = LT(1);
if (nextToken != null)
{
switch (nextToken.getType())
{
case TOKEN_LITERAL_NUMBER:
case TOKEN_LITERAL_HEX_NUMBER:
retVal.setEnd(nextToken.getEnd());
retVal.setEndLine(nextToken.getEndLine());
retVal.setEndColumn(nextToken.getEndColumn());
final StringBuilder builder = new StringBuilder(retVal.getText());
builder.append(nextToken.getText());
retVal.setText(poolString(builder.toString()));
consume(1);
retVal.setType(nextToken.getType());
break;
default:
// ignore other tokens
break;
}
}
}
return retVal;
}
//RECOGNIZE: for each
case TOKEN_KEYWORD_FOR:
{
final ASToken token = LT(1);
if (matches(token, TOKEN_RESERVED_WORD_EACH))
{
retVal.setEnd(token.getEnd());
retVal.setText(FOR_EACH);
consume(1);
return retVal;
}
if (!config.strictIdentifierNames && lastToken != null)
{
int lastTokenType = lastToken.getType();
switch (lastTokenType)
{
case TOKEN_KEYWORD_VAR:
case TOKEN_KEYWORD_FUNCTION:
case TOKEN_RESERVED_WORD_GET:
case TOKEN_RESERVED_WORD_SET:
case TOKEN_OPERATOR_MEMBER_ACCESS:
retVal.setType(TOKEN_IDENTIFIER);
}
}
return retVal;
}
//RECOGNIZE: default xml namespace
//default xml namespace must exist on the same line
case TOKEN_KEYWORD_DEFAULT:
{
final ASToken maybeNS = LT(2);
final boolean foundTokenNamespace = maybeNS != null &&
maybeNS.getType() == TOKEN_RESERVED_WORD_NAMESPACE;
final ASToken maybeXML = LT(1);
if (foundTokenNamespace)
{
final boolean foundTokenXML = maybeXML != null &&
maybeXML.getType() == TOKEN_IDENTIFIER &&
XML.equals(maybeXML.getText());
if (!foundTokenXML)
{
final ICompilerProblem problem =
new ExpectXmlBeforeNamespaceProblem(maybeNS);
problems.add(problem);
}
//combine all of these tokens together
retVal.setEnd(maybeNS.getEnd());
retVal.setText(DEFAULT_XML_NAMESPACE);
retVal.setType(TOKEN_DIRECTIVE_DEFAULT_XML);
consume(2);
}
// if this isn't "default xml namespace" then
// see if it is the default case in a switch
// otherwise, assume it is an identiferName
else if (!config.strictIdentifierNames &&
maybeXML != null &&
maybeXML.getType() != TOKEN_COLON)
retVal.setType(TOKEN_IDENTIFIER);
else if (!config.strictIdentifierNames && lastToken != null)
{
int lastTokenType = lastToken.getType();
switch (lastTokenType)
{
case TOKEN_KEYWORD_VAR:
case TOKEN_KEYWORD_FUNCTION:
case TOKEN_RESERVED_WORD_GET:
case TOKEN_RESERVED_WORD_SET:
case TOKEN_OPERATOR_MEMBER_ACCESS:
retVal.setType(TOKEN_IDENTIFIER);
}
}
return retVal;
}
case TOKEN_KEYWORD_VOID:
{
//check for void 0
final ASToken token = LT(1);
if (matches(token, TOKEN_LITERAL_NUMBER) && ZERO.equals(token.getText()))
{
retVal.setType(TOKEN_VOID_0);
combineText(retVal, token);
consume(1);
}
//check for void(0)
else if (matches(token, TOKEN_PAREN_OPEN))
{
final ASToken zeroT = LT(2);
if (matches(zeroT, TOKEN_LITERAL_NUMBER) && ZERO.equals(zeroT.getText()))
{
final ASToken closeParenT = LT(3);
if (matches(closeParenT, TOKEN_PAREN_CLOSE))
{
combineText(retVal, token);
combineText(retVal, zeroT);
combineText(retVal, closeParenT);
retVal.setType(TOKEN_VOID_0);
consume(3);
}
}
}
return retVal;
}
case TOKEN_IDENTIFIER:
{
//check for user-defined namespace before we return anything
processUserDefinedNamespace(retVal, 0);
return retVal;
}
//this is for metadata processing
case TOKEN_SQUARE_OPEN:
{
retVal = tryParseMetadata(retVal);
return retVal;
}
case HIDDEN_TOKEN_STAR_ASSIGNMENT:
{
//this is to solve an ambiguous case, where we can't tell the difference between
//var foo:*=null and foo *= null;
retVal.setType(TOKEN_OPERATOR_STAR);
retVal.setEnd(retVal.getEnd() - 1);
retVal.setText("*");
//add the equals
final ASToken nextToken = tokenizer.buildToken(TOKEN_OPERATOR_ASSIGNMENT,
retVal.getEnd() + 1, retVal.getEnd() + 2,
retVal.getLine(), retVal.getColumn(), "=");
nextToken.setSourcePath(sourcePath);
addTokenToBuffer(nextToken);
return retVal;
}
case TOKEN_SEMICOLON:
if (consumeSemi)
{
return next();
}
return retVal;
case TOKEN_VOID_0:
case TOKEN_LITERAL_REGEXP:
case TOKEN_COMMA:
case TOKEN_COLON:
case TOKEN_PAREN_OPEN:
case TOKEN_PAREN_CLOSE:
case TOKEN_SQUARE_CLOSE:
case TOKEN_ELLIPSIS:
case TOKEN_OPERATOR_PLUS_ASSIGNMENT:
case TOKEN_OPERATOR_MINUS_ASSIGNMENT:
case TOKEN_OPERATOR_MULTIPLICATION_ASSIGNMENT:
case TOKEN_OPERATOR_DIVISION_ASSIGNMENT:
case TOKEN_OPERATOR_MODULO_ASSIGNMENT:
case TOKEN_OPERATOR_BITWISE_AND_ASSIGNMENT:
case TOKEN_OPERATOR_BITWISE_OR_ASSIGNMENT:
case TOKEN_OPERATOR_BITWISE_XOR_ASSIGNMENT:
case TOKEN_OPERATOR_BITWISE_LEFT_SHIFT_ASSIGNMENT:
case TOKEN_OPERATOR_BITWISE_RIGHT_SHIFT_ASSIGNMENT:
case TOKEN_OPERATOR_BITWISE_UNSIGNED_RIGHT_SHIFT_ASSIGNMENT:
case TOKEN_OPERATOR_STAR:
case TOKEN_OPERATOR_NS_QUALIFIER:
case TOKEN_ASDOC_COMMENT:
case TOKEN_OPERATOR_DIVISION:
case TOKEN_OPERATOR_MODULO:
case TOKEN_OPERATOR_BITWISE_LEFT_SHIFT:
case TOKEN_OPERATOR_BITWISE_RIGHT_SHIFT:
case TOKEN_OPERATOR_BITWISE_UNSIGNED_RIGHT_SHIFT:
case TOKEN_OPERATOR_LESS_THAN:
case TOKEN_OPERATOR_GREATER_THAN:
case TOKEN_OPERATOR_LESS_THAN_EQUALS:
case TOKEN_OPERATOR_GREATER_THAN_EQUALS:
case TOKEN_OPERATOR_EQUAL:
case TOKEN_OPERATOR_NOT_EQUAL:
case TOKEN_OPERATOR_STRICT_EQUAL:
case TOKEN_OPERATOR_STRICT_NOT_EQUAL:
case TOKEN_OPERATOR_BITWISE_AND:
case TOKEN_OPERATOR_BITWISE_XOR:
case TOKEN_OPERATOR_BITWISE_OR:
case TOKEN_OPERATOR_LOGICAL_AND:
case TOKEN_OPERATOR_LOGICAL_OR:
case TOKEN_OPERATOR_LOGICAL_AND_ASSIGNMENT:
case TOKEN_OPERATOR_LOGICAL_OR_ASSIGNMENT:
case TOKEN_OPERATOR_NULLISH_COALESCING:
case TOKEN_TYPED_COLLECTION_OPEN:
case TOKEN_TYPED_COLLECTION_CLOSE:
case TOKEN_OPERATOR_MEMBER_ACCESS:
case TOKEN_RESERVED_WORD_NAMESPACE:
case TOKEN_RESERVED_WORD_GET:
case TOKEN_RESERVED_WORD_SET:
case TOKEN_OPERATOR_ASSIGNMENT:
case TOKEN_TYPED_LITERAL_CLOSE:
case TOKEN_TYPED_LITERAL_OPEN:
case TOKEN_OPERATOR_TERNARY:
case TOKEN_OPERATOR_DECREMENT:
case TOKEN_OPERATOR_INCREMENT:
case TOKEN_OPERATOR_ATSIGN:
case TOKEN_OPERATOR_BITWISE_NOT:
case TOKEN_OPERATOR_LOGICAL_NOT:
case TOKEN_E4X_BINDING_CLOSE:
case TOKEN_E4X_BINDING_OPEN:
case TOKEN_OPERATOR_DESCENDANT_ACCESS:
case TOKEN_OPERATOR_NULL_CONDITIONAL_ACCESS:
case TOKEN_NAMESPACE_ANNOTATION:
case TOKEN_NAMESPACE_NAME:
case TOKEN_BLOCK_OPEN:
case TOKEN_BLOCK_CLOSE:
case TOKEN_KEYWORD_FUNCTION:
return retVal;
case HIDDEN_TOKEN_MULTI_LINE_COMMENT:
case HIDDEN_TOKEN_SINGLE_LINE_COMMENT:
if (tokenizer.isCollectingComments())
{
return retVal;
}
assert (false);
return null;
case TOKEN_KEYWORD_INSTANCEOF:
case TOKEN_KEYWORD_AS:
case TOKEN_KEYWORD_IN:
case TOKEN_KEYWORD_IS:
if(!config.strictIdentifierNames)
{
if (lastToken != null)
{
int lastTokenType = lastToken.getType();
switch (lastTokenType)
{
case TOKEN_SEMICOLON:
case TOKEN_BLOCK_OPEN:
case TOKEN_COMMA:
retVal.setType(TOKEN_IDENTIFIER);
return retVal;
}
}
else
{
// we are first token so assume identifier
retVal.setType(TOKEN_IDENTIFIER);
return retVal;
}
}
// and fall through
case TOKEN_KEYWORD_DELETE:
if(!config.strictIdentifierNames)
{
ASToken nextToken = LT(1);
if (nextToken != null)
{
int nextTokenType = nextToken.getType();
switch (nextTokenType)
{
// if followed by a token assume it is the
// keyword and not the identiferName;
case TOKEN_IDENTIFIER:
return retVal;
// followed by a comma or semicolon
// probably being used in an expression
case TOKEN_COMMA:
case TOKEN_SEMICOLON:
retVal.setType(TOKEN_IDENTIFIER);
return retVal;
}
}
}
// and fall through
case TOKEN_KEYWORD_BREAK:
case TOKEN_KEYWORD_CASE:
case TOKEN_KEYWORD_CATCH:
case TOKEN_KEYWORD_CLASS:
case TOKEN_KEYWORD_CONST:
case TOKEN_KEYWORD_CONTINUE:
case TOKEN_KEYWORD_DO:
case TOKEN_KEYWORD_ELSE:
case TOKEN_KEYWORD_FALSE:
case TOKEN_KEYWORD_FINALLY:
case TOKEN_KEYWORD_IF:
case TOKEN_KEYWORD_IMPORT:
case TOKEN_KEYWORD_INTERFACE:
case TOKEN_KEYWORD_NULL:
case TOKEN_KEYWORD_PACKAGE:
case TOKEN_KEYWORD_SUPER:
case TOKEN_KEYWORD_SWITCH:
case TOKEN_KEYWORD_THIS:
case TOKEN_KEYWORD_TRUE:
case TOKEN_KEYWORD_TRY:
case TOKEN_KEYWORD_TYPEOF:
case TOKEN_KEYWORD_USE:
case TOKEN_KEYWORD_VAR:
case TOKEN_KEYWORD_WHILE:
case TOKEN_KEYWORD_WITH:
case TOKEN_KEYWORD_RETURN:
case TOKEN_KEYWORD_THROW:
case TOKEN_KEYWORD_NEW:
if (!config.strictIdentifierNames && lastToken != null)
{
int lastTokenType = lastToken.getType();
switch (lastTokenType)
{
case TOKEN_KEYWORD_VAR:
case TOKEN_KEYWORD_FUNCTION:
case TOKEN_RESERVED_WORD_GET:
case TOKEN_RESERVED_WORD_SET:
case TOKEN_OPERATOR_MEMBER_ACCESS:
retVal.setType(TOKEN_IDENTIFIER);
}
}
return retVal;
default:
if (ASToken.isE4X(tokenType))
return retVal;
if (retVal.isKeywordOrContextualReservedWord() || retVal.isLiteral())
return retVal;
// If we reach here, the token fails to match any processing logic.
final UnexpectedTokenProblem problem = new UnexpectedTokenProblem(
retVal,
ASTokenKind.UNKNOWN);
problems.add(problem);
}
}
catch (final Exception e)
{
if (lastException != null)
{
if (lastException.getClass().isInstance(e))
{
ICompilerProblem problem = new InternalCompilerProblem2(sourcePath, e, "StreamingASTokenizer");
problems.add(problem);
return null;
}
}
else
{
lastException = e;
retVal = null;
return next();
}
}
finally
{
consumeSemi = false;
lastToken = retVal;
if (retVal == null || (retVal.getType() != HIDDEN_TOKEN_SINGLE_LINE_COMMENT && retVal.getType() != HIDDEN_TOKEN_MULTI_LINE_COMMENT)) {
lastTokenNotRegularComment = retVal;
}
if (retVal == null || (retVal.getType() != HIDDEN_TOKEN_SINGLE_LINE_COMMENT && retVal.getType() != HIDDEN_TOKEN_MULTI_LINE_COMMENT && retVal.getType() != TOKEN_ASDOC_COMMENT)) {
lastTokenNotRegularCommentOrASDocComment = retVal;
}
}
return null;
}