in modules/luni/src/main/java/java/io/StreamTokenizer.java [254:477]
public int nextToken() throws IOException {
if (pushBackToken) {
pushBackToken = false;
if (ttype != TT_UNKNOWN) {
return ttype;
}
}
sval = null; // Always reset sval to null
int currentChar = peekChar == -2 ? read() : peekChar;
if (lastCr && currentChar == '\n') {
lastCr = false;
currentChar = read();
}
if (currentChar == -1) {
return (ttype = TT_EOF);
}
byte currentType = currentChar > 255 ? TOKEN_WORD
: tokenTypes[currentChar];
while ((currentType & TOKEN_WHITE) != 0) {
/**
* Skip over white space until we hit a new line or a real token
*/
if (currentChar == '\r') {
lineNumber++;
if (isEOLSignificant) {
lastCr = true;
peekChar = -2;
return (ttype = TT_EOL);
}
if ((currentChar = read()) == '\n') {
currentChar = read();
}
} else if (currentChar == '\n') {
lineNumber++;
if (isEOLSignificant) {
peekChar = -2;
return (ttype = TT_EOL);
}
currentChar = read();
} else {
// Advance over this white space character and try again.
currentChar = read();
}
if (currentChar == -1) {
return (ttype = TT_EOF);
}
currentType = currentChar > 255 ? TOKEN_WORD
: tokenTypes[currentChar];
}
/**
* Check for digits before checking for words since digits can be
* contained within words.
*/
if ((currentType & TOKEN_DIGIT) != 0) {
StringBuilder digits = new StringBuilder(20);
boolean haveDecimal = false, checkJustNegative = currentChar == '-';
while (true) {
if (currentChar == '.') {
haveDecimal = true;
}
digits.append((char) currentChar);
currentChar = read();
if ((currentChar < '0' || currentChar > '9')
&& (haveDecimal || currentChar != '.')) {
break;
}
}
peekChar = currentChar;
if (checkJustNegative && digits.length() == 1) {
// Didn't get any other digits other than '-'
return (ttype = '-');
}
try {
nval = Double.valueOf(digits.toString()).doubleValue();
} catch (NumberFormatException e) {
// Unsure what to do, will write test.
nval = 0;
}
return (ttype = TT_NUMBER);
}
// Check for words
if ((currentType & TOKEN_WORD) != 0) {
StringBuilder word = new StringBuilder(20);
while (true) {
word.append((char) currentChar);
currentChar = read();
if (currentChar == -1
|| (currentChar < 256 && (tokenTypes[currentChar] & (TOKEN_WORD | TOKEN_DIGIT)) == 0)) {
break;
}
}
peekChar = currentChar;
sval = forceLowercase ? word.toString().toLowerCase() : word
.toString();
return (ttype = TT_WORD);
}
// Check for quoted character
if (currentType == TOKEN_QUOTE) {
int matchQuote = currentChar;
StringBuilder quoteString = new StringBuilder();
int peekOne = read();
while (peekOne >= 0 && peekOne != matchQuote && peekOne != '\r'
&& peekOne != '\n') {
boolean readPeek = true;
if (peekOne == '\\') {
int c1 = read();
// Check for quoted octal IE: \377
if (c1 <= '7' && c1 >= '0') {
int digitValue = c1 - '0';
c1 = read();
if (c1 > '7' || c1 < '0') {
readPeek = false;
} else {
digitValue = digitValue * 8 + (c1 - '0');
c1 = read();
// limit the digit value to a byte
if (digitValue > 037 || c1 > '7' || c1 < '0') {
readPeek = false;
} else {
digitValue = digitValue * 8 + (c1 - '0');
}
}
if (!readPeek) {
// We've consumed one to many
quoteString.append((char) digitValue);
peekOne = c1;
} else {
peekOne = digitValue;
}
} else {
switch (c1) {
case 'a':
peekOne = 0x7;
break;
case 'b':
peekOne = 0x8;
break;
case 'f':
peekOne = 0xc;
break;
case 'n':
peekOne = 0xA;
break;
case 'r':
peekOne = 0xD;
break;
case 't':
peekOne = 0x9;
break;
case 'v':
peekOne = 0xB;
break;
default:
peekOne = c1;
}
}
}
if (readPeek) {
quoteString.append((char) peekOne);
peekOne = read();
}
}
if (peekOne == matchQuote) {
peekOne = read();
}
peekChar = peekOne;
ttype = matchQuote;
sval = quoteString.toString();
return ttype;
}
// Do comments, both "//" and "/*stuff*/"
if (currentChar == '/' && (slashSlashComments || slashStarComments)) {
if ((currentChar = read()) == '*' && slashStarComments) {
int peekOne = read();
while (true) {
currentChar = peekOne;
peekOne = read();
if (currentChar == -1) {
peekChar = -1;
return (ttype = TT_EOF);
}
if (currentChar == '\r') {
if (peekOne == '\n') {
peekOne = read();
}
lineNumber++;
} else if (currentChar == '\n') {
lineNumber++;
} else if (currentChar == '*' && peekOne == '/') {
peekChar = read();
return nextToken();
}
}
} else if (currentChar == '/' && slashSlashComments) {
// Skip to EOF or new line then return the next token
while ((currentChar = read()) >= 0 && currentChar != '\r'
&& currentChar != '\n') {
// Intentionally empty
}
peekChar = currentChar;
return nextToken();
} else if (currentType != TOKEN_COMMENT) {
// Was just a slash by itself
peekChar = currentChar;
return (ttype = '/');
}
}
// Check for comment character
if (currentType == TOKEN_COMMENT) {
// Skip to EOF or new line then return the next token
while ((currentChar = read()) >= 0 && currentChar != '\r'
&& currentChar != '\n') {
// Intentionally empty
}
peekChar = currentChar;
return nextToken();
}
peekChar = read();
return (ttype = currentChar);
}