001: /*
002: * Copyright 1995-2005 Sun Microsystems, Inc. All Rights Reserved.
003: * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
004: *
005: * This code is free software; you can redistribute it and/or modify it
006: * under the terms of the GNU General Public License version 2 only, as
007: * published by the Free Software Foundation. Sun designates this
008: * particular file as subject to the "Classpath" exception as provided
009: * by Sun in the LICENSE file that accompanied this code.
010: *
011: * This code is distributed in the hope that it will be useful, but WITHOUT
012: * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
013: * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
014: * version 2 for more details (a copy is included in the LICENSE file that
015: * accompanied this code).
016: *
017: * You should have received a copy of the GNU General Public License version
018: * 2 along with this work; if not, write to the Free Software Foundation,
019: * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
020: *
021: * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
022: * CA 95054 USA or visit www.sun.com if you need additional information or
023: * have any questions.
024: */
025:
026: package java.io;
027:
028: import java.util.Arrays;
029:
030: /**
031: * The <code>StreamTokenizer</code> class takes an input stream and
032: * parses it into "tokens", allowing the tokens to be
033: * read one at a time. The parsing process is controlled by a table
034: * and a number of flags that can be set to various states. The
035: * stream tokenizer can recognize identifiers, numbers, quoted
036: * strings, and various comment styles.
037: * <p>
038: * Each byte read from the input stream is regarded as a character
039: * in the range <code>'\u0000'</code> through <code>'\u00FF'</code>.
040: * The character value is used to look up five possible attributes of
041: * the character: <i>white space</i>, <i>alphabetic</i>,
042: * <i>numeric</i>, <i>string quote</i>, and <i>comment character</i>.
043: * Each character can have zero or more of these attributes.
044: * <p>
045: * In addition, an instance has four flags. These flags indicate:
046: * <ul>
047: * <li>Whether line terminators are to be returned as tokens or treated
048: * as white space that merely separates tokens.
049: * <li>Whether C-style comments are to be recognized and skipped.
050: * <li>Whether C++-style comments are to be recognized and skipped.
051: * <li>Whether the characters of identifiers are converted to lowercase.
052: * </ul>
053: * <p>
054: * A typical application first constructs an instance of this class,
055: * sets up the syntax tables, and then repeatedly loops calling the
056: * <code>nextToken</code> method in each iteration of the loop until
057: * it returns the value <code>TT_EOF</code>.
058: *
059: * @author James Gosling
060: * @version 1.53, 05/05/07
061: * @see java.io.StreamTokenizer#nextToken()
062: * @see java.io.StreamTokenizer#TT_EOF
063: * @since JDK1.0
064: */
065:
066: public class StreamTokenizer {
067:
068: /* Only one of these will be non-null */
069: private Reader reader = null;
070: private InputStream input = null;
071:
072: private char buf[] = new char[20];
073:
074: /**
075: * The next character to be considered by the nextToken method. May also
076: * be NEED_CHAR to indicate that a new character should be read, or SKIP_LF
077: * to indicate that a new character should be read and, if it is a '\n'
078: * character, it should be discarded and a second new character should be
079: * read.
080: */
081: private int peekc = NEED_CHAR;
082:
083: private static final int NEED_CHAR = Integer.MAX_VALUE;
084: private static final int SKIP_LF = Integer.MAX_VALUE - 1;
085:
086: private boolean pushedBack;
087: private boolean forceLower;
088: /** The line number of the last token read */
089: private int LINENO = 1;
090:
091: private boolean eolIsSignificantP = false;
092: private boolean slashSlashCommentsP = false;
093: private boolean slashStarCommentsP = false;
094:
095: private byte ctype[] = new byte[256];
096: private static final byte CT_WHITESPACE = 1;
097: private static final byte CT_DIGIT = 2;
098: private static final byte CT_ALPHA = 4;
099: private static final byte CT_QUOTE = 8;
100: private static final byte CT_COMMENT = 16;
101:
102: /**
103: * After a call to the <code>nextToken</code> method, this field
104: * contains the type of the token just read. For a single character
105: * token, its value is the single character, converted to an integer.
106: * For a quoted string token, its value is the quote character.
107: * Otherwise, its value is one of the following:
108: * <ul>
109: * <li><code>TT_WORD</code> indicates that the token is a word.
110: * <li><code>TT_NUMBER</code> indicates that the token is a number.
111: * <li><code>TT_EOL</code> indicates that the end of line has been read.
112: * The field can only have this value if the
113: * <code>eolIsSignificant</code> method has been called with the
114: * argument <code>true</code>.
115: * <li><code>TT_EOF</code> indicates that the end of the input stream
116: * has been reached.
117: * </ul>
118: * <p>
119: * The initial value of this field is -4.
120: *
121: * @see java.io.StreamTokenizer#eolIsSignificant(boolean)
122: * @see java.io.StreamTokenizer#nextToken()
123: * @see java.io.StreamTokenizer#quoteChar(int)
124: * @see java.io.StreamTokenizer#TT_EOF
125: * @see java.io.StreamTokenizer#TT_EOL
126: * @see java.io.StreamTokenizer#TT_NUMBER
127: * @see java.io.StreamTokenizer#TT_WORD
128: */
129: public int ttype = TT_NOTHING;
130:
131: /**
132: * A constant indicating that the end of the stream has been read.
133: */
134: public static final int TT_EOF = -1;
135:
136: /**
137: * A constant indicating that the end of the line has been read.
138: */
139: public static final int TT_EOL = '\n';
140:
141: /**
142: * A constant indicating that a number token has been read.
143: */
144: public static final int TT_NUMBER = -2;
145:
146: /**
147: * A constant indicating that a word token has been read.
148: */
149: public static final int TT_WORD = -3;
150:
151: /* A constant indicating that no token has been read, used for
152: * initializing ttype. FIXME This could be made public and
153: * made available as the part of the API in a future release.
154: */
155: private static final int TT_NOTHING = -4;
156:
157: /**
158: * If the current token is a word token, this field contains a
159: * string giving the characters of the word token. When the current
160: * token is a quoted string token, this field contains the body of
161: * the string.
162: * <p>
163: * The current token is a word when the value of the
164: * <code>ttype</code> field is <code>TT_WORD</code>. The current token is
165: * a quoted string token when the value of the <code>ttype</code> field is
166: * a quote character.
167: * <p>
168: * The initial value of this field is null.
169: *
170: * @see java.io.StreamTokenizer#quoteChar(int)
171: * @see java.io.StreamTokenizer#TT_WORD
172: * @see java.io.StreamTokenizer#ttype
173: */
174: public String sval;
175:
176: /**
177: * If the current token is a number, this field contains the value
178: * of that number. The current token is a number when the value of
179: * the <code>ttype</code> field is <code>TT_NUMBER</code>.
180: * <p>
181: * The initial value of this field is 0.0.
182: *
183: * @see java.io.StreamTokenizer#TT_NUMBER
184: * @see java.io.StreamTokenizer#ttype
185: */
186: public double nval;
187:
188: /** Private constructor that initializes everything except the streams. */
189: private StreamTokenizer() {
190: wordChars('a', 'z');
191: wordChars('A', 'Z');
192: wordChars(128 + 32, 255);
193: whitespaceChars(0, ' ');
194: commentChar('/');
195: quoteChar('"');
196: quoteChar('\'');
197: parseNumbers();
198: }
199:
200: /**
201: * Creates a stream tokenizer that parses the specified input
202: * stream. The stream tokenizer is initialized to the following
203: * default state:
204: * <ul>
205: * <li>All byte values <code>'A'</code> through <code>'Z'</code>,
206: * <code>'a'</code> through <code>'z'</code>, and
207: * <code>'\u00A0'</code> through <code>'\u00FF'</code> are
208: * considered to be alphabetic.
209: * <li>All byte values <code>'\u0000'</code> through
210: * <code>'\u0020'</code> are considered to be white space.
211: * <li><code>'/'</code> is a comment character.
212: * <li>Single quote <code>'\''</code> and double quote <code>'"'</code>
213: * are string quote characters.
214: * <li>Numbers are parsed.
215: * <li>Ends of lines are treated as white space, not as separate tokens.
216: * <li>C-style and C++-style comments are not recognized.
217: * </ul>
218: *
219: * @deprecated As of JDK version 1.1, the preferred way to tokenize an
220: * input stream is to convert it into a character stream, for example:
221: * <blockquote><pre>
222: * Reader r = new BufferedReader(new InputStreamReader(is));
223: * StreamTokenizer st = new StreamTokenizer(r);
224: * </pre></blockquote>
225: *
226: * @param is an input stream.
227: * @see java.io.BufferedReader
228: * @see java.io.InputStreamReader
229: * @see java.io.StreamTokenizer#StreamTokenizer(java.io.Reader)
230: */
231: @Deprecated
232: public StreamTokenizer(InputStream is) {
233: this ();
234: if (is == null) {
235: throw new NullPointerException();
236: }
237: input = is;
238: }
239:
240: /**
241: * Create a tokenizer that parses the given character stream.
242: *
243: * @param r a Reader object providing the input stream.
244: * @since JDK1.1
245: */
246: public StreamTokenizer(Reader r) {
247: this ();
248: if (r == null) {
249: throw new NullPointerException();
250: }
251: reader = r;
252: }
253:
254: /**
255: * Resets this tokenizer's syntax table so that all characters are
256: * "ordinary." See the <code>ordinaryChar</code> method
257: * for more information on a character being ordinary.
258: *
259: * @see java.io.StreamTokenizer#ordinaryChar(int)
260: */
261: public void resetSyntax() {
262: for (int i = ctype.length; --i >= 0;)
263: ctype[i] = 0;
264: }
265:
266: /**
267: * Specifies that all characters <i>c</i> in the range
268: * <code>low <= <i>c</i> <= high</code>
269: * are word constituents. A word token consists of a word constituent
270: * followed by zero or more word constituents or number constituents.
271: *
272: * @param low the low end of the range.
273: * @param hi the high end of the range.
274: */
275: public void wordChars(int low, int hi) {
276: if (low < 0)
277: low = 0;
278: if (hi >= ctype.length)
279: hi = ctype.length - 1;
280: while (low <= hi)
281: ctype[low++] |= CT_ALPHA;
282: }
283:
284: /**
285: * Specifies that all characters <i>c</i> in the range
286: * <code>low <= <i>c</i> <= high</code>
287: * are white space characters. White space characters serve only to
288: * separate tokens in the input stream.
289: *
290: * <p>Any other attribute settings for the characters in the specified
291: * range are cleared.
292: *
293: * @param low the low end of the range.
294: * @param hi the high end of the range.
295: */
296: public void whitespaceChars(int low, int hi) {
297: if (low < 0)
298: low = 0;
299: if (hi >= ctype.length)
300: hi = ctype.length - 1;
301: while (low <= hi)
302: ctype[low++] = CT_WHITESPACE;
303: }
304:
305: /**
306: * Specifies that all characters <i>c</i> in the range
307: * <code>low <= <i>c</i> <= high</code>
308: * are "ordinary" in this tokenizer. See the
309: * <code>ordinaryChar</code> method for more information on a
310: * character being ordinary.
311: *
312: * @param low the low end of the range.
313: * @param hi the high end of the range.
314: * @see java.io.StreamTokenizer#ordinaryChar(int)
315: */
316: public void ordinaryChars(int low, int hi) {
317: if (low < 0)
318: low = 0;
319: if (hi >= ctype.length)
320: hi = ctype.length - 1;
321: while (low <= hi)
322: ctype[low++] = 0;
323: }
324:
325: /**
326: * Specifies that the character argument is "ordinary"
327: * in this tokenizer. It removes any special significance the
328: * character has as a comment character, word component, string
329: * delimiter, white space, or number character. When such a character
330: * is encountered by the parser, the parser treats it as a
331: * single-character token and sets <code>ttype</code> field to the
332: * character value.
333: *
334: * <p>Making a line terminator character "ordinary" may interfere
335: * with the ability of a <code>StreamTokenizer</code> to count
336: * lines. The <code>lineno</code> method may no longer reflect
337: * the presence of such terminator characters in its line count.
338: *
339: * @param ch the character.
340: * @see java.io.StreamTokenizer#ttype
341: */
342: public void ordinaryChar(int ch) {
343: if (ch >= 0 && ch < ctype.length)
344: ctype[ch] = 0;
345: }
346:
347: /**
348: * Specified that the character argument starts a single-line
349: * comment. All characters from the comment character to the end of
350: * the line are ignored by this stream tokenizer.
351: *
352: * <p>Any other attribute settings for the specified character are cleared.
353: *
354: * @param ch the character.
355: */
356: public void commentChar(int ch) {
357: if (ch >= 0 && ch < ctype.length)
358: ctype[ch] = CT_COMMENT;
359: }
360:
361: /**
362: * Specifies that matching pairs of this character delimit string
363: * constants in this tokenizer.
364: * <p>
365: * When the <code>nextToken</code> method encounters a string
366: * constant, the <code>ttype</code> field is set to the string
367: * delimiter and the <code>sval</code> field is set to the body of
368: * the string.
369: * <p>
370: * If a string quote character is encountered, then a string is
371: * recognized, consisting of all characters after (but not including)
372: * the string quote character, up to (but not including) the next
373: * occurrence of that same string quote character, or a line
374: * terminator, or end of file. The usual escape sequences such as
375: * <code>"\n"</code> and <code>"\t"</code> are recognized and
376: * converted to single characters as the string is parsed.
377: *
378: * <p>Any other attribute settings for the specified character are cleared.
379: *
380: * @param ch the character.
381: * @see java.io.StreamTokenizer#nextToken()
382: * @see java.io.StreamTokenizer#sval
383: * @see java.io.StreamTokenizer#ttype
384: */
385: public void quoteChar(int ch) {
386: if (ch >= 0 && ch < ctype.length)
387: ctype[ch] = CT_QUOTE;
388: }
389:
390: /**
391: * Specifies that numbers should be parsed by this tokenizer. The
392: * syntax table of this tokenizer is modified so that each of the twelve
393: * characters:
394: * <blockquote><pre>
395: * 0 1 2 3 4 5 6 7 8 9 . -
396: * </pre></blockquote>
397: * <p>
398: * has the "numeric" attribute.
399: * <p>
400: * When the parser encounters a word token that has the format of a
401: * double precision floating-point number, it treats the token as a
402: * number rather than a word, by setting the <code>ttype</code>
403: * field to the value <code>TT_NUMBER</code> and putting the numeric
404: * value of the token into the <code>nval</code> field.
405: *
406: * @see java.io.StreamTokenizer#nval
407: * @see java.io.StreamTokenizer#TT_NUMBER
408: * @see java.io.StreamTokenizer#ttype
409: */
410: public void parseNumbers() {
411: for (int i = '0'; i <= '9'; i++)
412: ctype[i] |= CT_DIGIT;
413: ctype['.'] |= CT_DIGIT;
414: ctype['-'] |= CT_DIGIT;
415: }
416:
417: /**
418: * Determines whether or not ends of line are treated as tokens.
419: * If the flag argument is true, this tokenizer treats end of lines
420: * as tokens; the <code>nextToken</code> method returns
421: * <code>TT_EOL</code> and also sets the <code>ttype</code> field to
422: * this value when an end of line is read.
423: * <p>
424: * A line is a sequence of characters ending with either a
425: * carriage-return character (<code>'\r'</code>) or a newline
426: * character (<code>'\n'</code>). In addition, a carriage-return
427: * character followed immediately by a newline character is treated
428: * as a single end-of-line token.
429: * <p>
430: * If the <code>flag</code> is false, end-of-line characters are
431: * treated as white space and serve only to separate tokens.
432: *
433: * @param flag <code>true</code> indicates that end-of-line characters
434: * are separate tokens; <code>false</code> indicates that
435: * end-of-line characters are white space.
436: * @see java.io.StreamTokenizer#nextToken()
437: * @see java.io.StreamTokenizer#ttype
438: * @see java.io.StreamTokenizer#TT_EOL
439: */
440: public void eolIsSignificant(boolean flag) {
441: eolIsSignificantP = flag;
442: }
443:
444: /**
445: * Determines whether or not the tokenizer recognizes C-style comments.
446: * If the flag argument is <code>true</code>, this stream tokenizer
447: * recognizes C-style comments. All text between successive
448: * occurrences of <code>/*</code> and <code>*/</code> are discarded.
449: * <p>
450: * If the flag argument is <code>false</code>, then C-style comments
451: * are not treated specially.
452: *
453: * @param flag <code>true</code> indicates to recognize and ignore
454: * C-style comments.
455: */
456: public void slashStarComments(boolean flag) {
457: slashStarCommentsP = flag;
458: }
459:
460: /**
461: * Determines whether or not the tokenizer recognizes C++-style comments.
462: * If the flag argument is <code>true</code>, this stream tokenizer
463: * recognizes C++-style comments. Any occurrence of two consecutive
464: * slash characters (<code>'/'</code>) is treated as the beginning of
465: * a comment that extends to the end of the line.
466: * <p>
467: * If the flag argument is <code>false</code>, then C++-style
468: * comments are not treated specially.
469: *
470: * @param flag <code>true</code> indicates to recognize and ignore
471: * C++-style comments.
472: */
473: public void slashSlashComments(boolean flag) {
474: slashSlashCommentsP = flag;
475: }
476:
477: /**
478: * Determines whether or not word token are automatically lowercased.
479: * If the flag argument is <code>true</code>, then the value in the
480: * <code>sval</code> field is lowercased whenever a word token is
481: * returned (the <code>ttype</code> field has the
482: * value <code>TT_WORD</code> by the <code>nextToken</code> method
483: * of this tokenizer.
484: * <p>
485: * If the flag argument is <code>false</code>, then the
486: * <code>sval</code> field is not modified.
487: *
488: * @param fl <code>true</code> indicates that all word tokens should
489: * be lowercased.
490: * @see java.io.StreamTokenizer#nextToken()
491: * @see java.io.StreamTokenizer#ttype
492: * @see java.io.StreamTokenizer#TT_WORD
493: */
494: public void lowerCaseMode(boolean fl) {
495: forceLower = fl;
496: }
497:
498: /** Read the next character */
499: private int read() throws IOException {
500: if (reader != null)
501: return reader.read();
502: else if (input != null)
503: return input.read();
504: else
505: throw new IllegalStateException();
506: }
507:
508: /**
509: * Parses the next token from the input stream of this tokenizer.
510: * The type of the next token is returned in the <code>ttype</code>
511: * field. Additional information about the token may be in the
512: * <code>nval</code> field or the <code>sval</code> field of this
513: * tokenizer.
514: * <p>
515: * Typical clients of this
516: * class first set up the syntax tables and then sit in a loop
517: * calling nextToken to parse successive tokens until TT_EOF
518: * is returned.
519: *
520: * @return the value of the <code>ttype</code> field.
521: * @exception IOException if an I/O error occurs.
522: * @see java.io.StreamTokenizer#nval
523: * @see java.io.StreamTokenizer#sval
524: * @see java.io.StreamTokenizer#ttype
525: */
526: public int nextToken() throws IOException {
527: if (pushedBack) {
528: pushedBack = false;
529: return ttype;
530: }
531: byte ct[] = ctype;
532: sval = null;
533:
534: int c = peekc;
535: if (c < 0)
536: c = NEED_CHAR;
537: if (c == SKIP_LF) {
538: c = read();
539: if (c < 0)
540: return ttype = TT_EOF;
541: if (c == '\n')
542: c = NEED_CHAR;
543: }
544: if (c == NEED_CHAR) {
545: c = read();
546: if (c < 0)
547: return ttype = TT_EOF;
548: }
549: ttype = c; /* Just to be safe */
550:
551: /* Set peekc so that the next invocation of nextToken will read
552: * another character unless peekc is reset in this invocation
553: */
554: peekc = NEED_CHAR;
555:
556: int ctype = c < 256 ? ct[c] : CT_ALPHA;
557: while ((ctype & CT_WHITESPACE) != 0) {
558: if (c == '\r') {
559: LINENO++;
560: if (eolIsSignificantP) {
561: peekc = SKIP_LF;
562: return ttype = TT_EOL;
563: }
564: c = read();
565: if (c == '\n')
566: c = read();
567: } else {
568: if (c == '\n') {
569: LINENO++;
570: if (eolIsSignificantP) {
571: return ttype = TT_EOL;
572: }
573: }
574: c = read();
575: }
576: if (c < 0)
577: return ttype = TT_EOF;
578: ctype = c < 256 ? ct[c] : CT_ALPHA;
579: }
580:
581: if ((ctype & CT_DIGIT) != 0) {
582: boolean neg = false;
583: if (c == '-') {
584: c = read();
585: if (c != '.' && (c < '0' || c > '9')) {
586: peekc = c;
587: return ttype = '-';
588: }
589: neg = true;
590: }
591: double v = 0;
592: int decexp = 0;
593: int seendot = 0;
594: while (true) {
595: if (c == '.' && seendot == 0)
596: seendot = 1;
597: else if ('0' <= c && c <= '9') {
598: v = v * 10 + (c - '0');
599: decexp += seendot;
600: } else
601: break;
602: c = read();
603: }
604: peekc = c;
605: if (decexp != 0) {
606: double denom = 10;
607: decexp--;
608: while (decexp > 0) {
609: denom *= 10;
610: decexp--;
611: }
612: /* Do one division of a likely-to-be-more-accurate number */
613: v = v / denom;
614: }
615: nval = neg ? -v : v;
616: return ttype = TT_NUMBER;
617: }
618:
619: if ((ctype & CT_ALPHA) != 0) {
620: int i = 0;
621: do {
622: if (i >= buf.length) {
623: buf = Arrays.copyOf(buf, buf.length * 2);
624: }
625: buf[i++] = (char) c;
626: c = read();
627: ctype = c < 0 ? CT_WHITESPACE : c < 256 ? ct[c]
628: : CT_ALPHA;
629: } while ((ctype & (CT_ALPHA | CT_DIGIT)) != 0);
630: peekc = c;
631: sval = String.copyValueOf(buf, 0, i);
632: if (forceLower)
633: sval = sval.toLowerCase();
634: return ttype = TT_WORD;
635: }
636:
637: if ((ctype & CT_QUOTE) != 0) {
638: ttype = c;
639: int i = 0;
640: /* Invariants (because \Octal needs a lookahead):
641: * (i) c contains char value
642: * (ii) d contains the lookahead
643: */
644: int d = read();
645: while (d >= 0 && d != ttype && d != '\n' && d != '\r') {
646: if (d == '\\') {
647: c = read();
648: int first = c; /* To allow \377, but not \477 */
649: if (c >= '0' && c <= '7') {
650: c = c - '0';
651: int c2 = read();
652: if ('0' <= c2 && c2 <= '7') {
653: c = (c << 3) + (c2 - '0');
654: c2 = read();
655: if ('0' <= c2 && c2 <= '7' && first <= '3') {
656: c = (c << 3) + (c2 - '0');
657: d = read();
658: } else
659: d = c2;
660: } else
661: d = c2;
662: } else {
663: switch (c) {
664: case 'a':
665: c = 0x7;
666: break;
667: case 'b':
668: c = '\b';
669: break;
670: case 'f':
671: c = 0xC;
672: break;
673: case 'n':
674: c = '\n';
675: break;
676: case 'r':
677: c = '\r';
678: break;
679: case 't':
680: c = '\t';
681: break;
682: case 'v':
683: c = 0xB;
684: break;
685: }
686: d = read();
687: }
688: } else {
689: c = d;
690: d = read();
691: }
692: if (i >= buf.length) {
693: buf = Arrays.copyOf(buf, buf.length * 2);
694: }
695: buf[i++] = (char) c;
696: }
697:
698: /* If we broke out of the loop because we found a matching quote
699: * character then arrange to read a new character next time
700: * around; otherwise, save the character.
701: */
702: peekc = (d == ttype) ? NEED_CHAR : d;
703:
704: sval = String.copyValueOf(buf, 0, i);
705: return ttype;
706: }
707:
708: if (c == '/' && (slashSlashCommentsP || slashStarCommentsP)) {
709: c = read();
710: if (c == '*' && slashStarCommentsP) {
711: int prevc = 0;
712: while ((c = read()) != '/' || prevc != '*') {
713: if (c == '\r') {
714: LINENO++;
715: c = read();
716: if (c == '\n') {
717: c = read();
718: }
719: } else {
720: if (c == '\n') {
721: LINENO++;
722: c = read();
723: }
724: }
725: if (c < 0)
726: return ttype = TT_EOF;
727: prevc = c;
728: }
729: return nextToken();
730: } else if (c == '/' && slashSlashCommentsP) {
731: while ((c = read()) != '\n' && c != '\r' && c >= 0)
732: ;
733: peekc = c;
734: return nextToken();
735: } else {
736: /* Now see if it is still a single line comment */
737: if ((ct['/'] & CT_COMMENT) != 0) {
738: while ((c = read()) != '\n' && c != '\r' && c >= 0)
739: ;
740: peekc = c;
741: return nextToken();
742: } else {
743: peekc = c;
744: return ttype = '/';
745: }
746: }
747: }
748:
749: if ((ctype & CT_COMMENT) != 0) {
750: while ((c = read()) != '\n' && c != '\r' && c >= 0)
751: ;
752: peekc = c;
753: return nextToken();
754: }
755:
756: return ttype = c;
757: }
758:
759: /**
760: * Causes the next call to the <code>nextToken</code> method of this
761: * tokenizer to return the current value in the <code>ttype</code>
762: * field, and not to modify the value in the <code>nval</code> or
763: * <code>sval</code> field.
764: *
765: * @see java.io.StreamTokenizer#nextToken()
766: * @see java.io.StreamTokenizer#nval
767: * @see java.io.StreamTokenizer#sval
768: * @see java.io.StreamTokenizer#ttype
769: */
770: public void pushBack() {
771: if (ttype != TT_NOTHING) /* No-op if nextToken() not called */
772: pushedBack = true;
773: }
774:
775: /**
776: * Return the current line number.
777: *
778: * @return the current line number of this stream tokenizer.
779: */
780: public int lineno() {
781: return LINENO;
782: }
783:
784: /**
785: * Returns the string representation of the current stream token and
786: * the line number it occurs on.
787: *
788: * <p>The precise string returned is unspecified, although the following
789: * example can be considered typical:
790: *
791: * <blockquote><pre>Token['a'], line 10</pre></blockquote>
792: *
793: * @return a string representation of the token
794: * @see java.io.StreamTokenizer#nval
795: * @see java.io.StreamTokenizer#sval
796: * @see java.io.StreamTokenizer#ttype
797: */
798: public String toString() {
799: String ret;
800: switch (ttype) {
801: case TT_EOF:
802: ret = "EOF";
803: break;
804: case TT_EOL:
805: ret = "EOL";
806: break;
807: case TT_WORD:
808: ret = sval;
809: break;
810: case TT_NUMBER:
811: ret = "n=" + nval;
812: break;
813: case TT_NOTHING:
814: ret = "NOTHING";
815: break;
816: default: {
817: /*
818: * ttype is the first character of either a quoted string or
819: * is an ordinary character. ttype can definitely not be less
820: * than 0, since those are reserved values used in the previous
821: * case statements
822: */
823: if (ttype < 256 && ((ctype[ttype] & CT_QUOTE) != 0)) {
824: ret = sval;
825: break;
826: }
827:
828: char s[] = new char[3];
829: s[0] = s[2] = '\'';
830: s[1] = (char) ttype;
831: ret = new String(s);
832: break;
833: }
834: }
835: return "Token[" + ret + "], line " + LINENO;
836: }
837:
838: }
|
