001: /*
002: * Copyright 1994-2004 Sun Microsystems, Inc. All Rights Reserved.
003: * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
004: *
005: * This code is free software; you can redistribute it and/or modify it
006: * under the terms of the GNU General Public License version 2 only, as
007: * published by the Free Software Foundation. Sun designates this
008: * particular file as subject to the "Classpath" exception as provided
009: * by Sun in the LICENSE file that accompanied this code.
010: *
011: * This code is distributed in the hope that it will be useful, but WITHOUT
012: * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
013: * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
014: * version 2 for more details (a copy is included in the LICENSE file that
015: * accompanied this code).
016: *
017: * You should have received a copy of the GNU General Public License version
018: * 2 along with this work; if not, write to the Free Software Foundation,
019: * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
020: *
021: * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
022: * CA 95054 USA or visit www.sun.com if you need additional information or
023: * have any questions.
024: */
025:
026: package java.util;
027:
028: import java.lang.*;
029:
030: /**
031: * The string tokenizer class allows an application to break a
032: * string into tokens. The tokenization method is much simpler than
033: * the one used by the <code>StreamTokenizer</code> class. The
034: * <code>StringTokenizer</code> methods do not distinguish among
035: * identifiers, numbers, and quoted strings, nor do they recognize
036: * and skip comments.
037: * <p>
038: * The set of delimiters (the characters that separate tokens) may
039: * be specified either at creation time or on a per-token basis.
040: * <p>
041: * An instance of <code>StringTokenizer</code> behaves in one of two
042: * ways, depending on whether it was created with the
043: * <code>returnDelims</code> flag having the value <code>true</code>
044: * or <code>false</code>:
045: * <ul>
046: * <li>If the flag is <code>false</code>, delimiter characters serve to
047: * separate tokens. A token is a maximal sequence of consecutive
048: * characters that are not delimiters.
049: * <li>If the flag is <code>true</code>, delimiter characters are themselves
050: * considered to be tokens. A token is thus either one delimiter
051: * character, or a maximal sequence of consecutive characters that are
052: * not delimiters.
053: * </ul><p>
054: * A <tt>StringTokenizer</tt> object internally maintains a current
055: * position within the string to be tokenized. Some operations advance this
056: * current position past the characters processed.<p>
057: * A token is returned by taking a substring of the string that was used to
058: * create the <tt>StringTokenizer</tt> object.
059: * <p>
060: * The following is one example of the use of the tokenizer. The code:
061: * <blockquote><pre>
062: * StringTokenizer st = new StringTokenizer("this is a test");
063: * while (st.hasMoreTokens()) {
064: * System.out.println(st.nextToken());
065: * }
066: * </pre></blockquote>
067: * <p>
068: * prints the following output:
069: * <blockquote><pre>
070: * this
071: * is
072: * a
073: * test
074: * </pre></blockquote>
075: *
076: * <p>
077: * <tt>StringTokenizer</tt> is a legacy class that is retained for
078: * compatibility reasons although its use is discouraged in new code. It is
079: * recommended that anyone seeking this functionality use the <tt>split</tt>
080: * method of <tt>String</tt> or the java.util.regex package instead.
081: * <p>
082: * The following example illustrates how the <tt>String.split</tt>
083: * method can be used to break up a string into its basic tokens:
084: * <blockquote><pre>
085: * String[] result = "this is a test".split("\\s");
086: * for (int x=0; x<result.length; x++)
087: * System.out.println(result[x]);
088: * </pre></blockquote>
089: * <p>
090: * prints the following output:
091: * <blockquote><pre>
092: * this
093: * is
094: * a
095: * test
096: * </pre></blockquote>
097: *
098: * @author unascribed
099: * @version 1.41, 05/05/07
100: * @see java.io.StreamTokenizer
101: * @since JDK1.0
102: */
103: public class StringTokenizer implements Enumeration<Object> {
104: private int currentPosition;
105: private int newPosition;
106: private int maxPosition;
107: private String str;
108: private String delimiters;
109: private boolean retDelims;
110: private boolean delimsChanged;
111:
112: /**
113: * maxDelimCodePoint stores the value of the delimiter character with the
114: * highest value. It is used to optimize the detection of delimiter
115: * characters.
116: *
117: * It is unlikely to provide any optimization benefit in the
118: * hasSurrogates case because most string characters will be
119: * smaller than the limit, but we keep it so that the two code
120: * paths remain similar.
121: */
122: private int maxDelimCodePoint;
123:
124: /**
125: * If delimiters include any surrogates (including surrogate
126: * pairs), hasSurrogates is true and the tokenizer uses the
127: * different code path. This is because String.indexOf(int)
128: * doesn't handle unpaired surrogates as a single character.
129: */
130: private boolean hasSurrogates = false;
131:
132: /**
133: * When hasSurrogates is true, delimiters are converted to code
134: * points and isDelimiter(int) is used to determine if the given
135: * codepoint is a delimiter.
136: */
137: private int[] delimiterCodePoints;
138:
139: /**
140: * Set maxDelimCodePoint to the highest char in the delimiter set.
141: */
142: private void setMaxDelimCodePoint() {
143: if (delimiters == null) {
144: maxDelimCodePoint = 0;
145: return;
146: }
147:
148: int m = 0;
149: int c;
150: int count = 0;
151: for (int i = 0; i < delimiters.length(); i += Character
152: .charCount(c)) {
153: c = delimiters.charAt(i);
154: if (c >= Character.MIN_HIGH_SURROGATE
155: && c <= Character.MAX_LOW_SURROGATE) {
156: c = delimiters.codePointAt(i);
157: hasSurrogates = true;
158: }
159: if (m < c)
160: m = c;
161: count++;
162: }
163: maxDelimCodePoint = m;
164:
165: if (hasSurrogates) {
166: delimiterCodePoints = new int[count];
167: for (int i = 0, j = 0; i < count; i++, j += Character
168: .charCount(c)) {
169: c = delimiters.codePointAt(j);
170: delimiterCodePoints[i] = c;
171: }
172: }
173: }
174:
175: /**
176: * Constructs a string tokenizer for the specified string. All
177: * characters in the <code>delim</code> argument are the delimiters
178: * for separating tokens.
179: * <p>
180: * If the <code>returnDelims</code> flag is <code>true</code>, then
181: * the delimiter characters are also returned as tokens. Each
182: * delimiter is returned as a string of length one. If the flag is
183: * <code>false</code>, the delimiter characters are skipped and only
184: * serve as separators between tokens.
185: * <p>
186: * Note that if <tt>delim</tt> is <tt>null</tt>, this constructor does
187: * not throw an exception. However, trying to invoke other methods on the
188: * resulting <tt>StringTokenizer</tt> may result in a
189: * <tt>NullPointerException</tt>.
190: *
191: * @param str a string to be parsed.
192: * @param delim the delimiters.
193: * @param returnDelims flag indicating whether to return the delimiters
194: * as tokens.
195: * @exception NullPointerException if str is <CODE>null</CODE>
196: */
197: public StringTokenizer(String str, String delim,
198: boolean returnDelims) {
199: currentPosition = 0;
200: newPosition = -1;
201: delimsChanged = false;
202: this .str = str;
203: maxPosition = str.length();
204: delimiters = delim;
205: retDelims = returnDelims;
206: setMaxDelimCodePoint();
207: }
208:
209: /**
210: * Constructs a string tokenizer for the specified string. The
211: * characters in the <code>delim</code> argument are the delimiters
212: * for separating tokens. Delimiter characters themselves will not
213: * be treated as tokens.
214: * <p>
215: * Note that if <tt>delim</tt> is <tt>null</tt>, this constructor does
216: * not throw an exception. However, trying to invoke other methods on the
217: * resulting <tt>StringTokenizer</tt> may result in a
218: * <tt>NullPointerException</tt>.
219: *
220: * @param str a string to be parsed.
221: * @param delim the delimiters.
222: * @exception NullPointerException if str is <CODE>null</CODE>
223: */
224: public StringTokenizer(String str, String delim) {
225: this (str, delim, false);
226: }
227:
228: /**
229: * Constructs a string tokenizer for the specified string. The
230: * tokenizer uses the default delimiter set, which is
231: * <code>" \t\n\r\f"</code>: the space character,
232: * the tab character, the newline character, the carriage-return character,
233: * and the form-feed character. Delimiter characters themselves will
234: * not be treated as tokens.
235: *
236: * @param str a string to be parsed.
237: * @exception NullPointerException if str is <CODE>null</CODE>
238: */
239: public StringTokenizer(String str) {
240: this (str, " \t\n\r\f", false);
241: }
242:
243: /**
244: * Skips delimiters starting from the specified position. If retDelims
245: * is false, returns the index of the first non-delimiter character at or
246: * after startPos. If retDelims is true, startPos is returned.
247: */
248: private int skipDelimiters(int startPos) {
249: if (delimiters == null)
250: throw new NullPointerException();
251:
252: int position = startPos;
253: while (!retDelims && position < maxPosition) {
254: if (!hasSurrogates) {
255: char c = str.charAt(position);
256: if ((c > maxDelimCodePoint)
257: || (delimiters.indexOf(c) < 0))
258: break;
259: position++;
260: } else {
261: int c = str.codePointAt(position);
262: if ((c > maxDelimCodePoint) || !isDelimiter(c)) {
263: break;
264: }
265: position += Character.charCount(c);
266: }
267: }
268: return position;
269: }
270:
271: /**
272: * Skips ahead from startPos and returns the index of the next delimiter
273: * character encountered, or maxPosition if no such delimiter is found.
274: */
275: private int scanToken(int startPos) {
276: int position = startPos;
277: while (position < maxPosition) {
278: if (!hasSurrogates) {
279: char c = str.charAt(position);
280: if ((c <= maxDelimCodePoint)
281: && (delimiters.indexOf(c) >= 0))
282: break;
283: position++;
284: } else {
285: int c = str.codePointAt(position);
286: if ((c <= maxDelimCodePoint) && isDelimiter(c))
287: break;
288: position += Character.charCount(c);
289: }
290: }
291: if (retDelims && (startPos == position)) {
292: if (!hasSurrogates) {
293: char c = str.charAt(position);
294: if ((c <= maxDelimCodePoint)
295: && (delimiters.indexOf(c) >= 0))
296: position++;
297: } else {
298: int c = str.codePointAt(position);
299: if ((c <= maxDelimCodePoint) && isDelimiter(c))
300: position += Character.charCount(c);
301: }
302: }
303: return position;
304: }
305:
306: private boolean isDelimiter(int codePoint) {
307: for (int i = 0; i < delimiterCodePoints.length; i++) {
308: if (delimiterCodePoints[i] == codePoint) {
309: return true;
310: }
311: }
312: return false;
313: }
314:
315: /**
316: * Tests if there are more tokens available from this tokenizer's string.
317: * If this method returns <tt>true</tt>, then a subsequent call to
318: * <tt>nextToken</tt> with no argument will successfully return a token.
319: *
320: * @return <code>true</code> if and only if there is at least one token
321: * in the string after the current position; <code>false</code>
322: * otherwise.
323: */
324: public boolean hasMoreTokens() {
325: /*
326: * Temporarily store this position and use it in the following
327: * nextToken() method only if the delimiters haven't been changed in
328: * that nextToken() invocation.
329: */
330: newPosition = skipDelimiters(currentPosition);
331: return (newPosition < maxPosition);
332: }
333:
334: /**
335: * Returns the next token from this string tokenizer.
336: *
337: * @return the next token from this string tokenizer.
338: * @exception NoSuchElementException if there are no more tokens in this
339: * tokenizer's string.
340: */
341: public String nextToken() {
342: /*
343: * If next position already computed in hasMoreElements() and
344: * delimiters have changed between the computation and this invocation,
345: * then use the computed value.
346: */
347:
348: currentPosition = (newPosition >= 0 && !delimsChanged) ? newPosition
349: : skipDelimiters(currentPosition);
350:
351: /* Reset these anyway */
352: delimsChanged = false;
353: newPosition = -1;
354:
355: if (currentPosition >= maxPosition)
356: throw new NoSuchElementException();
357: int start = currentPosition;
358: currentPosition = scanToken(currentPosition);
359: return str.substring(start, currentPosition);
360: }
361:
362: /**
363: * Returns the next token in this string tokenizer's string. First,
364: * the set of characters considered to be delimiters by this
365: * <tt>StringTokenizer</tt> object is changed to be the characters in
366: * the string <tt>delim</tt>. Then the next token in the string
367: * after the current position is returned. The current position is
368: * advanced beyond the recognized token. The new delimiter set
369: * remains the default after this call.
370: *
371: * @param delim the new delimiters.
372: * @return the next token, after switching to the new delimiter set.
373: * @exception NoSuchElementException if there are no more tokens in this
374: * tokenizer's string.
375: * @exception NullPointerException if delim is <CODE>null</CODE>
376: */
377: public String nextToken(String delim) {
378: delimiters = delim;
379:
380: /* delimiter string specified, so set the appropriate flag. */
381: delimsChanged = true;
382:
383: setMaxDelimCodePoint();
384: return nextToken();
385: }
386:
387: /**
388: * Returns the same value as the <code>hasMoreTokens</code>
389: * method. It exists so that this class can implement the
390: * <code>Enumeration</code> interface.
391: *
392: * @return <code>true</code> if there are more tokens;
393: * <code>false</code> otherwise.
394: * @see java.util.Enumeration
395: * @see java.util.StringTokenizer#hasMoreTokens()
396: */
397: public boolean hasMoreElements() {
398: return hasMoreTokens();
399: }
400:
401: /**
402: * Returns the same value as the <code>nextToken</code> method,
403: * except that its declared return value is <code>Object</code> rather than
404: * <code>String</code>. It exists so that this class can implement the
405: * <code>Enumeration</code> interface.
406: *
407: * @return the next token in the string.
408: * @exception NoSuchElementException if there are no more tokens in this
409: * tokenizer's string.
410: * @see java.util.Enumeration
411: * @see java.util.StringTokenizer#nextToken()
412: */
413: public Object nextElement() {
414: return nextToken();
415: }
416:
417: /**
418: * Calculates the number of times that this tokenizer's
419: * <code>nextToken</code> method can be called before it generates an
420: * exception. The current position is not advanced.
421: *
422: * @return the number of tokens remaining in the string using the current
423: * delimiter set.
424: * @see java.util.StringTokenizer#nextToken()
425: */
426: public int countTokens() {
427: int count = 0;
428: int currpos = currentPosition;
429: while (currpos < maxPosition) {
430: currpos = skipDelimiters(currpos);
431: if (currpos >= maxPosition)
432: break;
433: currpos = scanToken(currpos);
434: count++;
435: }
436: return count;
437: }
438: }
|
