    001    /*
    002     * Licensed to the Apache Software Foundation (ASF) under one or more
    003     * contributor license agreements.  See the NOTICE file distributed with
    004     * this work for additional information regarding copyright ownership.
    005     * The ASF licenses this file to You under the Apache License, Version 2.0
    006     * (the "License"); you may not use this file except in compliance with
    007     * the License.  You may obtain a copy of the License at
    008     *
    009     *      http://www.apache.org/licenses/LICENSE-2.0
    010     *
    011     * Unless required by applicable law or agreed to in writing, software
    012     * distributed under the License is distributed on an "AS IS" BASIS,
    013     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    014     * See the License for the specific language governing permissions and
    015     * limitations under the License.
    016     */
    017    package org.apache.commons.lang.text;
    018
    019    import java.util.ArrayList;
    020    import java.util.Collections;
    021    import java.util.List;
    022    import java.util.ListIterator;
    023    import java.util.NoSuchElementException;
    024
    025    /**
    026     * Tokenizes a string based based on delimiters (separators)
    027     * and supporting quoting and ignored character concepts.
    028     * <p>
    029     * This class can split a String into many smaller strings. It aims
    030     * to do a similar job to {@link java.util.StringTokenizer StringTokenizer},
    031     * however it offers much more control and flexibility including implementing
    032     * the <code>ListIterator</code> interface. By default, it is set up
    033     * like <code>StringTokenizer</code>.
    034     * <p>
    035     * The input String is split into a number of <i>tokens</i>.
    036     * Each token is separated from the next String by a <i>delimiter</i>.
    037     * One or more delimiter characters must be specified.
    038     * <p>
    039     * Each token may be surrounded by quotes.
    040     * The <i>quote</i> matcher specifies the quote character(s).
    041     * A quote may be escaped within a quoted section by duplicating itself.
    042     * <p>
    043     * Between each token and the delimiter are potentially characters that need trimming.
    044     * The <i>trimmer</i> matcher specifies these characters.
    045     * One usage might be to trim whitespace characters.
    046     * <p>
    047     * At any point outside the quotes there might potentially be invalid characters.
    048     * The <i>ignored</i> matcher specifies these characters to be removed.
    049     * One usage might be to remove new line characters.
    050     * <p>
    051     * Empty tokens may be removed or returned as null.
    052     * <pre>
    053     * "a,b,c"         - Three tokens "a","b","c"   (comma delimiter)
    054     * " a, b , c "    - Three tokens "a","b","c"   (default CSV processing trims whitespace)
    055     * "a, ", b ,", c" - Three tokens "a, " , " b ", ", c" (quoted text untouched)
    056     * </pre>
    057     * <p>
    058     *
    059     * This tokenizer has the following properties and options:
    060     *
    061     * <table>
    062     *  <tr>
    063     *   <th>Property</th><th>Type</th><th>Default</th>
    064     *  </tr>
    065     *  <tr>
    066     *   <td>delim</td><td>CharSetMatcher</td><td>{ \t\n\r\f}</td>
    067     *  </tr>
    068     *  <tr>
    069     *   <td>quote</td><td>NoneMatcher</td><td>{}</td>
    070     *  </tr>
    071     *  <tr>
    072     *   <td>ignore</td><td>NoneMatcher</td><td>{}</td>
    073     *  </tr>
    074     *  <tr>
    075     *   <td>emptyTokenAsNull</td><td>boolean</td><td>false</td>
    076     *  </tr>
    077     *  <tr>
    078     *   <td>ignoreEmptyTokens</td><td>boolean</td><td>true</td>
    079     *  </tr>
    080     * </table>
    081     *
    082     * @author Apache Software Foundation
    083     * @author Matthew Inger
    084     * @author Gary D. Gregory
    085     * @since 2.2
    086     * @version $Id: StrTokenizer.java 907631 2010-02-08 12:22:48Z sebb $
    087     */
    088    public class StrTokenizer implements ListIterator, Cloneable {
    089
    090        private static final StrTokenizer CSV_TOKENIZER_PROTOTYPE;
    091        private static final StrTokenizer TSV_TOKENIZER_PROTOTYPE;
    092        static {
    093            CSV_TOKENIZER_PROTOTYPE = new StrTokenizer();
    094            CSV_TOKENIZER_PROTOTYPE.setDelimiterMatcher(StrMatcher.commaMatcher());
    095            CSV_TOKENIZER_PROTOTYPE.setQuoteMatcher(StrMatcher.doubleQuoteMatcher());
    096            CSV_TOKENIZER_PROTOTYPE.setIgnoredMatcher(StrMatcher.noneMatcher());
    097            CSV_TOKENIZER_PROTOTYPE.setTrimmerMatcher(StrMatcher.trimMatcher());
    098            CSV_TOKENIZER_PROTOTYPE.setEmptyTokenAsNull(false);
    099            CSV_TOKENIZER_PROTOTYPE.setIgnoreEmptyTokens(false);
    100
    101            TSV_TOKENIZER_PROTOTYPE = new StrTokenizer();
    102            TSV_TOKENIZER_PROTOTYPE.setDelimiterMatcher(StrMatcher.tabMatcher());
    103            TSV_TOKENIZER_PROTOTYPE.setQuoteMatcher(StrMatcher.doubleQuoteMatcher());
    104            TSV_TOKENIZER_PROTOTYPE.setIgnoredMatcher(StrMatcher.noneMatcher());
    105            TSV_TOKENIZER_PROTOTYPE.setTrimmerMatcher(StrMatcher.trimMatcher());
    106            TSV_TOKENIZER_PROTOTYPE.setEmptyTokenAsNull(false);
    107            TSV_TOKENIZER_PROTOTYPE.setIgnoreEmptyTokens(false);
    108        }
    109
    110        /** The text to work on. */
    111        private char chars[];
    112        /** The parsed tokens */
    113        private String tokens[];
    114        /** The current iteration position */
    115        private int tokenPos;
    116
    117        /** The delimiter matcher */
    118        private StrMatcher delimMatcher = StrMatcher.splitMatcher();
    119        /** The quote matcher */
    120        private StrMatcher quoteMatcher = StrMatcher.noneMatcher();
    121        /** The ignored matcher */
    122        private StrMatcher ignoredMatcher = StrMatcher.noneMatcher();
    123        /** The trimmer matcher */
    124        private StrMatcher trimmerMatcher = StrMatcher.noneMatcher();
    125
    126        /** Whether to return empty tokens as null */
    127        private boolean emptyAsNull = false;
    128        /** Whether to ignore empty tokens */
    129        private boolean ignoreEmptyTokens = true;
    130
    131        //-----------------------------------------------------------------------
    132
    133        /**
    134         * Returns a clone of <code>CSV_TOKENIZER_PROTOTYPE</code>.
    135         *
    136         * @return a clone of <code>CSV_TOKENIZER_PROTOTYPE</code>.
    137         */
    138        private static StrTokenizer getCSVClone() {
    139            return (StrTokenizer) CSV_TOKENIZER_PROTOTYPE.clone();
    140        }
    141
    142        /**
    143         * Gets a new tokenizer instance which parses Comma Separated Value strings
    144         * initializing it with the given input.  The default for CSV processing
    145         * will be trim whitespace from both ends (which can be overridden with
    146         * the setTrimmer method).
    147         * <p>
    148         * You must call a "reset" method to set the string which you want to parse.
    149         * @return a new tokenizer instance which parses Comma Separated Value strings
    150         */
    151        public static StrTokenizer getCSVInstance() {
    152            return getCSVClone();
    153        }
    154
    155        /**
    156         * Gets a new tokenizer instance which parses Comma Separated Value strings
    157         * initializing it with the given input.  The default for CSV processing
    158         * will be trim whitespace from both ends (which can be overridden with
    159         * the setTrimmer method).
    160         *
    161         * @param input  the text to parse
    162         * @return a new tokenizer instance which parses Comma Separated Value strings
    163         */
    164        public static StrTokenizer getCSVInstance(String input) {
    165            StrTokenizer tok = getCSVClone();
    166            tok.reset(input);
    167            return tok;
    168        }
    169
    170        /**
    171         * Gets a new tokenizer instance which parses Comma Separated Value strings
    172         * initializing it with the given input.  The default for CSV processing
    173         * will be trim whitespace from both ends (which can be overridden with
    174         * the setTrimmer method).
    175         *
    176         * @param input  the text to parse
    177         * @return a new tokenizer instance which parses Comma Separated Value strings
    178         */
    179        public static StrTokenizer getCSVInstance(char[] input) {
    180            StrTokenizer tok = getCSVClone();
    181            tok.reset(input);
    182            return tok;
    183        }
    184
    185        /**
    186         * Returns a clone of <code>TSV_TOKENIZER_PROTOTYPE</code>.
    187         *
    188         * @return a clone of <code>TSV_TOKENIZER_PROTOTYPE</code>.
    189         */
    190        private static StrTokenizer getTSVClone() {
    191            return (StrTokenizer) TSV_TOKENIZER_PROTOTYPE.clone();
    192        }
    193
    194
    195        /**
    196         * Gets a new tokenizer instance which parses Tab Separated Value strings.
    197         * The default for CSV processing will be trim whitespace from both ends
    198         * (which can be overridden with the setTrimmer method).
    199         * <p>
    200         * You must call a "reset" method to set the string which you want to parse.
    201         * @return a new tokenizer instance which parses Tab Separated Value strings.
    202         */
    203        public static StrTokenizer getTSVInstance() {
    204            return getTSVClone();
    205        }
    206
    207        /**
    208         * Gets a new tokenizer instance which parses Tab Separated Value strings.
    209         * The default for CSV processing will be trim whitespace from both ends
    210         * (which can be overridden with the setTrimmer method).
    211         * @param input  the string to parse
    212         * @return a new tokenizer instance which parses Tab Separated Value strings.
    213         */
    214        public static StrTokenizer getTSVInstance(String input) {
    215            StrTokenizer tok = getTSVClone();
    216            tok.reset(input);
    217            return tok;
    218        }
    219
    220        /**
    221         * Gets a new tokenizer instance which parses Tab Separated Value strings.
    222         * The default for CSV processing will be trim whitespace from both ends
    223         * (which can be overridden with the setTrimmer method).
    224         * @param input  the string to parse
    225         * @return a new tokenizer instance which parses Tab Separated Value strings.
    226         */
    227        public static StrTokenizer getTSVInstance(char[] input) {
    228            StrTokenizer tok = getTSVClone();
    229            tok.reset(input);
    230            return tok;
    231        }
    232
    233        //-----------------------------------------------------------------------
    234        /**
    235         * Constructs a tokenizer splitting on space, tab, newline and formfeed
    236         * as per StringTokenizer, but with no text to tokenize.
    237         * <p>
    238         * This constructor is normally used with {@link #reset(String)}.
    239         */
    240        public StrTokenizer() {
    241            super();
    242            this.chars = null;
    243        }
    244
    245        /**
    246         * Constructs a tokenizer splitting on space, tab, newline and formfeed
    247         * as per StringTokenizer.
    248         *
    249         * @param input  the string which is to be parsed
    250         */
    251        public StrTokenizer(String input) {
    252            super();
    253            if (input != null) {
    254                chars = input.toCharArray();
    255            } else {
    256                chars = null;
    257            }
    258        }
    259
    260        /**
    261         * Constructs a tokenizer splitting on the specified delimiter character.
    262         *
    263         * @param input  the string which is to be parsed
    264         * @param delim  the field delimiter character
    265         */
    266        public StrTokenizer(String input, char delim) {
    267            this(input);
    268            setDelimiterChar(delim);
    269        }
    270
    271        /**
    272         * Constructs a tokenizer splitting on the specified delimiter string.
    273         *
    274         * @param input  the string which is to be parsed
    275         * @param delim  the field delimiter string
    276         */
    277        public StrTokenizer(String input, String delim) {
    278            this(input);
    279            setDelimiterString(delim);
    280        }
    281
    282        /**
    283         * Constructs a tokenizer splitting using the specified delimiter matcher.
    284         *
    285         * @param input  the string which is to be parsed
    286         * @param delim  the field delimiter matcher
    287         */
    288        public StrTokenizer(String input, StrMatcher delim) {
    289            this(input);
    290            setDelimiterMatcher(delim);
    291        }
    292
    293        /**
    294         * Constructs a tokenizer splitting on the specified delimiter character
    295         * and handling quotes using the specified quote character.
    296         *
    297         * @param input  the string which is to be parsed
    298         * @param delim  the field delimiter character
    299         * @param quote  the field quoted string character
    300         */
    301        public StrTokenizer(String input, char delim, char quote) {
    302            this(input, delim);
    303            setQuoteChar(quote);
    304        }
    305
    306        /**
    307         * Constructs a tokenizer splitting using the specified delimiter matcher
    308         * and handling quotes using the specified quote matcher.
    309         *
    310         * @param input  the string which is to be parsed
    311         * @param delim  the field delimiter matcher
    312         * @param quote  the field quoted string matcher
    313         */
    314        public StrTokenizer(String input, StrMatcher delim, StrMatcher quote) {
    315            this(input, delim);
    316            setQuoteMatcher(quote);
    317        }
    318
    319        /**
    320         * Constructs a tokenizer splitting on space, tab, newline and formfeed
    321         * as per StringTokenizer.
    322         * <p>
    323         * The input character array is not cloned, and must not be altered after
    324         * passing in to this method.
    325         *
    326         * @param input  the string which is to be parsed, not cloned
    327         */
    328        public StrTokenizer(char[] input) {
    329            super();
    330            this.chars = input;
    331        }
    332
    333        /**
    334         * Constructs a tokenizer splitting on the specified character.
    335         * <p>
    336         * The input character array is not cloned, and must not be altered after
    337         * passing in to this method.
    338         *
    339         * @param input  the string which is to be parsed, not cloned
    340         * @param delim the field delimiter character
    341         */
    342        public StrTokenizer(char[] input, char delim) {
    343            this(input);
    344            setDelimiterChar(delim);
    345        }
    346
    347        /**
    348         * Constructs a tokenizer splitting on the specified string.
    349         * <p>
    350         * The input character array is not cloned, and must not be altered after
    351         * passing in to this method.
    352         *
    353         * @param input  the string which is to be parsed, not cloned
    354         * @param delim the field delimiter string
    355         */
    356        public StrTokenizer(char[] input, String delim) {
    357            this(input);
    358            setDelimiterString(delim);
    359        }
    360
    361        /**
    362         * Constructs a tokenizer splitting using the specified delimiter matcher.
    363         * <p>
    364         * The input character array is not cloned, and must not be altered after
    365         * passing in to this method.
    366         *
    367         * @param input  the string which is to be parsed, not cloned
    368         * @param delim  the field delimiter matcher
    369         */
    370        public StrTokenizer(char[] input, StrMatcher delim) {
    371            this(input);
    372            setDelimiterMatcher(delim);
    373        }
    374
    375        /**
    376         * Constructs a tokenizer splitting on the specified delimiter character
    377         * and handling quotes using the specified quote character.
    378         * <p>
    379         * The input character array is not cloned, and must not be altered after
    380         * passing in to this method.
    381         *
    382         * @param input  the string which is to be parsed, not cloned
    383         * @param delim  the field delimiter character
    384         * @param quote  the field quoted string character
    385         */
    386        public StrTokenizer(char[] input, char delim, char quote) {
    387            this(input, delim);
    388            setQuoteChar(quote);
    389        }
    390
    391        /**
    392         * Constructs a tokenizer splitting using the specified delimiter matcher
    393         * and handling quotes using the specified quote matcher.
    394         * <p>
    395         * The input character array is not cloned, and must not be altered after
    396         * passing in to this method.
    397         *
    398         * @param input  the string which is to be parsed, not cloned
    399         * @param delim  the field delimiter character
    400         * @param quote  the field quoted string character
    401         */
    402        public StrTokenizer(char[] input, StrMatcher delim, StrMatcher quote) {
    403            this(input, delim);
    404            setQuoteMatcher(quote);
    405        }
    406
    407        // API
    408        //-----------------------------------------------------------------------
    409        /**
    410         * Gets the number of tokens found in the String.
    411         *
    412         * @return the number of matched tokens
    413         */
    414        public int size() {
    415            checkTokenized();
    416            return tokens.length;
    417        }
    418
    419        /**
    420         * Gets the next token from the String.
    421         * Equivalent to {@link #next()} except it returns null rather than
    422         * throwing {@link NoSuchElementException} when no tokens remain.
    423         *
    424         * @return the next sequential token, or null when no more tokens are found
    425         */
    426        public String nextToken() {
    427            if (hasNext()) {
    428                return tokens[tokenPos++];
    429            }
    430            return null;
    431        }
    432
    433        /**
    434         * Gets the previous token from the String.
    435         *
    436         * @return the previous sequential token, or null when no more tokens are found
    437         */
    438        public String previousToken() {
    439            if (hasPrevious()) {
    440                return tokens[--tokenPos];
    441            }
    442            return null;
    443        }
    444
    445        /**
    446         * Gets a copy of the full token list as an independent modifiable array.
    447         *
    448         * @return the tokens as a String array
    449         */
    450        public String[] getTokenArray() {
    451            checkTokenized();
    452            return (String[]) tokens.clone();
    453        }
    454
    455        /**
    456         * Gets a copy of the full token list as an independent modifiable list.
    457         *
    458         * @return the tokens as a String array
    459         */
    460        public List getTokenList() {
    461            checkTokenized();
    462            List list = new ArrayList(tokens.length);
    463            for (int i = 0; i < tokens.length; i++) {
    464                list.add(tokens[i]);
    465            }
    466            return list;
    467        }
    468
    469        /**
    470         * Resets this tokenizer, forgetting all parsing and iteration already completed.
    471         * <p>
    472         * This method allows the same tokenizer to be reused for the same String.
    473         *
    474         * @return this, to enable chaining
    475         */
    476        public StrTokenizer reset() {
    477            tokenPos = 0;
    478            tokens = null;
    479            return this;
    480        }
    481
    482        /**
    483         * Reset this tokenizer, giving it a new input string to parse.
    484         * In this manner you can re-use a tokenizer with the same settings
    485         * on multiple input lines.
    486         *
    487         * @param input  the new string to tokenize, null sets no text to parse
    488         * @return this, to enable chaining
    489         */
    490        public StrTokenizer reset(String input) {
    491            reset();
    492            if (input != null) {
    493                this.chars = input.toCharArray();
    494            } else {
    495                this.chars = null;
    496            }
    497            return this;
    498        }
    499
    500        /**
    501         * Reset this tokenizer, giving it a new input string to parse.
    502         * In this manner you can re-use a tokenizer with the same settings
    503         * on multiple input lines.
    504         * <p>
    505         * The input character array is not cloned, and must not be altered after
    506         * passing in to this method.
    507         *
    508         * @param input  the new character array to tokenize, not cloned, null sets no text to parse
    509         * @return this, to enable chaining
    510         */
    511        public StrTokenizer reset(char[] input) {
    512            reset();
    513            this.chars = input;
    514            return this;
    515        }
    516
    517        // ListIterator
    518        //-----------------------------------------------------------------------
    519        /**
    520         * Checks whether there are any more tokens.
    521         *
    522         * @return true if there are more tokens
    523         */
    524        public boolean hasNext() {
    525            checkTokenized();
    526            return tokenPos < tokens.length;
    527        }
    528
    529        /**
    530         * Gets the next token.
    531         *
    532         * @return the next String token
    533         * @throws NoSuchElementException if there are no more elements
    534         */
    535        public Object next() {
    536            if (hasNext()) {
    537                return tokens[tokenPos++];
    538            }
    539            throw new NoSuchElementException();
    540        }
    541
    542        /**
    543         * Gets the index of the next token to return.
    544         *
    545         * @return the next token index
    546         */
    547        public int nextIndex() {
    548            return tokenPos;
    549        }
    550
    551        /**
    552         * Checks whether there are any previous tokens that can be iterated to.
    553         *
    554         * @return true if there are previous tokens
    555         */
    556        public boolean hasPrevious() {
    557            checkTokenized();
    558            return tokenPos > 0;
    559        }
    560
    561        /**
    562         * Gets the token previous to the last returned token.
    563         *
    564         * @return the previous token
    565         */
    566        public Object previous() {
    567            if (hasPrevious()) {
    568                return tokens[--tokenPos];
    569            }
    570            throw new NoSuchElementException();
    571        }
    572
    573        /**
    574         * Gets the index of the previous token.
    575         *
    576         * @return the previous token index
    577         */
    578        public int previousIndex() {
    579            return tokenPos - 1;
    580        }
    581
    582        /**
    583         * Unsupported ListIterator operation.
    584         *
    585         * @throws UnsupportedOperationException always
    586         */
    587        public void remove() {
    588            throw new UnsupportedOperationException("remove() is unsupported");
    589        }
    590
    591        /**
    592         * Unsupported ListIterator operation.
    593         * @param obj this parameter ignored.
    594         * @throws UnsupportedOperationException always
    595         */
    596        public void set(Object obj) {
    597            throw new UnsupportedOperationException("set() is unsupported");
    598        }
    599
    600        /**
    601         * Unsupported ListIterator operation.
    602         * @param obj this parameter ignored.
    603         * @throws UnsupportedOperationException always
    604         */
    605        public void add(Object obj) {
    606            throw new UnsupportedOperationException("add() is unsupported");
    607        }
    608
    609        // Implementation
    610        //-----------------------------------------------------------------------
    611        /**
    612         * Checks if tokenization has been done, and if not then do it.
    613         */
    614        private void checkTokenized() {
    615            if (tokens == null) {
    616                if (chars == null) {
    617                    // still call tokenize as subclass may do some work
    618                    List split = tokenize(null, 0, 0);
    619                    tokens = (String[]) split.toArray(new String[split.size()]);
    620                } else {
    621                    List split = tokenize(chars, 0, chars.length);
    622                    tokens = (String[]) split.toArray(new String[split.size()]);
    623                }
    624            }
    625        }
    626
    627        /**
    628         * Internal method to performs the tokenization.
    629         * <p>
    630         * Most users of this class do not need to call this method. This method
    631         * will be called automatically by other (public) methods when required.
    632         * <p>
    633         * This method exists to allow subclasses to add code before or after the
    634         * tokenization. For example, a subclass could alter the character array,
    635         * offset or count to be parsed, or call the tokenizer multiple times on
    636         * multiple strings. It is also be possible to filter the results.
    637         * <p>
    638         * <code>StrTokenizer</code> will always pass a zero offset and a count
    639         * equal to the length of the array to this method, however a subclass
    640         * may pass other values, or even an entirely different array.
    641         *
    642         * @param chars  the character array being tokenized, may be null
    643         * @param offset  the start position within the character array, must be valid
    644         * @param count  the number of characters to tokenize, must be valid
    645         * @return the modifiable list of String tokens, unmodifiable if null array or zero count
    646         */
    647        protected List tokenize(char[] chars, int offset, int count) {
    648            if (chars == null || count == 0) {
    649                return Collections.EMPTY_LIST;
    650            }
    651            StrBuilder buf = new StrBuilder();
    652            List tokens = new ArrayList();
    653            int pos = offset;
    654
    655            // loop around the entire buffer
    656            while (pos >= 0 && pos < count) {
    657                // find next token
    658                pos = readNextToken(chars, pos, count, buf, tokens);
    659
    660                // handle case where end of string is a delimiter
    661                if (pos >= count) {
    662                    addToken(tokens, "");
    663                }
    664            }
    665            return tokens;
    666        }
    667
    668        /**
    669         * Adds a token to a list, paying attention to the parameters we've set.
    670         *
    671         * @param list  the list to add to
    672         * @param tok  the token to add
    673         */
    674        private void addToken(List list, String tok) {
    675            if (tok == null || tok.length() == 0) {
    676                if (isIgnoreEmptyTokens()) {
    677                    return;
    678                }
    679                if (isEmptyTokenAsNull()) {
    680                    tok = null;
    681                }
    682            }
    683            list.add(tok);
    684        }
    685
    686        /**
    687         * Reads character by character through the String to get the next token.
    688         *
    689         * @param chars  the character array being tokenized
    690         * @param start  the first character of field
    691         * @param len  the length of the character array being tokenized
    692         * @param workArea  a temporary work area
    693         * @param tokens  the list of parsed tokens
    694         * @return the starting position of the next field (the character
    695         *  immediately after the delimiter), or -1 if end of string found
    696         */
    697        private int readNextToken(char[] chars, int start, int len, StrBuilder workArea, List tokens) {
    698            // skip all leading whitespace, unless it is the
    699            // field delimiter or the quote character
    700            while (start < len) {
    701                int removeLen = Math.max(
    702                        getIgnoredMatcher().isMatch(chars, start, start, len),
    703                        getTrimmerMatcher().isMatch(chars, start, start, len));
    704                if (removeLen == 0 ||
    705                    getDelimiterMatcher().isMatch(chars, start, start, len) > 0 ||
    706                    getQuoteMatcher().isMatch(chars, start, start, len) > 0) {
    707                    break;
    708                }
    709                start += removeLen;
    710            }
    711
    712            // handle reaching end
    713            if (start >= len) {
    714                addToken(tokens, "");
    715                return -1;
    716            }
    717
    718            // handle empty token
    719            int delimLen = getDelimiterMatcher().isMatch(chars, start, start, len);
    720            if (delimLen > 0) {
    721                addToken(tokens, "");
    722                return start + delimLen;
    723            }
    724
    725            // handle found token
    726            int quoteLen = getQuoteMatcher().isMatch(chars, start, start, len);
    727            if (quoteLen > 0) {
    728                return readWithQuotes(chars, start + quoteLen, len, workArea, tokens, start, quoteLen);
    729            }
    730            return readWithQuotes(chars, start, len, workArea, tokens, 0, 0);
    731        }
    732
    733        /**
    734         * Reads a possibly quoted string token.
    735         *
    736         * @param chars  the character array being tokenized
    737         * @param start  the first character of field
    738         * @param len  the length of the character array being tokenized
    739         * @param workArea  a temporary work area
    740         * @param tokens  the list of parsed tokens
    741         * @param quoteStart  the start position of the matched quote, 0 if no quoting
    742         * @param quoteLen  the length of the matched quote, 0 if no quoting
    743         * @return the starting position of the next field (the character
    744         *  immediately after the delimiter, or if end of string found,
    745         *  then the length of string
    746         */
    747        private int readWithQuotes(char[] chars, int start, int len, StrBuilder workArea,
    748                                   List tokens, int quoteStart, int quoteLen)
    749        {
    750            // Loop until we've found the end of the quoted
    751            // string or the end of the input
    752            workArea.clear();
    753            int pos = start;
    754            boolean quoting = (quoteLen > 0);
    755            int trimStart = 0;
    756
    757            while (pos < len) {
    758                // quoting mode can occur several times throughout a string
    759                // we must switch between quoting and non-quoting until we
    760                // encounter a non-quoted delimiter, or end of string
    761                if (quoting) {
    762                    // In quoting mode
    763
    764                    // If we've found a quote character, see if it's
    765                    // followed by a second quote.  If so, then we need
    766                    // to actually put the quote character into the token
    767                    // rather than end the token.
    768                    if (isQuote(chars, pos, len, quoteStart, quoteLen)) {
    769                        if (isQuote(chars, pos + quoteLen, len, quoteStart, quoteLen)) {
    770                            // matched pair of quotes, thus an escaped quote
    771                            workArea.append(chars, pos, quoteLen);
    772                            pos += (quoteLen * 2);
    773                            trimStart = workArea.size();
    774                            continue;
    775                        }
    776
    777                        // end of quoting
    778                        quoting = false;
    779                        pos += quoteLen;
    780                        continue;
    781                    }
    782
    783                    // copy regular character from inside quotes
    784                    workArea.append(chars[pos++]);
    785                    trimStart = workArea.size();
    786
    787                } else {
    788                    // Not in quoting mode
    789
    790                    // check for delimiter, and thus end of token
    791                    int delimLen = getDelimiterMatcher().isMatch(chars, pos, start, len);
    792                    if (delimLen > 0) {
    793                        // return condition when end of token found
    794                        addToken(tokens, workArea.substring(0, trimStart));
    795                        return pos + delimLen;
    796                    }
    797
    798                    // check for quote, and thus back into quoting mode
    799                    if (quoteLen > 0) {
    800                        if (isQuote(chars, pos, len, quoteStart, quoteLen)) {
    801                            quoting = true;
    802                            pos += quoteLen;
    803                            continue;
    804                        }
    805                    }
    806
    807                    // check for ignored (outside quotes), and ignore
    808                    int ignoredLen = getIgnoredMatcher().isMatch(chars, pos, start, len);
    809                    if (ignoredLen > 0) {
    810                        pos += ignoredLen;
    811                        continue;
    812                    }
    813
    814                    // check for trimmed character
    815                    // don't yet know if its at the end, so copy to workArea
    816                    // use trimStart to keep track of trim at the end
    817                    int trimmedLen = getTrimmerMatcher().isMatch(chars, pos, start, len);
    818                    if (trimmedLen > 0) {
    819                        workArea.append(chars, pos, trimmedLen);
    820                        pos += trimmedLen;
    821                        continue;
    822                    }
    823
    824                    // copy regular character from outside quotes
    825                    workArea.append(chars[pos++]);
    826                    trimStart = workArea.size();
    827                }
    828            }
    829
    830            // return condition when end of string found
    831            addToken(tokens, workArea.substring(0, trimStart));
    832            return -1;
    833        }
    834
    835        /**
    836         * Checks if the characters at the index specified match the quote
    837         * already matched in readNextToken().
    838         *
    839         * @param chars  the character array being tokenized
    840         * @param pos  the position to check for a quote
    841         * @param len  the length of the character array being tokenized
    842         * @param quoteStart  the start position of the matched quote, 0 if no quoting
    843         * @param quoteLen  the length of the matched quote, 0 if no quoting
    844         * @return true if a quote is matched
    845         */
    846        private boolean isQuote(char[] chars, int pos, int len, int quoteStart, int quoteLen) {
    847            for (int i = 0; i < quoteLen; i++) {
    848                if ((pos + i) >= len || chars[pos + i] != chars[quoteStart + i]) {
    849                    return false;
    850                }
    851            }
    852            return true;
    853        }
    854
    855        // Delimiter
    856        //-----------------------------------------------------------------------
    857        /**
    858         * Gets the field delimiter matcher.
    859         *
    860         * @return the delimiter matcher in use
    861         */
    862        public StrMatcher getDelimiterMatcher() {
    863            return this.delimMatcher;
    864        }
    865
    866        /**
    867         * Sets the field delimiter matcher.
    868         * <p>
    869         * The delimitier is used to separate one token from another.
    870         *
    871         * @param delim  the delimiter matcher to use
    872         * @return this, to enable chaining
    873         */
    874        public StrTokenizer setDelimiterMatcher(StrMatcher delim) {
    875            if (delim == null) {
    876                this.delimMatcher = StrMatcher.noneMatcher();
    877            } else {
    878                this.delimMatcher = delim;
    879            }
    880            return this;
    881        }
    882
    883        /**
    884         * Sets the field delimiter character.
    885         *
    886         * @param delim  the delimiter character to use
    887         * @return this, to enable chaining
    888         */
    889        public StrTokenizer setDelimiterChar(char delim) {
    890            return setDelimiterMatcher(StrMatcher.charMatcher(delim));
    891        }
    892
    893        /**
    894         * Sets the field delimiter string.
    895         *
    896         * @param delim  the delimiter string to use
    897         * @return this, to enable chaining
    898         */
    899        public StrTokenizer setDelimiterString(String delim) {
    900            return setDelimiterMatcher(StrMatcher.stringMatcher(delim));
    901        }
    902
    903        // Quote
    904        //-----------------------------------------------------------------------
    905        /**
    906         * Gets the quote matcher currently in use.
    907         * <p>
    908         * The quote character is used to wrap data between the tokens.
    909         * This enables delimiters to be entered as data.
    910         * The default value is '"' (double quote).
    911         *
    912         * @return the quote matcher in use
    913         */
    914        public StrMatcher getQuoteMatcher() {
    915            return quoteMatcher;
    916        }
    917
    918        /**
    919         * Set the quote matcher to use.
    920         * <p>
    921         * The quote character is used to wrap data between the tokens.
    922         * This enables delimiters to be entered as data.
    923         *
    924         * @param quote  the quote matcher to use, null ignored
    925         * @return this, to enable chaining
    926         */
    927        public StrTokenizer setQuoteMatcher(StrMatcher quote) {
    928            if (quote != null) {
    929                this.quoteMatcher = quote;
    930            }
    931            return this;
    932        }
    933
    934        /**
    935         * Sets the quote character to use.
    936         * <p>
    937         * The quote character is used to wrap data between the tokens.
    938         * This enables delimiters to be entered as data.
    939         *
    940         * @param quote  the quote character to use
    941         * @return this, to enable chaining
    942         */
    943        public StrTokenizer setQuoteChar(char quote) {
    944            return setQuoteMatcher(StrMatcher.charMatcher(quote));
    945        }
    946
    947        // Ignored
    948        //-----------------------------------------------------------------------
    949        /**
    950         * Gets the ignored character matcher.
    951         * <p>
    952         * These characters are ignored when parsing the String, unless they are
    953         * within a quoted region.
    954         * The default value is not to ignore anything.
    955         *
    956         * @return the ignored matcher in use
    957         */
    958        public StrMatcher getIgnoredMatcher() {
    959            return ignoredMatcher;
    960        }
    961
    962        /**
    963         * Set the matcher for characters to ignore.
    964         * <p>
    965         * These characters are ignored when parsing the String, unless they are
    966         * within a quoted region.
    967         *
    968         * @param ignored  the ignored matcher to use, null ignored
    969         * @return this, to enable chaining
    970         */
    971        public StrTokenizer setIgnoredMatcher(StrMatcher ignored) {
    972            if (ignored != null) {
    973                this.ignoredMatcher = ignored;
    974            }
    975            return this;
    976        }
    977
    978        /**
    979         * Set the character to ignore.
    980         * <p>
    981         * This character is ignored when parsing the String, unless it is
    982         * within a quoted region.
    983         *
    984         * @param ignored  the ignored character to use
    985         * @return this, to enable chaining
    986         */
    987        public StrTokenizer setIgnoredChar(char ignored) {
    988            return setIgnoredMatcher(StrMatcher.charMatcher(ignored));
    989        }
    990
    991        // Trimmer
    992        //-----------------------------------------------------------------------
    993        /**
    994         * Gets the trimmer character matcher.
    995         * <p>
    996         * These characters are trimmed off on each side of the delimiter
    997         * until the token or quote is found.
    998         * The default value is not to trim anything.
    999         *
    1000         * @return the trimmer matcher in use
    1001         */
    1002        public StrMatcher getTrimmerMatcher() {
    1003            return trimmerMatcher;
    1004        }
    1005
    1006        /**
    1007         * Sets the matcher for characters to trim.
    1008         * <p>
    1009         * These characters are trimmed off on each side of the delimiter
    1010         * until the token or quote is found.
    1011         *
    1012         * @param trimmer  the trimmer matcher to use, null ignored
    1013         * @return this, to enable chaining
    1014         */
    1015        public StrTokenizer setTrimmerMatcher(StrMatcher trimmer) {
    1016            if (trimmer != null) {
    1017                this.trimmerMatcher = trimmer;
    1018            }
    1019            return this;
    1020        }
    1021
    1022        //-----------------------------------------------------------------------
    1023        /**
    1024         * Gets whether the tokenizer currently returns empty tokens as null.
    1025         * The default for this property is false.
    1026         *
    1027         * @return true if empty tokens are returned as null
    1028         */
    1029        public boolean isEmptyTokenAsNull() {
    1030            return this.emptyAsNull;
    1031        }
    1032
    1033        /**
    1034         * Sets whether the tokenizer should return empty tokens as null.
    1035         * The default for this property is false.
    1036         *
    1037         * @param emptyAsNull  whether empty tokens are returned as null
    1038         * @return this, to enable chaining
    1039         */
    1040        public StrTokenizer setEmptyTokenAsNull(boolean emptyAsNull) {
    1041            this.emptyAsNull = emptyAsNull;
    1042            return this;
    1043        }
    1044
    1045        //-----------------------------------------------------------------------
    1046        /**
    1047         * Gets whether the tokenizer currently ignores empty tokens.
    1048         * The default for this property is true.
    1049         *
    1050         * @return true if empty tokens are not returned
    1051         */
    1052        public boolean isIgnoreEmptyTokens() {
    1053            return ignoreEmptyTokens;
    1054        }
    1055
    1056        /**
    1057         * Sets whether the tokenizer should ignore and not return empty tokens.
    1058         * The default for this property is true.
    1059         *
    1060         * @param ignoreEmptyTokens  whether empty tokens are not returned
    1061         * @return this, to enable chaining
    1062         */
    1063        public StrTokenizer setIgnoreEmptyTokens(boolean ignoreEmptyTokens) {
    1064            this.ignoreEmptyTokens = ignoreEmptyTokens;
    1065            return this;
    1066        }
    1067
    1068        //-----------------------------------------------------------------------
    1069        /**
    1070         * Gets the String content that the tokenizer is parsing.
    1071         *
    1072         * @return the string content being parsed
    1073         */
    1074        public String getContent() {
    1075            if (chars == null) {
    1076                return null;
    1077            }
    1078            return new String(chars);
    1079        }
    1080
    1081        //-----------------------------------------------------------------------
    1082        /**
    1083         * Creates a new instance of this Tokenizer. The new instance is reset so
    1084         * that it will be at the start of the token list.
    1085         * If a {@link CloneNotSupportedException} is caught, return <code>null</code>.
    1086         *
    1087         * @return a new instance of this Tokenizer which has been reset.
    1088         */
    1089        public Object clone() {
    1090            try {
    1091                return cloneReset();
    1092            } catch (CloneNotSupportedException ex) {
    1093                return null;
    1094            }
    1095        }
    1096
    1097        /**
    1098         * Creates a new instance of this Tokenizer. The new instance is reset so that
    1099         * it will be at the start of the token list.
    1100         *
    1101         * @return a new instance of this Tokenizer which has been reset.
    1102         * @throws CloneNotSupportedException if there is a problem cloning
    1103         */
    1104        Object cloneReset() throws CloneNotSupportedException {
    1105            // this method exists to enable 100% test coverage
    1106            StrTokenizer cloned = (StrTokenizer) super.clone();
    1107            if (cloned.chars != null) {
    1108                cloned.chars = (char[]) cloned.chars.clone();
    1109            }
    1110            cloned.reset();
    1111            return cloned;
    1112        }
    1113
    1114        //-----------------------------------------------------------------------
    1115        /**
    1116         * Gets the String content that the tokenizer is parsing.
    1117         *
    1118         * @return the string content being parsed
    1119         */
    1120        public String toString() {
    1121            if (tokens == null) {
    1122                return "StrTokenizer[not tokenized yet]";
    1123            }
    1124            return "StrTokenizer" + getTokenList();
    1125        }
    1126
    1127    }