Save This Page
Home » lucene-3.0.1-src » org.apache » lucene » analysis » cn » [javadoc | source]
    1   package org.apache.lucene.analysis.cn;
    2   
    3   /**
    4    * Licensed to the Apache Software Foundation (ASF) under one or more
    5    * contributor license agreements.  See the NOTICE file distributed with
    6    * this work for additional information regarding copyright ownership.
    7    * The ASF licenses this file to You under the Apache License, Version 2.0
    8    * (the "License"); you may not use this file except in compliance with
    9    * the License.  You may obtain a copy of the License at
   10    *
   11    *     http://www.apache.org/licenses/LICENSE-2.0
   12    *
   13    * Unless required by applicable law or agreed to in writing, software
   14    * distributed under the License is distributed on an "AS IS" BASIS,
   15    * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   16    * See the License for the specific language governing permissions and
   17    * limitations under the License.
   18    */
   19   
   20   import java.io.IOException;
   21   import java.io.Reader;
   22   import org.apache.lucene.analysis.Analyzer;
   23   import org.apache.lucene.analysis.TokenStream;
   24   import org.apache.lucene.analysis.Tokenizer;
   25   
   26   /**
   27    * An {@link Analyzer} that tokenizes text with {@link ChineseTokenizer} and
   28    * filters with {@link ChineseFilter}
   29    *
   30    */
   31   
   32   public class ChineseAnalyzer extends Analyzer {
   33   
   34       public ChineseAnalyzer() {
   35       }
   36   
   37       /**
   38       * Creates a {@link TokenStream} which tokenizes all the text in the provided {@link Reader}.
   39       *
   40       * @return  A {@link TokenStream} built from a {@link ChineseTokenizer} 
   41       *   filtered with {@link ChineseFilter}.
   42       */
   43       @Override
   44       public final TokenStream tokenStream(String fieldName, Reader reader) {
   45           TokenStream result = new ChineseTokenizer(reader);
   46           result = new ChineseFilter(result);
   47           return result;
   48       }
   49       
   50       private class SavedStreams {
   51         Tokenizer source;
   52         TokenStream result;
   53       };
   54   
   55       /**
   56       * Returns a (possibly reused) {@link TokenStream} which tokenizes all the text in the
   57       * provided {@link Reader}.
   58       * 
   59       * @return A {@link TokenStream} built from a {@link ChineseTokenizer} 
   60       *   filtered with {@link ChineseFilter}.
   61       */
   62       @Override
   63       public final TokenStream reusableTokenStream(String fieldName, Reader reader)
   64         throws IOException {
   65         /* tokenStream() is final, no back compat issue */
   66         SavedStreams streams = (SavedStreams) getPreviousTokenStream();
   67         if (streams == null) {
   68           streams = new SavedStreams();
   69           streams.source = new ChineseTokenizer(reader);
   70           streams.result = new ChineseFilter(streams.source);
   71           setPreviousTokenStream(streams);
   72         } else {
   73           streams.source.reset(reader);
   74         }
   75         return streams.result;
   76       }
   77   }

Save This Page
Home » lucene-3.0.1-src » org.apache » lucene » analysis » cn » [javadoc | source]