Save This Page
Home » lucene-3.0.1-src » org.apache » lucene » analysis » cn » [javadoc | source]
    1   package org.apache.lucene.analysis.cn;
    2   
    3   /**
    4    * Licensed to the Apache Software Foundation (ASF) under one or more
    5    * contributor license agreements.  See the NOTICE file distributed with
    6    * this work for additional information regarding copyright ownership.
    7    * The ASF licenses this file to You under the Apache License, Version 2.0
    8    * (the "License"); you may not use this file except in compliance with
    9    * the License.  You may obtain a copy of the License at
   10    *
   11    *     http://www.apache.org/licenses/LICENSE-2.0
   12    *
   13    * Unless required by applicable law or agreed to in writing, software
   14    * distributed under the License is distributed on an "AS IS" BASIS,
   15    * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   16    * See the License for the specific language governing permissions and
   17    * limitations under the License.
   18    */
   19   
   20   import java.io.IOException;
   21   import java.util.Arrays;
   22   
   23   import org.apache.lucene.analysis.CharArraySet;
   24   import org.apache.lucene.analysis.TokenFilter;
   25   import org.apache.lucene.analysis.TokenStream;
   26   import org.apache.lucene.analysis.tokenattributes.TermAttribute;
   27   
   28   /**
   29    * A {@link TokenFilter} with a stop word table.  
   30    * <ul>
   31    * <li>Numeric tokens are removed.
   32    * <li>English tokens must be larger than 1 character.
   33    * <li>One Chinese character as one Chinese word.
   34    * </ul>
   35    * TO DO:
   36    * <ol>
   37    * <li>Add Chinese stop words, such as \ue400
   38    * <li>Dictionary based Chinese word extraction
   39    * <li>Intelligent Chinese word extraction
   40    * </ol>
   41    * 
   42    * @version 1.0
   43    *
   44    */
   45   
   46   public final class ChineseFilter extends TokenFilter {
   47   
   48   
   49       // Only English now, Chinese to be added later.
   50       public static final String[] STOP_WORDS = {
   51       "and", "are", "as", "at", "be", "but", "by",
   52       "for", "if", "in", "into", "is", "it",
   53       "no", "not", "of", "on", "or", "such",
   54       "that", "the", "their", "then", "there", "these",
   55       "they", "this", "to", "was", "will", "with"
   56       };
   57   
   58   
   59       private CharArraySet stopTable;
   60   
   61       private TermAttribute termAtt;
   62       
   63       public ChineseFilter(TokenStream in) {
   64           super(in);
   65   
   66           stopTable = new CharArraySet(Arrays.asList(STOP_WORDS), false);
   67           termAtt = addAttribute(TermAttribute.class);
   68       }
   69   
   70       @Override
   71       public boolean incrementToken() throws IOException {
   72   
   73           while (input.incrementToken()) {
   74               char text[] = termAtt.termBuffer();
   75               int termLength = termAtt.termLength();
   76   
   77             // why not key off token type here assuming ChineseTokenizer comes first?
   78               if (!stopTable.contains(text, 0, termLength)) {
   79                   switch (Character.getType(text[0])) {
   80   
   81                   case Character.LOWERCASE_LETTER:
   82                   case Character.UPPERCASE_LETTER:
   83   
   84                       // English word/token should larger than 1 character.
   85                       if (termLength>1) {
   86                           return true;
   87                       }
   88                       break;
   89                   case Character.OTHER_LETTER:
   90   
   91                       // One Chinese character as one Chinese word.
   92                       // Chinese word extraction to be added later here.
   93   
   94                       return true;
   95                   }
   96   
   97               }
   98   
   99           }
  100           return false;
  101       }
  102   
  103   }

Save This Page
Home » lucene-3.0.1-src » org.apache » lucene » analysis » cn » [javadoc | source]