Save This Page
Home » lucene-3.0.1-src » org.apache » lucene » analysis » [javadoc | source]
    1   package org.apache.lucene.analysis;
    2   
    3   /**
    4    * Licensed to the Apache Software Foundation (ASF) under one or more
    5    * contributor license agreements.  See the NOTICE file distributed with
    6    * this work for additional information regarding copyright ownership.
    7    * The ASF licenses this file to You under the Apache License, Version 2.0
    8    * (the "License"); you may not use this file except in compliance with
    9    * the License.  You may obtain a copy of the License at
   10    *
   11    *     http://www.apache.org/licenses/LICENSE-2.0
   12    *
   13    * Unless required by applicable law or agreed to in writing, software
   14    * distributed under the License is distributed on an "AS IS" BASIS,
   15    * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   16    * See the License for the specific language governing permissions and
   17    * limitations under the License.
   18    */
   19   
   20   import java.io.IOException;
   21   import java.util.Arrays;
   22   import java.util.Set;
   23   import java.util.List;
   24   
   25   import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
   26   import org.apache.lucene.analysis.tokenattributes.TermAttribute;
   27   import org.apache.lucene.queryParser.QueryParser; // for javadoc
   28   import org.apache.lucene.util.Version;
   29   
   30   /**
   31    * Removes stop words from a token stream.
   32    */
   33   
   34   public final class StopFilter extends TokenFilter {
   35   
   36     private final CharArraySet stopWords;
   37     private boolean enablePositionIncrements = false;
   38   
   39     private TermAttribute termAtt;
   40     private PositionIncrementAttribute posIncrAtt;
   41   
   42     /**
   43      * Construct a token stream filtering the given input.
   44      * If <code>stopWords</code> is an instance of {@link CharArraySet} (true if
   45      * <code>makeStopSet()</code> was used to construct the set) it will be directly used
   46      * and <code>ignoreCase</code> will be ignored since <code>CharArraySet</code>
   47      * directly controls case sensitivity.
   48      * <p/>
   49      * If <code>stopWords</code> is not an instance of {@link CharArraySet},
   50      * a new CharArraySet will be constructed and <code>ignoreCase</code> will be
   51      * used to specify the case sensitivity of that set.
   52      *
   53      * @param enablePositionIncrements true if token positions should record the removed stop words
   54      * @param input Input TokenStream
   55      * @param stopWords A Set of Strings or char[] or any other toString()-able set representing the stopwords
   56      * @param ignoreCase if true, all words are lower cased first
   57      */
   58     public StopFilter(boolean enablePositionIncrements, TokenStream input, Set<?> stopWords, boolean ignoreCase)
   59     {
   60       super(input);
   61       if (stopWords instanceof CharArraySet) {
   62         this.stopWords = (CharArraySet)stopWords;
   63       } else {
   64         this.stopWords = new CharArraySet(stopWords.size(), ignoreCase);
   65         this.stopWords.addAll(stopWords);
   66       }
   67       this.enablePositionIncrements = enablePositionIncrements;
   68       termAtt = addAttribute(TermAttribute.class);
   69       posIncrAtt = addAttribute(PositionIncrementAttribute.class);
   70     }
   71   
   72     /**
   73      * Constructs a filter which removes words from the input
   74      * TokenStream that are named in the Set.
   75      *
   76      * @param enablePositionIncrements true if token positions should record the removed stop words
   77      * @param in Input stream
   78      * @param stopWords A Set of Strings or char[] or any other toString()-able set representing the stopwords
   79      * @see #makeStopSet(java.lang.String[])
   80      */
   81     public StopFilter(boolean enablePositionIncrements, TokenStream in, Set<?> stopWords) {
   82       this(enablePositionIncrements, in, stopWords, false);
   83     }
   84   
   85     /**
   86      * Builds a Set from an array of stop words,
   87      * appropriate for passing into the StopFilter constructor.
   88      * This permits this stopWords construction to be cached once when
   89      * an Analyzer is constructed.
   90      * 
   91      * @see #makeStopSet(java.lang.String[], boolean) passing false to ignoreCase
   92      */
   93     public static final Set<Object> makeStopSet(String... stopWords) {
   94       return makeStopSet(stopWords, false);
   95     }
   96   
   97     /**
   98      * Builds a Set from an array of stop words,
   99      * appropriate for passing into the StopFilter constructor.
  100      * This permits this stopWords construction to be cached once when
  101      * an Analyzer is constructed.
  102      * @param stopWords A List of Strings or char[] or any other toString()-able list representing the stopwords
  103      * @return A Set ({@link CharArraySet}) containing the words
  104      * @see #makeStopSet(java.lang.String[], boolean) passing false to ignoreCase
  105      */
  106     public static final Set<Object> makeStopSet(List<?> stopWords) {
  107       return makeStopSet(stopWords, false);
  108     }
  109       
  110     /**
  111      * 
  112      * @param stopWords An array of stopwords
  113      * @param ignoreCase If true, all words are lower cased first.  
  114      * @return a Set containing the words
  115      */    
  116     public static final Set<Object> makeStopSet(String[] stopWords, boolean ignoreCase) {
  117       CharArraySet stopSet = new CharArraySet(stopWords.length, ignoreCase);
  118       stopSet.addAll(Arrays.asList(stopWords));
  119       return stopSet;
  120     }
  121   
  122     /**
  123      *
  124      * @param stopWords A List of Strings or char[] or any other toString()-able list representing the stopwords
  125      * @param ignoreCase if true, all words are lower cased first
  126      * @return A Set ({@link CharArraySet}) containing the words
  127      */
  128     public static final Set<Object> makeStopSet(List<?> stopWords, boolean ignoreCase){
  129       CharArraySet stopSet = new CharArraySet(stopWords.size(), ignoreCase);
  130       stopSet.addAll(stopWords);
  131       return stopSet;
  132     }
  133     
  134     /**
  135      * Returns the next input Token whose term() is not a stop word.
  136      */
  137     @Override
  138     public final boolean incrementToken() throws IOException {
  139       // return the first non-stop word found
  140       int skippedPositions = 0;
  141       while (input.incrementToken()) {
  142         if (!stopWords.contains(termAtt.termBuffer(), 0, termAtt.termLength())) {
  143           if (enablePositionIncrements) {
  144             posIncrAtt.setPositionIncrement(posIncrAtt.getPositionIncrement() + skippedPositions);
  145           }
  146           return true;
  147         }
  148         skippedPositions += posIncrAtt.getPositionIncrement();
  149       }
  150       // reached EOS -- return false
  151       return false;
  152     }
  153   
  154     /**
  155      * Returns version-dependent default for
  156      * enablePositionIncrements.  Analyzers that embed
  157      * StopFilter use this method when creating the
  158      * StopFilter.  Prior to 2.9, this returns false.  On 2.9
  159      * or later, it returns true.
  160      */
  161     public static boolean getEnablePositionIncrementsVersionDefault(Version matchVersion) {
  162       return matchVersion.onOrAfter(Version.LUCENE_29);
  163     }
  164   
  165     /**
  166      * @see #setEnablePositionIncrements(boolean). 
  167      */
  168     public boolean getEnablePositionIncrements() {
  169       return enablePositionIncrements;
  170     }
  171   
  172     /**
  173      * If <code>true</code>, this StopFilter will preserve
  174      * positions of the incoming tokens (ie, accumulate and
  175      * set position increments of the removed stop tokens).
  176      * Generally, <code>true</code> is best as it does not
  177      * lose information (positions of the original tokens)
  178      * during indexing.
  179      * 
  180      * <p> When set, when a token is stopped
  181      * (omitted), the position increment of the following
  182      * token is incremented.
  183      *
  184      * <p> <b>NOTE</b>: be sure to also
  185      * set {@link QueryParser#setEnablePositionIncrements} if
  186      * you use QueryParser to create queries.
  187      */
  188     public void setEnablePositionIncrements(boolean enable) {
  189       this.enablePositionIncrements = enable;
  190     }
  191   }

Save This Page
Home » lucene-3.0.1-src » org.apache » lucene » analysis » [javadoc | source]