CLucene - a full-featured, c++ search engine
API Documentation

---

Analyzers.h

Go to the documentation of this file.

/*------------------------------------------------------------------------------
* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team
* 
* Distributable under the terms of either the Apache License (Version 2.0) or 
* the GNU Lesser General Public License, as specified in the COPYING file.
------------------------------------------------------------------------------*/
#ifndef _lucene_analysis_Analyzers_
#define _lucene_analysis_Analyzers_

#if defined(_LUCENE_PRAGMA_ONCE)
# pragma once
#endif

#include "CLucene/util/Reader.h"
#include "AnalysisHeader.h"
#include "CLucene/util/Misc.h"
#include "CLucene/util/VoidMapSetDefinitions.h"

CL_NS_DEF(analysis)


class CharTokenizer:public Tokenizer {
private:
        int32_t offset, bufferIndex, dataLen;
        TCHAR buffer[LUCENE_MAX_WORD_LEN+1];
        const TCHAR* ioBuffer;
protected:
    
        virtual bool isTokenChar(const TCHAR c) const = 0;

        virtual TCHAR normalize(const TCHAR c) const;

public:
        CharTokenizer(CL_NS(util)::Reader* in);
        virtual ~CharTokenizer();
        bool next(Token* token);
};


class LetterTokenizer:public CharTokenizer {
public:
        // Construct a new LetterTokenizer. 
        LetterTokenizer(CL_NS(util)::Reader* in);
    virtual ~LetterTokenizer();
protected:
        bool isTokenChar(const TCHAR c) const;
};



class LowerCaseTokenizer:public LetterTokenizer {
public:
        LowerCaseTokenizer(CL_NS(util)::Reader* in);
    virtual ~LowerCaseTokenizer();
protected:
        TCHAR normalize(const TCHAR chr) const;
};


class WhitespaceTokenizer: public CharTokenizer {
public:
        WhitespaceTokenizer(CL_NS(util)::Reader* in);
        virtual ~WhitespaceTokenizer();
protected:
        bool isTokenChar(const TCHAR c) const;
};


class WhitespaceAnalyzer: public Analyzer {
public:
    WhitespaceAnalyzer();
    TokenStream* tokenStream(const TCHAR* fieldName, CL_NS(util)::Reader* reader);
    virtual ~WhitespaceAnalyzer();
};

class CLUCENE_EXPORT SimpleAnalyzer: public Analyzer {
public:
    SimpleAnalyzer();
        TokenStream* tokenStream(const TCHAR* fieldName, CL_NS(util)::Reader* reader);
        virtual ~SimpleAnalyzer();
};



class LowerCaseFilter: public TokenFilter {
public:
        LowerCaseFilter(TokenStream* in, bool deleteTokenStream);
        virtual ~LowerCaseFilter();
        bool next(Token* token);
};


class StopFilter: public TokenFilter {
private:
        //bvk: i found this to work faster with a non-hash table. the number of items
        //in the stop table is not like to make it worth having hashing.
        //ish: implement a radix/patricia tree for this?
        CLTCSetList* stopWords;
        bool deleteStopTable;

        bool enablePositionIncrements;
        const bool ignoreCase;
public:
        static bool ENABLE_POSITION_INCREMENTS_DEFAULT;

        // Constructs a filter which removes words from the input
        //      TokenStream that are named in the array of words. 
        StopFilter(TokenStream* in, bool deleteTokenStream, const TCHAR** _stopWords, const bool _ignoreCase = false);

        virtual ~StopFilter();

        StopFilter(TokenStream* in, bool deleteTokenStream, CLTCSetList* stopTable, bool _deleteStopTable=false);
        
        static void fillStopTable(CLTCSetList* stopTable,
                                      const TCHAR** stopWords, const bool _ignoreCase = false);

        bool next(Token* token);


        static bool getEnablePositionIncrementsDefault();

        static void setEnablePositionIncrementsDefault(const bool defaultValue);

        bool getEnablePositionIncrements() const;

        void setEnablePositionIncrements(const bool enable);

};

class WordlistLoader {
public:
        static CLTCSetList* getWordSet(const char* wordfilePath, const char* enc = NULL, CLTCSetList* stopTable = NULL);

        static CLTCSetList* getWordSet(CL_NS(util)::Reader* reader, CLTCSetList* stopTable = NULL, const bool bDeleteReader = false);
};


class StopAnalyzer: public Analyzer {
        CLTCSetList* stopTable;

public:
    StopAnalyzer();
    virtual ~StopAnalyzer();
    
    StopAnalyzer( const TCHAR** stopWords );

        StopAnalyzer(const char* stopwordsFile, const char* enc = NULL);

        StopAnalyzer(CL_NS(util)::Reader* stopwordsReader, const bool _bDeleteReader = false);

    TokenStream* tokenStream(const TCHAR* fieldName, CL_NS(util)::Reader* reader);
        
    static const TCHAR* ENGLISH_STOP_WORDS[];
};



class PerFieldAnalyzerWrapper : public Analyzer {
private:
    Analyzer* defaultAnalyzer;
    
    typedef CL_NS(util)::CLHashMap<const TCHAR*, Analyzer*, CL_NS(util)::Compare::TChar,
        CL_NS(util)::Equals::TChar, CL_NS(util)::Deletor::tcArray,CL_NS(util)::Deletor::Void<Analyzer> > AnalyzerMapType;
    AnalyzerMapType* analyzerMap;
public:
    PerFieldAnalyzerWrapper(Analyzer* defaultAnalyzer);
    virtual ~PerFieldAnalyzerWrapper();
    
    void addAnalyzer(const TCHAR* fieldName, Analyzer* analyzer);
    TokenStream* tokenStream(const TCHAR* fieldName, CL_NS(util)::Reader* reader);
};


class ISOLatin1AccentFilter: public TokenFilter {
public:
        ISOLatin1AccentFilter(TokenStream* input, bool deleteTs);
        
        bool next(Token* token);
        
        virtual ~ISOLatin1AccentFilter();
};


class KeywordTokenizer: public Tokenizer {
private:
    LUCENE_STATIC_CONSTANT(int, DEFAULT_BUFFER_SIZE = 256);
    bool done;
    int bufferSize;
public:
    KeywordTokenizer(CL_NS(util)::Reader* input, int bufferSize=-1);
    virtual ~KeywordTokenizer();
    bool next(Token* token);
};

class KeywordAnalyzer: public Analyzer {
public:
    TokenStream* tokenStream(const TCHAR* fieldName, CL_NS(util)::Reader* reader);
    virtual ~KeywordAnalyzer();
};

    
class LengthFilter: public TokenFilter {
private:
    size_t _min;
    size_t _max;
public:
    LengthFilter(TokenStream* in, const size_t _min, const size_t _max);
    
    bool next(Token* token);
};


CL_NS_END
#endif

---

clucene.sourceforge.net