edu.illinois.cs.cogcomp.bigdata.lucene

Class Lucene

java.lang.Object

edu.illinois.cs.cogcomp.bigdata.lucene.Lucene

public class Lucene
extends Object

Simplifed Lucene interface

Author:

cheng88

Field Summary

Fields

Modifier and Type	Field and Description
static org.apache.lucene.analysis.Analyzer	AGGRESSIVE_TRANSFORM
static org.apache.lucene.analysis.Analyzer	ENGLISH
static org.apache.lucene.document.FieldType	FULL_INDEX use this when you want to store term vectors etc.
static org.apache.lucene.document.FieldType	JUST_INDEX
static org.apache.lucene.analysis.Analyzer	KEYWORD
static org.apache.lucene.analysis.Analyzer	MINIMAL
static org.apache.lucene.analysis.Analyzer	SIMPLE
static org.apache.lucene.analysis.Analyzer	STANDARD
static org.apache.lucene.util.Version	version
static org.apache.lucene.analysis.Analyzer	WHITESPACE

Constructor Summary

Constructors

Constructor and Description
Lucene()

Method Summary

Modifier and Type	Method and Description
static double	getIdf(org.apache.lucene.index.IndexReader reader, String textField, String term) returns idf for a given term
static Map<String,Float>	getIdfs(org.apache.lucene.index.IndexReader reader, String field) uses defaultSimilarity to compute idf.
static Map<String,Float>	getIdfs(org.apache.lucene.index.IndexReader reader, String field, org.apache.lucene.search.similarities.TFIDFSimilarity tfidfSIM) uses custom similarity to compute idf, use this if you want to implement IDF(numDocs,docFreq)
static int	getLuceneDocId(org.apache.lucene.index.IndexReader reader, String docIdField, String docId) returns tf-idf = (term_freq/inversve_doc_freq) for a given doc and a term.
static double	getTf(org.apache.lucene.index.IndexReader reader, int luceneDocId, String textField, String term) returns tf for a given term in a given doc
static double	getTf(org.apache.lucene.index.IndexReader reader, String docIdField, String docId, String textField, String term) returns tf for a given term in a given doc
static double	getTfIdf(org.apache.lucene.index.IndexReader reader, int luceneDocId, String textField, String term) returns tf-idf = (term_freq/inversve_doc_freq) for a given doc and a term.
static double	getTfIdf(org.apache.lucene.index.IndexReader reader, String docIdField, String docId, String textField, String term) returns tf-idf = (term_freq/inversve_doc_freq) for a given doc and a term.
static Map<String,Float>	getTfs(org.apache.lucene.index.IndexReader reader, String field, int docID) returns term freq for a given doc.
static boolean	indexExists(String pageIndex)
static boolean	isDeleted(org.apache.lucene.index.IndexReader indexReader, int docID) checks if docId is deleted in the index
static org.apache.lucene.index.IndexWriterConfig	newConfig(org.apache.lucene.analysis.Analyzer analyzer) creates a config for a writer using a specified analyzer
static org.apache.lucene.search.Query	newQuery(String field, String query) Uses default query parser
static org.apache.lucene.queryparser.classic.QueryParser	newQueryParser(String string) Uses default analyzer
static org.apache.lucene.queryparser.classic.QueryParser	newQueryParser(String field, org.apache.lucene.analysis.Analyzer analyzer)
static org.apache.lucene.index.IndexReader	ramReader(String pathToIndex)
static org.apache.lucene.index.IndexReader	reader(String pathToIndex)
static org.apache.lucene.index.IndexReader	reader(String dir, String... children)
static org.apache.lucene.index.TermsEnum	safeEnum(org.apache.lucene.index.Terms terms)
static org.apache.lucene.search.IndexSearcher	searcher(org.apache.lucene.store.Directory dir)
static org.apache.lucene.search.IndexSearcher	searcher(org.apache.lucene.index.IndexReader reader)
static org.apache.lucene.search.IndexSearcher	searcher(String path)
static org.apache.lucene.search.IndexSearcher	searcher(String dir, String... children)
static org.apache.lucene.index.IndexWriter	simpleStemmingWriter(String indexDir)
static org.apache.lucene.index.IndexWriter	simpleWriter(String pathToIndexDir) returns a index writer configured to use a simple analyzer
static org.apache.lucene.index.IndexWriter	storeOnlyWriter(String pathToIndexDir)
static Iterable<org.apache.lucene.index.TermsEnum>	terms(org.apache.lucene.index.Terms terms)
static org.apache.lucene.index.IndexWriter	writer(String pathToIndexDir, org.apache.lucene.index.IndexWriterConfig config) returns a index writer with the specified writer config.

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

FULL_INDEX

public static org.apache.lucene.document.FieldType FULL_INDEX

use this when you want to store term vectors etc.

JUST_INDEX

public static org.apache.lucene.document.FieldType JUST_INDEX

version

public static final org.apache.lucene.util.Version version

ENGLISH

public static final org.apache.lucene.analysis.Analyzer ENGLISH

STANDARD

public static final org.apache.lucene.analysis.Analyzer STANDARD

SIMPLE

public static final org.apache.lucene.analysis.Analyzer SIMPLE

KEYWORD

public static final org.apache.lucene.analysis.Analyzer KEYWORD

WHITESPACE

public static final org.apache.lucene.analysis.Analyzer WHITESPACE

AGGRESSIVE_TRANSFORM

public static final org.apache.lucene.analysis.Analyzer AGGRESSIVE_TRANSFORM

MINIMAL

public static final org.apache.lucene.analysis.Analyzer MINIMAL

Constructor Detail

Lucene

public Lucene()

Method Detail

newConfig

public static org.apache.lucene.index.IndexWriterConfig newConfig(org.apache.lucene.analysis.Analyzer analyzer)

creates a config for a writer using a specified analyzer

Parameters:

analyzer -

Returns:

writer

public static org.apache.lucene.index.IndexWriter writer(String pathToIndexDir,
                                                         org.apache.lucene.index.IndexWriterConfig config)
                                                  throws IOException

returns a index writer with the specified writer config.

Parameters:

pathToIndexDir -

config -

Returns:

Throws:

IOException

simpleWriter

public static org.apache.lucene.index.IndexWriter simpleWriter(String pathToIndexDir)
                                                        throws IOException

returns a index writer configured to use a simple analyzer

Parameters:

pathToIndexDir -

Returns:

Throws:

IOException

simpleStemmingWriter

public static org.apache.lucene.index.IndexWriter simpleStemmingWriter(String indexDir)
                                                                throws IOException

Throws:

IOException

isDeleted

public static boolean isDeleted(org.apache.lucene.index.IndexReader indexReader,
                                int docID)

checks if docId is deleted in the index

Parameters:

indexReader -

docID -

Returns:

storeOnlyWriter

public static org.apache.lucene.index.IndexWriter storeOnlyWriter(String pathToIndexDir)
                                                           throws IOException

Throws:

IOException

ramReader

public static org.apache.lucene.index.IndexReader ramReader(String pathToIndex)
                                                     throws IOException

Throws:

IOException

reader

public static org.apache.lucene.index.IndexReader reader(String dir,
                                                         String... children)
                                                  throws IOException

Throws:

IOException

reader

public static org.apache.lucene.index.IndexReader reader(String pathToIndex)
                                                  throws IOException

Throws:

IOException

searcher

public static org.apache.lucene.search.IndexSearcher searcher(org.apache.lucene.index.IndexReader reader)
                                                       throws IOException

Throws:

IOException

searcher

public static org.apache.lucene.search.IndexSearcher searcher(org.apache.lucene.store.Directory dir)
                                                       throws IOException

Throws:

IOException

searcher

public static org.apache.lucene.search.IndexSearcher searcher(String dir,
                                                              String... children)
                                                       throws IOException

Throws:

IOException

searcher

public static org.apache.lucene.search.IndexSearcher searcher(String path)
                                                       throws IOException

Throws:

IOException

terms

public static Iterable<org.apache.lucene.index.TermsEnum> terms(org.apache.lucene.index.Terms terms)

Parameters:

terms -

Returns:

an iterable terms representation of the terms in the index

safeEnum

public static org.apache.lucene.index.TermsEnum safeEnum(org.apache.lucene.index.Terms terms)

Parameters:

terms -

Returns:

termsEnum without throwing exceptions

newQueryParser

public static org.apache.lucene.queryparser.classic.QueryParser newQueryParser(String string)

Uses default analyzer

Parameters:

string -

Returns:

newQuery

public static org.apache.lucene.search.Query newQuery(String field,
                                                      String query)
                                               throws org.apache.lucene.queryparser.classic.ParseException

Uses default query parser

Parameters:

field -

query -

Returns:

Throws:

org.apache.lucene.queryparser.classic.ParseException

indexExists

public static boolean indexExists(String pageIndex)

newQueryParser

public static org.apache.lucene.queryparser.classic.QueryParser newQueryParser(String field,
                                                                               org.apache.lucene.analysis.Analyzer analyzer)

getIdfs

public static Map<String,Float> getIdfs(org.apache.lucene.index.IndexReader reader,
                                        String field)
                                 throws IOException

uses defaultSimilarity to compute idf. DefaultSimilarity computes idf as 1 + log (numDocs/ docFreq + 1)

Parameters:

reader -

field -

Returns:

Throws:

IOException

getIdfs

public static Map<String,Float> getIdfs(org.apache.lucene.index.IndexReader reader,
                                        String field,
                                        org.apache.lucene.search.similarities.TFIDFSimilarity tfidfSIM)
                                 throws IOException

uses custom similarity to compute idf, use this if you want to implement IDF(numDocs,docFreq)

Parameters:

reader -

field -

tfidfSIM -

Returns:

Throws:

IOException

getTfs

public static Map<String,Float> getTfs(org.apache.lucene.index.IndexReader reader,
                                       String field,
                                       int docID)
                                throws IOException

returns term freq for a given doc.

Parameters:

reader -

field -

Returns:

Throws:

IOException

getLuceneDocId

public static int getLuceneDocId(org.apache.lucene.index.IndexReader reader,
                                 String docIdField,
                                 String docId)
                          throws IOException

returns tf-idf = (term_freq/inversve_doc_freq) for a given doc and a term.

Parameters:

reader -

docIdField -

docId -

Returns:

Throws:

IOException

getTfIdf

public static double getTfIdf(org.apache.lucene.index.IndexReader reader,
                              String docIdField,
                              String docId,
                              String textField,
                              String term)
                       throws IOException

returns tf-idf = (term_freq/inversve_doc_freq) for a given doc and a term.

Parameters:

reader -

docIdField -

docId -

textField -

term -

Returns:

Throws:

IOException

getTfIdf

public static double getTfIdf(org.apache.lucene.index.IndexReader reader,
                              int luceneDocId,
                              String textField,
                              String term)
                       throws IOException

returns tf-idf = (term_freq/inversve_doc_freq) for a given doc and a term.

Parameters:

reader -

luceneDocId -

textField -

term -

Returns:

Throws:

IOException

getTf

public static double getTf(org.apache.lucene.index.IndexReader reader,
                           int luceneDocId,
                           String textField,
                           String term)
                    throws IOException

returns tf for a given term in a given doc

Parameters:

reader -

luceneDocId -

textField -

term -

Returns:

Throws:

IOException

getTf

public static double getTf(org.apache.lucene.index.IndexReader reader,
                           String docIdField,
                           String docId,
                           String textField,
                           String term)
                    throws IOException

returns tf for a given term in a given doc

Parameters:

reader -

docIdField -

docId -

textField -

term -

Returns:

Throws:

IOException

getIdf

public static double getIdf(org.apache.lucene.index.IndexReader reader,
                            String textField,
                            String term)
                     throws IOException

returns idf for a given term

Parameters:

reader -

textField -

term -

Returns:

Throws:

IOException