lucene搜索脚手架

本文旨在做一个lucene的搜索模板，提供类似于优酷的视频搜索服务。

 效果：

基于lucene3.6.1

由于数量少就没有分页，实际上是可以分页的，例子是20个结果一页。

有些重复用相同原理的搜索选项就没弄上去了。

中文分词用的是IKAnalyzer2012.tag的作用在于增加匹配的权重，例子的tag是sfa.

既然是模板，当然主要是用了java设计模式里面的模板方法了。

 建索引的模板：

package index;

import java.io.File;
import java.io.IOException;
import java.sql.SQLException;
import java.text.ParseException;
import java.util.List;
import java.util.Map;

import javax.sql.DataSource;

import org.apache.commons.dbutils.QueryRunner;
import org.apache.commons.dbutils.handlers.MapListHandler;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.store.FSDirectory;
import org.springframework.context.ApplicationContext;
import org.springframework.context.support.ClassPathXmlApplicationContext;
import org.wltea.analyzer.lucene.IKAnalyzer;

import com.mchange.v2.c3p0.ComboPooledDataSource;

public abstract class Abstract_Index_builder implements Index_Builder {
    private Analyzer analyzer = new IKAnalyzer();

    public DataSource get_data_source() {
        ApplicationContext ctx = new ClassPathXmlApplicationContext(
                new String[] { "applicationContext.xml" });//spring配置
        ComboPooledDataSource c3po_ds = (ComboPooledDataSource) ctx
                .getBean("dataSource");//spring bean,也就是c3po连接池的实例
        return (DataSource) c3po_ds;
    }

    public List<Map<String, Object>> do_query(DataSource ds, String sql)
            throws SQLException {
        QueryRunner qr = new QueryRunner();//commons dbutils的queryRunner查询数据库
        List<Map<String, Object>> results = (List<Map<String, Object>>) qr
                .query(ds.getConnection(), sql, new MapListHandler());//在继承模板的之类中具体实现查询 
        return results;
    }

    public abstract void add_to_writer(List<Map<String, Object>> data,
            IndexWriter writer);

    public void generate_index(String path, String sql) throws IOException,
            SQLException, ParseException {//开始建索引
        IndexWriterConfig conf = new IndexWriterConfig(
                org.apache.lucene.util.Version.LUCENE_36, analyzer);
        conf.setOpenMode(OpenMode.CREATE);
        File file = new File(path);
        FSDirectory directory = FSDirectory.open(file);
        IndexWriter writer = new IndexWriter(directory, conf);
        List<Map<String, Object>> data = do_query(get_data_source(), sql);
        add_to_writer(data, writer);//写入
        writer.close();
    }
}

具体的实现子类：

package index;

import java.io.IOException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.List;
import java.util.Map;

import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.NumericField;
import org.apache.lucene.document.Field.Index;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.index.IndexWriter;

public class Generate_Video_Index extends Abstract_Index_builder {

    @Override
    public void add_to_writer(List<Map<String, Object>> data, IndexWriter writer) {
        SimpleDateFormat sdf = new SimpleDateFormat("yyyy.MM.dd");
        for (int i = 0, len = data.size(); i < len; i++) {
            Document doc = new Document();
            Date date = null;
            try {
                date = sdf
                        .parse(data.get(i).get("video_uploadtime").toString());
            } catch (ParseException e) {
                e.printStackTrace();
            }
            Field video_name = new Field("video_name", data.get(i)
                    .get("video_name").toString(), Store.YES, Index.ANALYZED);
            Field tag = new Field("tag", data.get(i).get("tag").toString(),
                    Store.YES, Index.ANALYZED);
            NumericField video_uploadtime = new NumericField(
                    "video_uploadtime", Store.YES, true).setLongValue(date
                    .getTime());//发布时间
            NumericField duration = new NumericField("duration", Store.YES,
                    true).setIntValue((int) data.get(i).get("duration"));//片长
            Field id = new Field("id", data.get(i).get("id").toString(),
                    Store.YES, Index.NOT_ANALYZED);//用户id
            NumericField video_type_id = new NumericField("video_type_id",
                    Store.YES, true).setIntValue((int) data.get(i).get(
                    "video_type_id"));//分类
            NumericField chanel_id = new NumericField("chanel_id", Store.YES,
                    true).setIntValue((int) data.get(i).get("chanel_id"));
            NumericField video_id = new NumericField("video_id", Store.YES,
                    true).setIntValue((int) data.get(i).get("video_id"));
            Field name = new Field("name", data.get(i).get("name").toString(),
                    Store.YES, Index.NOT_ANALYZED);
            Field watch = new Field("watch", data.get(i)
                    .get("video_watchtimes").toString(), Store.YES,
                    Index.NOT_ANALYZED);//播放次数
            doc.add(video_id);
            doc.add(chanel_id);
            doc.add(video_name);
            doc.add(tag);
            doc.add(video_uploadtime);
            doc.add(id);
            doc.add(name);
            doc.add(watch);
            doc.add(duration);
            doc.add(video_type_id);
            try {
                writer.addDocument(doc);
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    }
}

调用建立索引模板：

实际上不用把需要展示的所有搜索结果项都写入lucene，只用把id之类的写入就可以了。lucene完成查询后取出id再从数据库查询一遍。例子只是为了简单方便才那样做的。

package index;

import java.io.IOException;
import java.sql.SQLException;
import java.text.ParseException;

public class Build_Index {
    public static void build_index() throws SQLException, IOException,
            java.text.ParseException {
        Abstract_Index_builder video_index_builder = new Generate_Video_Index();
        video_index_builder
                .generate_index(
                        "E://lucene/video",
                        "select user.name,vi.video_id,vi.video_name,vi.video_watchtimes,vi.video_uploadtime,vi.tag,vi.id,vi.duration,type.video_type_id,vi.chanel_id "
                                + "from video_info vi inner join user as "
                                + "user on vi.id=user.id inner join video_sub_type as sub_type on sub_type.video_sub_type_id=vi.video_sub_type_id inner "
                                + "join video_type as type on type.video_type_id=sub_type.video_type_id");
    }

    public static void main(String[] args) throws SQLException, IOException,
            ParseException {
        Build_Index.build_index();
    }
}

搜索模板：

package search;

import java.io.File;
import java.io.IOException;
import java.util.List;
import java.util.Map;

import javax.servlet.http.HttpServletRequest;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.TopDocsCollector;
import org.apache.lucene.search.TopScoreDocCollector;
import org.apache.lucene.search.highlight.Formatter;
import org.apache.lucene.search.highlight.Fragmenter;
import org.apache.lucene.search.highlight.Highlighter;
import org.apache.lucene.search.highlight.QueryScorer;
import org.apache.lucene.search.highlight.Scorer;
import org.apache.lucene.search.highlight.SimpleFragmenter;
import org.apache.lucene.search.highlight.SimpleHTMLFormatter;
import org.apache.lucene.store.FSDirectory;
import org.apache.struts2.interceptor.ServletRequestAware;
import org.wltea.analyzer.lucene.IKAnalyzer;

import com.opensymphony.xwork2.ActionSupport;

public abstract class Abstract_Search_builder extends ActionSupport implements
        Search_Builder, ServletRequestAware {//struts2
    protected Analyzer analyzer = new IKAnalyzer();
    public HttpServletRequest request;
    protected int count;

    public int getCount() {
        return count;
    }

    public void setCount(int count) {
        this.count = count;
    }

    public abstract List<Map<String, Object>> get_search_result(
            ScoreDoc[] docs, Highlighter highlighter, IndexSearcher search,
            int count);//获取搜索结果写入request

    public void setServletRequest(HttpServletRequest request) {
        this.request = request;
    }

    public List<Map<String, Object>> do_search(int start, int offset,
            TopDocsCollector c, Query query1, String path) throws IOException {
        FSDirectory d = FSDirectory.open(new File(path));
        IndexReader r = IndexReader.open(d);
        IndexSearcher search = new IndexSearcher(r);
        if (c == null) {//不是分页请求
            c = TopScoreDocCollector.create(search.maxDoc(), false);
            offset = search.maxDoc();
            start = 0;
        }
        search.search(query1, c);
        int count = c.getTotalHits();
        ScoreDoc[] docs = c.topDocs(offset * start, offset).scoreDocs;
        Formatter formatter = new SimpleHTMLFormatter("<font color='red'>",
                "</font>");
        Scorer fragmentScore = new QueryScorer(query1);
        Highlighter highlighter = new Highlighter(formatter, fragmentScore);
        Fragmenter fragmenter = new SimpleFragmenter(100);
        highlighter.setTextFragmenter(fragmenter);
        List<Map<String, Object>> list = get_search_result(docs, highlighter,
                search, count);
        search.close();
        return list;
    }
}

搜索的实现子类：

package search;

import java.io.IOException;
import java.sql.SQLException;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.queryParser.QueryParser;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.TopDocsCollector;
import org.apache.lucene.search.TopFieldCollector;
import org.apache.lucene.search.TopScoreDocCollector;
import org.apache.lucene.search.highlight.Highlighter;
import org.apache.lucene.search.highlight.InvalidTokenOffsetsException;
import org.apache.lucene.util.Version;

import output.CommonOutput;
import output.Strategy;

import util.LuceneUtil;

public class Video_Search extends Abstract_Search_builder {
    private String word;
    private String tag;
    private int start;
    private int order_type;
    private int limit_date;
    private Album_Search album_search;
    private Map<String, Object> info = null;
    private int offset = 20;
    private int watch_filter;
    private int time_filter;
    private int type_filter;

    public void setAlbum_search(Album_Search album_search) {
        this.album_search = album_search;
    }

    public int getLimit_date() {
        return limit_date;
    }

    public void setLimit_date(int limit_date) {
        this.limit_date = limit_date;
    }

    public int getWatch_filter() {
        return watch_filter;
    }

    public void setWatch_filter(int watch_filter) {
        this.watch_filter = watch_filter;
    }

    public int getTime_filter() {
        return time_filter;
    }

    public void setTime_filter(int time_filter) {
        this.time_filter = time_filter;
    }

    public int getType_filter() {
        return type_filter;
    }

    public void setType_filter(int type_filter) {
        this.type_filter = type_filter;
    }

    public int getOrder_type() {
        return order_type;
    }

    public void setOrder_type(int order_type) {
        this.order_type = order_type;
    }

    public int getStart() {
        return start;
    }

    public void setStart(int start) {
        this.start = start;
    }

    public String getWord() {
        return word;
    }

    public void setWord(String word) {
        this.word = word;
    }

    public String getTag() {
        return tag;
    }

    public void setTag(String tag) {
        this.tag = tag;
    }

    public Query complex_video_query(String w, String t, Query q3, Query q4,
            Query q5) throws ParseException,
            org.apache.lucene.queryParser.ParseException {
        QueryParser qp = new QueryParser(Version.LUCENE_36, "video_name",
                analyzer);
        Query q1 = qp.parse(w);
        QueryParser parser = new QueryParser(Version.LUCENE_36, "tag", analyzer);
        Query q2 = parser.parse(t);
        BooleanQuery query1 = new BooleanQuery();
        query1.add(q1, BooleanClause.Occur.MUST);
        query1.add(q2, BooleanClause.Occur.SHOULD);
        LuceneUtil.build_bool_query(query1, q3);
        LuceneUtil.build_bool_query(query1, q4);
        LuceneUtil.build_bool_query(query1, q5);//将查询条件组合
        return query1;
    }

    public String video_search() throws IOException, ParseException,
            InvalidTokenOffsetsException, NumberFormatException,
            java.text.ParseException, SQLException,
            org.apache.lucene.queryParser.ParseException {//请求从这里进入
        Sort sort = null;
        TopDocsCollector c = null;
        Query q3 = null;
        Query q4 = null;
        Query q5 = null;
        int max_d = 0;
        int min_d = 0;
        switch (getOrder_type()) {//排序
        case 1:
            sort = new Sort(new SortField("video_uploadtime", SortField.LONG,
                    true));
            break;
        case 2:
            sort = new Sort(new SortField("watch", SortField.INT, true));
            break;
        default:
            break;
        }
        if (sort == null)
            c = TopScoreDocCollector
                    .create(offset * getStart() + offset, false);
        else
            c = TopFieldCollector.create(sort, offset * getStart() + offset,
                    false, false, false, false);
        switch (getWatch_filter()) {//结果过滤
        case 1:
            max_d = 10800;
            min_d = 3600;
            break;
        case 2:
            max_d = 3600;
            min_d = 1800;
            break;
        case 3:
            max_d = 1800;
            min_d = 600;
            break;
        case 4:
            max_d = 600;
            min_d = 0;
            break;
        default:
            max_d = 0;
            min_d = 0;
            break;
        }
        if (getWatch_filter() < 5 && getWatch_filter() > 0) {
            q3 = LuceneUtil.int_range_query(min_d, max_d, "duration");
        }
        if (getTime_filter() == 1) {
            Date before_date = LuceneUtil.getDateBefore(new Date(),
                    getLimit_date());
            q4 = LuceneUtil.long_range_query(before_date.getTime(),
                    new Date().getTime(), "video_uploadtime");
        }
        if (getType_filter() != 0)
            q5 = LuceneUtil.int_range_query(getType_filter(), getType_filter(),
                    "video_type_id");
        Query query1 = complex_video_query(getWord(), getTag(), q3, q4, q5);
        List<Map<String, Object>> list = do_search(getStart(), offset, c,
                query1, "E://lucene/video");
        Strategy strategy1 = new CommonOutput();
        strategy1.output(list, request, info, "result");
        request.setAttribute("q", getWord());
        request.setAttribute("s", getOrder_type());
        request.setAttribute("wf", getWatch_filter());
        request.setAttribute("timef", getTime_filter());
        request.setAttribute("typef", getType_filter());
        request.setAttribute("ld", getLimit_date());
        request.setAttribute("cur", getStart());
        request.setAttribute("count", Math.ceil(this.getCount() / offset));
        return "success";
    }

    @Override
    public List<Map<String, Object>> get_search_result(ScoreDoc[] docs,
            Highlighter highlighter, IndexSearcher search, int count) {
        this.setCount(count);
        List<Map<String, Object>> list = new ArrayList<Map<String, Object>>();
        for (int i = 0; i < docs.length; i++) {
            int sd = docs[i].doc;
            Map<String, Object> map = new HashMap<String, Object>();
            String video_name1 = null;
            String video_name = null;
            String name = null;
            String tag1 = null;
            String tag = null;
            String duration = null;
            String id = null;
            String watch = null;
            String video_type_id = null;
            String video_id = null;
            try {
                video_name1 = search.doc(sd).get("video_name");
                if (video_name1.length() > 12)
                    video_name1 = video_name1.substring(0, 12) + "..";
                tag1 = search.doc(sd).get("tag");
                duration = search.doc(sd).get("duration");
                id = search.doc(sd).get("id");
                video_id = search.doc(sd).get("video_id");
                name = search.doc(sd).get("name");
                watch = search.doc(sd).get("watch");
                video_type_id = search.doc(sd).get("video_type_id");
            } catch (CorruptIndexException e1) {
                e1.printStackTrace();
            } catch (IOException e1) {
                e1.printStackTrace();
            }
            try {
                video_name = highlighter.getBestFragment(analyzer,
                        "video_name", video_name1);
                tag = highlighter.getBestFragment(analyzer, "tag", tag1);
            } catch (IOException | InvalidTokenOffsetsException e1) {
                e1.printStackTrace();
            }
            if (tag == null)
                tag = tag1;
            if (video_name == null)
                video_name = video_name1;
            map.put("duration",
                    LuceneUtil.cal_duration(Integer.parseInt(duration)));
            // try {
            // map.put("video_uploadtime",
            // Long.parseLong(search.doc(sd).get("video_uploadtime")));
            // } catch (NumberFormatException | IOException e) {
            // // TODO Auto-generated catch block
            // e.printStackTrace();
            // }
            try {
                map.put("duration",
                        LuceneUtil.cal_duration(Integer.parseInt(duration)));
                map.put("video_uploadtime",
                        LuceneUtil.dateDiff(
                                Long.parseLong(search.doc(sd).get(
                                        "video_uploadtime")),
                                new Date().getTime()));
            } catch (NumberFormatException | ParseException | IOException e) {
                e.printStackTrace();
            }
            map.put("video_id", video_id);
            map.put("tag", tag);
            map.put("video_name", video_name);
            map.put("id", id);
            map.put("name", name);
            map.put("watch", watch);
            map.put("video_type_id", video_type_id);
            list.add(map);
        }
        return list;
    }
}

工具类:

package util;

import java.sql.SQLException;
import java.util.Calendar;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.Set;

import javax.sql.DataSource;

import org.apache.commons.dbutils.QueryRunner;
import org.apache.commons.dbutils.handlers.MapListHandler;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.NumericRangeQuery;
import org.apache.lucene.search.Query;
import org.json.JSONArray;
import org.json.JSONObject;
import org.springframework.context.ApplicationContext;
import org.springframework.context.support.ClassPathXmlApplicationContext;

import com.mchange.v2.c3p0.ComboPooledDataSource;

public class LuceneUtil {
    public static String toJson(List<Map<String, Object>> list)
            throws Exception {//转换成json
        JSONArray array = new JSONArray();
        JSONObject member = null;
        for (Map<String, Object> map : list) {
            member = new JSONObject();
            Set<Map.Entry<String, Object>> sets = map.entrySet();
            for (Map.Entry<String, Object> entry : sets) {
                member.put((String) entry.getKey(), (Object) entry.getValue());
            }
            array.put(member);
        }
        return array.toString();
    }

    public static Query build_bool_query(BooleanQuery q1, Query q2) {
        if (q2 != null)
            q1.add(q2, BooleanClause.Occur.MUST);
        return q1;
    }

    public static String cal_duration(int second) {//片长格式化
        String h = "0";
        String d = "0";
        String s = "0";
        int temp = second % 3600;
        if (second > 3600) {
            h = String.valueOf(second / 3600);
            if (Integer.parseInt(h) < 10) {
                h = "0" + h;
            }
            if (temp != 0) {
                if (temp > 60) {
                    d = String.valueOf(temp / 60);
                    if (Integer.parseInt(d) < 10) {
                        d = "0" + d;
                    }
                    if (temp % 60 != 0) {
                        s = String.valueOf(temp % 60);
                        if (Integer.parseInt(s) < 10) {
                            s = "0" + s;
                        }
                    }
                } else {
                    s = String.valueOf(temp);
                    d = "00";
                    if (Integer.parseInt(s) < 10) {
                        s = "0" + s;
                    }
                }
            } else {
                d = "00";
                s = "00";
            }
        } else {
            h = "00";
            d = String.valueOf(second / 60);
            if (Integer.parseInt(d) < 10) {
                d = "0" + d;
            }
            if (Integer.parseInt(d) % 60 == 0) {

                h = String.valueOf((Integer.parseInt(d) / 60));
                if (Integer.parseInt(h) < 10) {
                    h = "0" + h;
                }
                d = "00";
            }
            s = "00";
            if (second % 60 != 0) {
                s = String.valueOf(second % 60);
                if (Integer.parseInt(s) < 10) {
                    s = "0" + s;
                }
            }
        }
        if (h == "00")
            return d + ":" + s;
        if (d == "00")
            return s;
        return h + ":" + d + ":" + s;
    }

    public static String dateDiff(long startTime, long endTime)
            throws java.text.ParseException {//将上传时间转换为多少日(月，年)前
        long nd = 1000 * 24 * 60 * 60;
        long diff = endTime - startTime;
        long day = diff / nd;
        if (day > 30 && day < 365)
            return day / 30 + "月前";
        if (0 <= day && day <= 30)
            return day + "天前";
        if (day > 365)
            return day / 365 + "年前";
        return null;
    }
    //范围查询
    public static Query long_range_query(final long min, final long max,
            final String field) {
        return NumericRangeQuery.newLongRange(field, min, max, true, true);
    }

    public static Query int_range_query(int min, int max, final String field) {
        return NumericRangeQuery.newIntRange(field, min, max, true, true);
    }

    public static Date getDateBefore(Date d, int day) {
        Calendar now = Calendar.getInstance();
        now.setTime(d);
        now.set(Calendar.DATE, now.get(Calendar.DATE) - day);
        return now.getTime();
    }

    public static DataSource get_data_source() {
        ApplicationContext ctx = new ClassPathXmlApplicationContext(
                new String[] { "applicationContext.xml" });
        ComboPooledDataSource c3po_ds = (ComboPooledDataSource) ctx
                .getBean("dataSource");
        return (DataSource) c3po_ds;
    }

    public static List<Map<String, Object>> simple_query(DataSource ds,
            String sql, Object[] params) throws SQLException {
        QueryRunner qr = new QueryRunner();
        List<Map<String, Object>> results = (List<Map<String, Object>>) qr
                .query(ds.getConnection(), sql, new MapListHandler(), params);
        return results;
    }
}

个人感觉优酷的视频搜索应该用的就是lucene或者solr构建的，因为它的搜索域的session cookie中有jsessionid,这个是即便手动从后台代码中显示更换名字都是不行，因为它默认就在servelet中给你加了，除非去改servelet的源码。

另外，lucene建索引的速度比sphinx慢很多，而查询速度也比sphinx慢一点。所以如果应用的大部分不是构建在java上或者应用不大，还是用sphinx就已经可以很好的满足上面那些查询条件的需求了。

转载请注明:TheViper    http://www.cnblogs.com/TheViper