Lucene.Net+KTDictSeg中文分词搭建全文检索引擎

What is Lucene——Apache Lucene

Apache Lucene(TM) is a high-performance, full-featured text search engine library written entirely in Java. It is a technology suitable for nearly any application that requires full-text search, especially cross-platform.

Apache Lucene is an open source project available for free download. Please use the links on the left to access Lucene.

使用Lucene的一些心得

最先用Lucene的时候不是使用Java下的开发包，而是用了.Net下的Lucene.Net.dll，用于搭建一个全文检索的引擎，用以全文检索用户上传的文档。效果图如下：

 

英文的全文检索相对于中文来说，简单一些，因为英文单词可以以空格为分割的依据，而中文的分割要根据语义，单词的解析模糊，很难界定。例如，最经典的：

长春市长春药店。→长春市|长春药店 →长春市长|春药店...

中文分词算法，分为几大类：基于字符串匹配的分词算法；基于理解的分词算法；基于统计的分词算法。

对于这些算法，网上有很多资料，感兴趣的话可以看看。

 

而我这个系统中，使用的是eaglet前辈的KTDictSeg分词组件，(eaglet前辈是著名的盘古分词的作者)。

OK，现在讲一讲我如何使用Lucene.net.dll和KTDictSeg分词组件来实现全文检索：

1.为项目添加引用，如下：

 Lucene.Net.dll

 

2.现在系统便有了Lucene和中文分词功能的支持。添加一个类，字段可以与你要索引的文件的字段一样。

这里我们假设我们添加了A类，字段有：DomID，Title，SynTime，HtmPath，KeyWords

 

3.添加完类，添加一个检索类，用于新建索引和搜索，代码如下：

using System;
using System.Collections.Generic;
using System.Linq;
using System.Web;
using System.Text;
using System.Text.RegularExpressions;
using System.IO;
using Lucene.Net;
using Lucene.Net.Analysis;
using Lucene.Net.Analysis.Standard;
using Lucene.Net.Documents;
using Lucene.Net.Index;
using Lucene.Net.QueryParsers;
using Lucene.Net.Search;
using Lucene.Net.Store;
using Lucene.Net.Util;
using Lucene.Net.Analysis.KTDictSeg;
using KTDictSeg.HighLight;

namespace LearnAndPracticeBLL
{
    public class DictLucenceSearch
    {
        //用于存储检索出数据总数 
        private int _count;
        public int Count
        {
            get { return _count; }
            set { _count = value; }
        }

        //搜索关键字 
        private string _keyword; 
        public string KeyWord
        {
            get { return _keyword; }
            set { _keyword = value; }
        }

        //搜索时间 
        private double _time;        
        public double Time
        {
            get { return _time; }
            set { _time = value; }
        }

        //索引的路径
        private string _myindex; 

        public string Myindex
        {
            get { return _myindex; }
            set
            {
                _myindex = System.Web.HttpContext.Current.Server.MapPath("~/"+value);
            }
        }

        ///<summary>
        ///对多个文档添加索引
        ///</summary>
        ///<param name="writer"></param>
        ///<param name="mydoc"></param>
        public static void AddDocument(IndexWriter writer,List<LearnAndPracticeModel.Document> mydocs)
        {
            for (int i = 0; i < mydocs.Count; i++)
            {
                Lucene.Net.Documents.Document doc = new Lucene.Net.Documents.Document();
                Field DomID = new Field("DomID", mydocs[i].DomID.ToString(), Field.Store.YES, Field.Index.TOKENIZED);
                Field Title = new Field("Title", mydocs[i].Title, Field.Store.YES, Field.Index.TOKENIZED);
                Field HtmPath = new Field("HtmlPath", mydocs[i].HtmPath, Field.Store.YES, Field.Index.NO);
                Field KeyWords = new Field("KeyWords", mydocs[i].KeyWords, Field.Store.YES, Field.Index.TOKENIZED);
                Field Content = new Field("Content", GetHTML(mydocs[i].HtmPath), Field.Store.YES, Field.Index.TOKENIZED);

                //为字段设置权重，默认为1
                Title.SetBoost(1.5f);

                KeyWords.SetBoost(1.2f);                
                doc.Add(DomID);
                doc.Add(Title);
                doc.Add(HtmPath);
                doc.Add(KeyWords);
                doc.Add(SynTime);
                doc.Add(Content);

                writer.AddDocument(doc);
            }
        }

        ///<summary> 
        ///对多个文档建立索引 
        ///</summary> 
        public void Index(List<A>doms)
        {
            string INDEX_STORE_PATH = Myindex;

            //使用eaglet的KTDictSeg的分词器
            Analyzer analyzer = new KTDictSegAnalyzer();

            //lucene.net 默认分词器
            //Analyzer analyzer = new StandardAnalyzer();

            FSDirectory fsDir;
            if (System.IO.Directory.Exists(INDEX_STORE_PATH))
            {
                 fsDir= FSDirectory.GetDirectory(INDEX_STORE_PATH, false);
            }
            else
            {
                 fsDir = FSDirectory.GetDirectory(INDEX_STORE_PATH, true);
            }

            IndexWriter writer = new IndexWriter(fsDir, analyzer, true);

            AddDocument(writer, doms);
            writer.Optimize();
            writer.Close();

        }

        ///<summary> 
        ///词进行分词 
        ///</summary> 
        ///<param name="keyWords">要搜索的词</param> 
        ///<param name="ktTokenizer">分词对象</param> 
        ///<returns>分词后的结果</returns> 
        public string GetKeyWordSplitBySpace(string keyWords, KTDictSegTokenizer ktTokenizer)
        {
            StringBuilder builder = new StringBuilder();
            List<FTAlgorithm.T_WordInfo> words = ktTokenizer.SegmentToWordInfos(keyWords);

            foreach (FTAlgorithm.T_WordInfo word in words)
            {
                if (word == null)
                {
                    continue;
                }

                KeyWord = KeyWord + word + ",";
                builder.AppendFormat("{0}^{1}", word.Word, (int)Math.Pow(3, word.Rank));
            }
            KeyWord = KeyWord.Substring(0, KeyWord.Length - 1);
            return builder.ToString().Trim();
        }

        ///<summary> 
        ///检索信息 
        ///</summary> 
        ///<param name="keyWord">检索关键字</param> 
        ///<param name="pageNumber">当前第几条</param> 
        ///<param name="pageSize">每页显示的条数</param> 
        ///<returns></returns> 
        public List<TheIndex> Search(string keyWord, int pageNumber, int pageSize)
        {
            string word = GetKeyWordSplitBySpace(keyWord, new KTDictSegTokenizer());

            IndexSearcher search = new IndexSearcher(Myindex);
            StringBuilder builder = new StringBuilder();
            KTDictSegAnalyzer analyzer = new KTDictSegAnalyzer(true);
            
            //多字段搜索字段 
            MultiFieldQueryParser parser = new MultiFieldQueryParser(new string[] { "Title", "Content","KeyWords" }, analyzer);
            
            //分词 
            Query query = parser.Parse(word);
            Hits hits = search.Search(query);
            Count = hits.Length();

            int num = 0;//记录每页最后一条 
            if (Count < pageNumber + pageSize)
            {
                num = Count;
            }

            else
            {
                num = pageSize + pageNumber;
            }

            DateTime begin = DateTime.Now;

            List<TheIndex> theindexs = new List<TheIndex>();
            for (int i = 0; i < num; i++)
            {
                //使用KTDictSeg.HighLight高亮组件为结果添加高亮效果
                KTDictSeg.HighLight.SimpleHTMLFormatter simpleHTMLFormatter =
                    new KTDictSeg.HighLight.SimpleHTMLFormatter("<font color=\"red\">", "</font>");

                KTDictSeg.HighLight.Highlighter highlighter =
                        new KTDictSeg.HighLight.Highlighter(simpleHTMLFormatter,
                        new Lucene.Net.Analysis.KTDictSeg.KTDictSegTokenizer());

                highlighter.FragmentSize = 300;

                TheIndex theindex = new TheIndex();
                theindex.DomID =Convert.ToInt32(hits.Doc(i).Get("DomID"));
                theindex.Title= hits.Doc(i).Get("Title");

                string title = highlighter.GetBestFragment(keyWord,theindex.Title);
                if (!string.IsNullOrEmpty(title))
                {
                    theindex.Title = title;
                }

                theindex.HtmPath = hits.Doc(i).Get("HtmlPath");
                theindex.Content =hits.Doc(i).Get("Content");

                string content=highlighter.GetBestFragment(keyWord,theindex.Content);
                if (!string.IsNullOrEmpty(content))
                {
                    theindex.Content = content;
                }

                theindex.Content = theindex.Content.Substring(0, theindex.Content.Length>300?300:theindex.Content.Length);
                theindex.SynTime =Convert.ToDateTime(hits.Doc(i).Get("SynTime")).ToString("yyyy年MM月dd日");
                theindex.KeyWords = hits.Doc(i).Get("KeyWords");

                string keywords = highlighter.GetBestFragment(keyWord,theindex.KeyWords);
                if (!string.IsNullOrEmpty(keywords))
                {
                    theindex.KeyWords = keywords;
                }

                theindexs.Add(theindex);
            }

            DateTime end = DateTime.Now;
            double ts = (end - begin).TotalMilliseconds;
            Time = ts / 1000.000;
            search.Close();//关闭检索器 
            return theindexs;
        }

 
        //定义方法，用来解析HTML，用以为HTML文档添加索引
        public static string GetHTML(string Path)
        {
            System.Text.Encoding encoding = System.Text.Encoding.GetEncoding("gb2312");
            StreamReader sr = new StreamReader(HttpContext.Current.Server.MapPath("~/") + Path, encoding);

            string str = "";
 
            while (sr.Peek() != -1)
            {
                char[] buffer = new char[4096];
                int bufferFillSize = sr.ReadBlock(buffer, 0, 4096);
                str = str + new string(buffer);
            }
            sr.Close();
            
            string strResult = findUsedFromHtml(str);
            return strResult;
        }

        ///<summary>
        ///获取Html的body部分并进行格式化
        ///</summary>
        ///<param name="strHtml"></param>
        ///<returns></returns>
        private static string findUsedFromHtml(string strHtml)
        {
            string strBody;

            int bodyStart = strHtml.IndexOf("<body");
            int bodyEnd = strHtml.IndexOf("</body>");

            //Body部分
            strBody =StripHTML(strHtml.Substring(bodyStart, bodyEnd - bodyStart + 7));

            return strBody;
        }

        ///<Header>去除HTML的tag
        ///</Header>
        ///<param name="HTML">源</param>
        ///<returns>结果</returns>
        public static string StripHTML(string HTML)
        {
            string[] Regexs =
            {
                @"<body[^>]*?>",
                @"<script[^>]*?>.*?</script>",
                @"<(\/\s*)?!?((\w+:)?\w+)(\w+(\s*=?\s*(([""'])(\\[""'tbnr]|[^\7])*?\7|\w+)|.{0})|\s)*?(\/\s*)?>",
                @"([\r\n])[\s]+",
                @"&(quot|#34);",
                @"&(amp|#38);",
                @"&(lt|#60);",
                @"&(gt|#62);",
                @"&(nbsp|#160);",
                @"&(iexcl|#161);",
                @"&(cent|#162);",
                @"&(pound|#163);",
                @"&(copy|#169);",
                @"&#(\d+);",
                @"-->",
                @"<!--.*\n"
            };

            string[] Replaces =
            {
                "",
                "",
                "",
                "",
                "\"",
                "&",
                "<",
                ">",
                " ",
                "\xa1", //chr(161),
                "\xa2", //chr(162),
                "\xa3", //chr(163),
                "\xa9", //chr(169),
                "",
                "\r\n",
                ""
            };
            
            string s = HTML;
            for (int i = 0; i < Regexs.Length; i++)
            {
                s = new Regex(Regexs[i], RegexOptions.Multiline | RegexOptions.IgnoreCase).Replace(s, Replaces[i]);
            }
            s.Replace("<", "");
            s.Replace(">", "");
            return s;
        }
    }
}

新建完这个类，我们就可以为文档添加索引了，并搜索了，呵呵。下次再写写再Java下的实现。

呵呵，博客新手+初级程序员，所以写出的文章难免干巴巴并且涉及比较少，请不要见笑。最近实习也比较忙，初稿就这样，呵呵，有什么问题，请发到我的邮箱：three_zone@163.com