全文搜索

全文搜索

一、建立es表结构、创建索引

package models

import (
	"context"
	"gvb_server/global"
)

type FullTextModel struct {
	ID    string `json:"id" structs:"id"`
	Key   string `json:"key"`
	Body  string `json:"body" structs:"body"`
	Title string `json:"title" structs:"title"`
	Slug  string `json:"slug" structs:"slug"`
}

func (FullTextModel) Index() string {
	return "full_text_index"
}

func (FullTextModel) Mapping() string {
	return `
	{
		"settings": {
			"index": {
				"max_result_window": 100000
			}
		},
		"mappings": {
			"properties": {
				"key": {
					"type": "keyword"
				},
				"body": {
					"type": "text"
				},
				"title": {
					"type": "text"
				},
				"slug": {
					"type": "keyword"
				}
			}
		}
	}
	`
}

func (a FullTextModel) IndexExits() bool {
	exists, err := global.ESClient.IndexExists(a.Index()).Do(context.Background())
	if err != nil {
		global.Log.Error(err.Error())
		return exists
	}
	return exists
}

func (a FullTextModel) CreateIndex() error {
	if a.IndexExits() {
		a.RemoveIndex()
	}

	createIndex, err := global.ESClient.CreateIndex(a.Index()).BodyString(a.Mapping()).Do(context.Background())

	if err != nil {
		global.Log.Errorf("创建索引失败 %s", err.Error())
		return err
	} else {
		if !createIndex.Acknowledged {
			global.Log.Error("创建失败")
			return nil
		} else {
			global.Log.Infof("索引 %s 创建成功", a.Index())
			return nil
		}
	}
}

func (a FullTextModel) RemoveIndex() error {
	global.Log.Info("索引存在，正在删除...")
	deleteIndex, err := global.ESClient.DeleteIndex(a.Index()).Do(context.Background())

	if err != nil {
		global.Log.Errorf("删除索引失败 %s", err.Error())
		return err
	} else {
		if !deleteIndex.Acknowledged {
			global.Log.Error("删除索引失败")
			return nil
		} else {
			global.Log.Info("删除索引成功")
			return nil
		}
	}
}

二、获取相关文章的全文索引doc列表

我们的搜索跳转格式为

http://localhost:8888/article/:id#title

type SearchData struct {
	Key   string `json:"key"`	// 查询文章相关的索引
	Body  string `json:"body"`	// 标题下的内容
	Title string `json:"title"`	// 标题
	Slug  string `json:"slug"`	// 跳转的位置
}

func getSearchIndexByContent(id, title, content string) (searchDataList []SearchData) {
	var headerList []string
	var bodyList []string

	var dataList = strings.Split(content, "\n")
	var isCode = false
	var body string

	headerList = append(headerList, title)

	for _, d := range dataList {
		if strings.HasPrefix(d, "```") {
			isCode = !isCode
		}

		if strings.HasPrefix(d, "#") && !isCode {
			header := getHeader(d)
			headerList = append(headerList, header)

			bodyList = append(bodyList, body)
			body = ""
		} else {
			d = getBody(d)
			body += d
		}
	}
	bodyList = append(bodyList, body)

	for i, v := range headerList {
		searchDataList = append(searchDataList, SearchData{
			Key:   id,
			Body:  bodyList[i],
			Title: v,
			Slug:  id + getSlug(v),
		})
	}

	return
}

func getHeader(header string) string {
	header = strings.ReplaceAll(header, "#", "")
	header = strings.TrimSpace(header)
	return header
}

func getBody(body string) string {
	unsafe := blackfriday.MarkdownCommon([]byte(body))
	doc, _ := goquery.NewDocumentFromReader(strings.NewReader(string(unsafe)))
	return doc.Text()
}

func getSlug(slug string) string {
	return "#" + slug
}

三、将获取的索引doc列表同步到es中（在添加文章成功的时候调用）

func AsyncFullSearch(id, title, content string) {
	indexList := getSearchIndexByContent(id, title, content)

	bulkService := global.ESClient.Bulk().Index(models.FullTextModel{}.Index())
	for _, index := range indexList {
		req := elastic.NewBulkCreateRequest().Doc(index)
		bulkService.Add(req)
	}
	result, err := bulkService.Do(context.Background())
	if err != nil {
		global.Log.Error(err.Error())
	}

	global.Log.Infof("%s: 共添加 %d 条文章索引", title, len(result.Succeeded()))
}

通过doc内容批量添加的代码

bulkService := global.ESClient.
	Bulk().
	Index(models.FullTextModel{}.Index())
for _, index := range indexList {
    req := elastic.NewBulkCreateRequest().Doc(index)
    bulkService.Add(req)
}
result, err := bulkService.Do(context.Background())

四、删除全文搜索索引doc列表（在删除文章的时候调用）

func DeleteFullSearch(id string) {
	query := elastic.NewTermQuery("key", id)
	res, err := global.ESClient.
		DeleteByQuery().
		Index(models.FullTextModel{}.Index()).
		Query(query).
		Size(10000).
		Do(context.Background())
	if err != nil {
		global.Log.Error(err.Error())
		return
	}
	global.Log.Infof("共删除 %d 条文章索引", res.Deleted)
}

通过keyword字段匹配文章id批量删除doc的代码

query := elastic.NewTermQuery("key", id)
res, err := global.ESClient.
    DeleteByQuery().
    Index(models.FullTextModel{}.Index()).
    Query(query).
    Size(10000).
    Do(context.Background())

通过id删除文章的es代码

bulkService := global.ESClient.
	Bulk().
	Index(models.ArticleModel{}.Index()).
	Refresh("true")
for _, id := range cr.IDList {
    req := elastic.NewBulkDeleteRequest().Id(id)
    bulkService.Add(req)
    go es_ser.DeleteFullSearch(id)
}
result, err := bulkService.Do(context.Background())	// 请不要忘记执行删除的操作

---

在我不知道DeleteByQuery()这个函数的时候，我的DeleteFullSearch()是这样写的，hhh，多此一举了

func DeleteFullSearch(id string) {
	var idList []string
	query := elastic.NewTermQuery("key", id)
	res, err := global.ESClient.
		Search(models.FullTextModel{}.Index()).
		Query(query).
		Size(10000).
		Do(context.Background())
	if err != nil {
		global.Log.Error(err.Error())
		return
	}

	for _, hit := range res.Hits.Hits {
		idList = append(idList, hit.Id)
	}
	bulkService := global.ESClient.Bulk().Index(models.FullTextModel{}.Index())
	for _, id := range idList {
		req := elastic.NewBulkDeleteRequest().Id(id)
		bulkService.Add(req)
	}
	result, err := bulkService.Do(context.Background())
	if err != nil {
		global.Log.Error(err.Error())
		return
	}

	global.Log.Infof("共删除 %d 条文章索引", len(result.Succeeded()))
}

五、更新文章索引doc列表（在文章标题或者内容更新的时候调用）

// 主要是修改api/article_api/article_update中更新部分的代码

	// 获取更新之前的文章
	article.GetDataByID(cr.ID)

	_, err = global.ESClient.Update().
		Index(models.ArticleModel{}.Index()).
		Id(cr.ID).
		Doc(maps).
		Refresh("true").
		Do(context.Background())
	if err != nil {
		res.FailWithMessage("更新文章失败", ctx)
		global.Log.Error(err)
		return
	}

	// 获取更新之后的文章
	newArticle, _ := es_ser.ComDetailID(cr.ID)

	// 检查标题或者内容是否更改
	if article.Title != newArticle.Title || article.Content != newArticle.Content {
		go es_ser.DeleteFullSearch(cr.ID)
		go es_ser.AsyncFullSearch(cr.ID, newArticle.Title, newArticle.Content)
	}

六、编写返回指定文章的全文搜索索引列表的view

func (ArticleApi) FullTextSearchView(ctx *gin.Context) {
	var cr models.PageInfo
	err := ctx.ShouldBindQuery(&cr)
	if err != nil {
		res.FailWithCode(res.ArgumentError, ctx)
		return
	}

	boolQuery := elastic.NewBoolQuery()
	if cr.Key != "" {
		boolQuery.Must(elastic.NewMultiMatchQuery(cr.Key, "title", "body"))
	}

	result, err := global.ESClient.
		Search(models.FullTextModel{}.Index()).
		Query(boolQuery).
		Highlight(elastic.NewHighlight().Field("body")).
		Size(10000).
		Do(context.Background())
	if err != nil {
		global.Log.Error(err.Error())
		res.FailWithMessage("查询文章索引失败", ctx)
		return
	}

	var list = make([]models.FullTextModel, 0)
	count := result.Hits.TotalHits.Value

	for _, hit := range result.Hits.Hits {
		var model models.FullTextModel
		err = json.Unmarshal(hit.Source, &model)
		if err != nil {
			global.Log.Error(err.Error())
			continue
		}

		// 搜索到的标题高亮
		body, ok := hit.Highlight["body"]
		if ok {
			model.Title = body[0]
		}

		list = append(list, model)
	}
	res.OKWithList(list, count, ctx)
}