package engine

import (
    "bufio"
    "log"
    "os"
)

type StopTokens struct {
    stopTokens map[string]bool
}

// 从stopTokenFile中读入停用词，一个词一行
// 文档索引建立时会跳过这些停用词
func (st *StopTokens) Init(stopTokenFile string) {
    st.stopTokens = make(map[string]bool)
    if stopTokenFile == "" {
        return
    }

    file, err := os.Open(stopTokenFile)
    if err != nil {
        log.Fatal(err)
    }
    defer file.Close()

    scanner := bufio.NewScanner(file)
    for scanner.Scan() {
        text := scanner.Text()
        if text != "" {
            st.stopTokens[text] = true
        }
    }

}

func (st *StopTokens) IsStopToken(token string) bool {
    _, found := st.stopTokens[token]
    return found
}