wukong引擎源码分析之搜索——docid有序的数组里二分归并求交集，如果用跳表的话，在插入索引时会更快

// 查找满足搜索条件的文档，此函数线程安全
func (engine *Engine) Search(request types.SearchRequest) (output types.SearchResponse) {
if !engine.initialized {
log.Fatal("必须先初始化引擎")
} var rankOptions types.RankOptions
if request.RankOptions == nil {
rankOptions = *engine.initOptions.DefaultRankOptions
} else {
rankOptions = *request.RankOptions
}
if rankOptions.ScoringCriteria == nil {
rankOptions.ScoringCriteria = engine.initOptions.DefaultRankOptions.ScoringCriteria
} // 收集关键词
tokens := []string{}
if request.Text != "" {
querySegments := engine.segmenter.Segment([]byte(request.Text))
for _, s := range querySegments {
token := s.Token().Text()
if !engine.stopTokens.IsStopToken(token) {
tokens = append(tokens, s.Token().Text())
}
}
} else {
for _, t := range request.Tokens {
tokens = append(tokens, t)
}
} // 建立排序器返回的通信通道
rankerReturnChannel := make(
chan rankerReturnRequest, engine.initOptions.NumShards) // 生成查找请求
lookupRequest := indexerLookupRequest{
countDocsOnly: request.CountDocsOnly,
tokens: tokens,
labels: request.Labels,
docIds: request.DocIds,
options: rankOptions,
rankerReturnChannel: rankerReturnChannel,
orderless: request.Orderless,
} // 向索引器发送查找请求
for shard := ; shard < engine.initOptions.NumShards; shard++ {
engine.indexerLookupChannels[shard] <- lookupRequest
} // 从通信通道读取排序器的输出
numDocs :=
rankOutput := types.ScoredDocuments{}
timeout := request.Timeout
isTimeout := false
if timeout <= {
// 不设置超时
for shard := ; shard < engine.initOptions.NumShards; shard++ {
rankerOutput := <-rankerReturnChannel
if !request.CountDocsOnly {
for _, doc := range rankerOutput.docs {
rankOutput = append(rankOutput, doc)
}
}
numDocs += rankerOutput.numDocs
}
} else {
// 设置超时
deadline := time.Now().Add(time.Millisecond * time.Duration(request.Timeout))
for shard := ; shard < engine.initOptions.NumShards; shard++ {
select {
case rankerOutput := <-rankerReturnChannel:
if !request.CountDocsOnly {
for _, doc := range rankerOutput.docs {
rankOutput = append(rankOutput, doc)
}
}
numDocs += rankerOutput.numDocs
case <-time.After(deadline.Sub(time.Now())):
isTimeout = true
break
}
}
} // 再排序
if !request.CountDocsOnly && !request.Orderless {
if rankOptions.ReverseOrder {
sort.Sort(sort.Reverse(rankOutput))
} else {
sort.Sort(rankOutput)
}
} // 准备输出
output.Tokens = tokens
// 仅当CountDocsOnly为false时才充填output.Docs
if !request.CountDocsOnly {
if request.Orderless {
// 无序状态无需对Offset截断
output.Docs = rankOutput
} else {
var start, end int
if rankOptions.MaxOutputs == {
start = utils.MinInt(rankOptions.OutputOffset, len(rankOutput))
end = len(rankOutput)
} else {
start = utils.MinInt(rankOptions.OutputOffset, len(rankOutput))
end = utils.MinInt(start+rankOptions.MaxOutputs, len(rankOutput))
}
output.Docs = rankOutput[start:end]
}
}
output.NumDocs = numDocs
output.Timeout = isTimeout
return
}

func (engine *Engine) indexerLookupWorker(shard int) {
for {
request := <-engine.indexerLookupChannels[shard] // 关键 var docs []types.IndexedDocument
var numDocs int
if request.docIds == nil {
docs, numDocs = engine.indexers[shard].Lookup(request.tokens, request.labels, nil, request.countDocsOnly)
} else {
docs, numDocs = engine.indexers[shard].Lookup(request.tokens, request.labels, request.docIds, request.countDocsOnly)
} if request.countDocsOnly {
request.rankerReturnChannel <- rankerReturnRequest{numDocs: numDocs}
continue
} if len(docs) == {
request.rankerReturnChannel <- rankerReturnRequest{}
continue
} if request.orderless {
var outputDocs []types.ScoredDocument
for _, d := range docs {
outputDocs = append(outputDocs, types.ScoredDocument{
DocId: d.DocId,
TokenSnippetLocations: d.TokenSnippetLocations,
TokenLocations: d.TokenLocations})
} request.rankerReturnChannel <- rankerReturnRequest{
docs: outputDocs,
numDocs: len(outputDocs),
}
continue
} rankerRequest := rankerRankRequest{
countDocsOnly: request.countDocsOnly,
docs: docs,
options: request.options,
rankerReturnChannel: request.rankerReturnChannel,
}
engine.rankerRankChannels[shard] <- rankerRequest
}
}

// 查找包含全部搜索键(AND操作)的文档
// 当docIds不为nil时仅从docIds指定的文档中查找
func (indexer *Indexer) Lookup(
tokens []string, labels []string, docIds map[uint64]bool, countDocsOnly bool) (docs []types.IndexedDocument, numDocs int) {
if indexer.initialized == false {
log.Fatal("索引器尚未初始化")
} indexer.DocInfosShard.RLock()
defer indexer.DocInfosShard.RUnlock() if indexer.DocInfosShard.NumDocuments == {
return
}
numDocs = // 合并关键词和标签为搜索键
keywords := make([]string, len(tokens)+len(labels))
copy(keywords, tokens)
copy(keywords[len(tokens):], labels) indexer.InvertedIndexShard.RLock() table := make([]*types.KeywordIndices, len(keywords))
for i, keyword := range keywords {
indices, found := indexer.InvertedIndexShard.InvertedIndex[keyword]
if !found {
// 当反向索引表中无此搜索键时直接返回
indexer.InvertedIndexShard.RUnlock()
return
} else {
// 否则加入反向表中
table[i] = indices
}
// 当没有找到时直接返回
if len(table) == {
indexer.InvertedIndexShard.RUnlock()
return
} // 归并查找各个搜索键出现文档的交集
// 从后向前查保证先输出DocId较大文档
indexPointers := make([]int, len(table))
for iTable := ; iTable < len(table); iTable++ {
indexPointers[iTable] = indexer.getIndexLength(table[iTable]) -
}
// 平均文本关键词长度，用于计算BM25
avgDocLength := indexer.InvertedIndexShard.TotalTokenLength / float32(indexer.DocInfosShard.NumDocuments)
indexer.InvertedIndexShard.RUnlock() for ; indexPointers[] >= ; indexPointers[]-- {
// 以第一个搜索键出现的文档作为基准，并遍历其他搜索键搜索同一文档
baseDocId := indexer.getDocId(table[], indexPointers[]) // 全局范围查找目标文档是否存在
if _, ok := indexer.DocInfosShard.DocInfos[baseDocId]; !ok {
// if !IsDocExist(baseDocId) {
// 文档信息中不存在反向索引文档时，跳过
// 该情况由不对称删除操作所造成
continue
} if docIds != nil {
_, found := docIds[baseDocId]
if !found {
continue
}
}
iTable :=
found := true
for ; iTable < len(table); iTable++ {
// 二分法比简单的顺序归并效率高，也有更高效率的算法，
// 但顺序归并也许是更好的选择，考虑到将来需要用链表重新实现
// 以避免反向表添加新文档时的写锁。
// TODO: 进一步研究不同求交集算法的速度和可扩展性。
position, foundBaseDocId := indexer.searchIndex(table[iTable],
, indexPointers[iTable], baseDocId)
if foundBaseDocId {
indexPointers[iTable] = position
} else {
if position == {
// 该搜索键中所有的文档ID都比baseDocId大，因此已经没有
// 继续查找的必要。
return
} else {
// 继续下一indexPointers[0]的查找
indexPointers[iTable] = position -
found = false
break
}
}
} if found {
indexedDoc := types.IndexedDocument{} // 当为LocationsIndex时计算关键词紧邻距离
if indexer.initOptions.IndexType == types.LocationsIndex {
// 计算有多少关键词是带有距离信息的
numTokensWithLocations :=
for i, t := range table[:len(tokens)] {
if len(t.Locations[indexPointers[i]]) > {
numTokensWithLocations++
}
}
if numTokensWithLocations != len(tokens) {
if !countDocsOnly {
docs = append(docs, types.IndexedDocument{
DocId: baseDocId,
})
}
numDocs++
break
}
// 计算搜索键在文档中的紧邻距离
tokenProximity, tokenLocations := computeTokenProximity(table[:len(tokens)], indexPointers, tokens)
indexedDoc.TokenProximity = int32(tokenProximity)
indexedDoc.TokenSnippetLocations = tokenLocations // 添加TokenLocations
indexedDoc.TokenLocations = make([][]int, len(tokens))
for i, t := range table[:len(tokens)] {
indexedDoc.TokenLocations[i] = t.Locations[indexPointers[i]]
}
} // 当为LocationsIndex或者FrequenciesIndex时计算BM25
if indexer.initOptions.IndexType == types.LocationsIndex ||
indexer.initOptions.IndexType == types.FrequenciesIndex {
bm25 := float32()
d := indexer.DocInfosShard.DocInfos[baseDocId].TokenLengths
for i, t := range table[:len(tokens)] {
var frequency float32
if indexer.initOptions.IndexType == types.LocationsIndex {
frequency = float32(len(t.Locations[indexPointers[i]]))
} else {
frequency = t.Frequencies[indexPointers[i]]
} // 计算BM25
if len(t.DocIds) > && frequency > && indexer.initOptions.BM25Parameters != nil && avgDocLength != {
// 带平滑的idf
idf := float32(math.Log2(float64(indexer.DocInfosShard.NumDocuments)/float64(len(t.DocIds)) + ))
k1 := indexer.initOptions.BM25Parameters.K1
b := indexer.initOptions.BM25Parameters.B
bm25 += idf * frequency * (k1 + ) / (frequency + k1*(-b+b*d/avgDocLength))
}
}
indexedDoc.BM25 = float32(bm25)
} indexedDoc.DocId = baseDocId
if !countDocsOnly {
docs = append(docs, indexedDoc)
}
numDocs++
}
}
return
}