in indexbuilder.go [323:405]
func (b *IndexBuilder) Add(doc Document) error {
hasher := crc64.New(crc64.MakeTable(crc64.ISO))
if idx := bytes.IndexByte(doc.Content, 0); idx >= 0 {
doc.SkipReason = fmt.Sprintf("binary content at byte offset %d", idx)
doc.Language = "binary"
}
if doc.SkipReason != "" {
doc.Content = []byte(notIndexedMarker + doc.SkipReason)
doc.Symbols = nil
if doc.Language == "" {
doc.Language = "skipped"
}
}
sort.Sort(docSectionSlice(doc.Symbols))
var last DocumentSection
for i, s := range doc.Symbols {
if i > 0 {
if last.End > s.Start {
return fmt.Errorf("sections overlap")
}
}
last = s
}
if last.End > uint32(len(doc.Content)) {
return fmt.Errorf("section goes past end of content")
}
if doc.SubRepositoryPath != "" {
rel, err := filepath.Rel(doc.SubRepositoryPath, doc.Name)
if err != nil || rel == doc.Name {
return fmt.Errorf("path %q must start subrepo path %q", doc.Name, doc.SubRepositoryPath)
}
}
docStr, runeSecs, err := b.contentPostings.newSearchableString(doc.Content, doc.Symbols)
if err != nil {
return err
}
nameStr, _, err := b.namePostings.newSearchableString([]byte(doc.Name), nil)
if err != nil {
return err
}
subRepoIdx, ok := b.subRepoIndices[doc.SubRepositoryPath]
if !ok {
return fmt.Errorf("unknown subrepo path %q", doc.SubRepositoryPath)
}
var mask uint64
for _, br := range doc.Branches {
m := b.branchMask(br)
if m == 0 {
return fmt.Errorf("no branch found for %s", br)
}
mask |= m
}
b.subRepos = append(b.subRepos, subRepoIdx)
hasher.Write(doc.Content)
b.contentStrings = append(b.contentStrings, docStr)
b.runeDocSections = append(b.runeDocSections, runeSecs...)
b.nameStrings = append(b.nameStrings, nameStr)
b.docSections = append(b.docSections, doc.Symbols)
b.branchMasks = append(b.branchMasks, mask)
b.checksums = append(b.checksums, hasher.Sum(nil)...)
langCode, ok := b.languageMap[doc.Language]
if !ok {
if len(b.languageMap) >= 255 {
return fmt.Errorf("too many languages")
}
langCode = byte(len(b.languageMap))
b.languageMap[doc.Language] = langCode
}
b.languages = append(b.languages, langCode)
return nil
}