in storage/indexer.go [155:203]
func (i *Indexer) updateIndex(ctx context.Context) error {
span, ctx := apm.StartSpan(ctx, "UpdateIndex", "app")
defer span.End()
i.logger.Debug("Update indices")
start := time.Now()
defer func() {
metrics.StorageIndexerUpdateIndexDurationSeconds.Observe(time.Since(start).Seconds())
}()
bucketName, rootStoragePath, err := extractBucketNameFromURL(i.options.PackageStorageBucketInternal)
if err != nil {
metrics.StorageIndexerUpdateIndexErrorsTotal.Inc()
return fmt.Errorf("can't extract bucket name from URL (url: %s): %w", i.options.PackageStorageBucketInternal, err)
}
storageCursor, err := loadCursor(ctx, i.logger, i.storageClient, bucketName, rootStoragePath)
if err != nil {
metrics.StorageIndexerUpdateIndexErrorsTotal.Inc()
return fmt.Errorf("can't load latest cursor: %w", err)
}
if storageCursor.Current == i.cursor {
i.logger.Info("cursor is up-to-date", zap.String("cursor.current", i.cursor))
return nil
}
i.logger.Info("cursor will be updated", zap.String("cursor.current", i.cursor), zap.String("cursor.next", storageCursor.Current))
anIndex, err := loadSearchIndexAll(ctx, i.logger, i.storageClient, bucketName, rootStoragePath, *storageCursor)
if err != nil {
metrics.StorageIndexerUpdateIndexErrorsTotal.Inc()
return fmt.Errorf("can't load the search-index-all index content: %w", err)
}
if anIndex == nil {
i.logger.Info("Downloaded new search-index-all index. No packages found.")
return nil
}
i.logger.Info("Downloaded new search-index-all index", zap.String("index.packages.size", fmt.Sprintf("%d", len(*anIndex))))
i.transformSearchIndexAllToPackages(anIndex)
i.m.Lock()
defer i.m.Unlock()
i.cursor = storageCursor.Current
i.packageList = *anIndex
metrics.StorageIndexerUpdateIndexSuccessTotal.Inc()
metrics.NumberIndexedPackages.Set(float64(len(i.packageList)))
return nil
}