banyand/measure/measure.go

// Licensed to Apache Software Foundation (ASF) under one or more contributor
// license agreements. See the NOTICE file distributed with
// this work for additional information regarding copyright
// ownership. Apache Software Foundation (ASF) licenses this file to you under
// the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

// Package measure implements a time-series-based storage which is consists of a sequence of data points.
// Each data point contains tags and fields. They arrive in a fixed interval. A data point could be updated
// by one with the identical entity(series_id) and timestamp.
package measure

import (
	"context"
	"math"
	"time"

	commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
	databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
	"github.com/apache/skywalking-banyandb/banyand/queue"
	"github.com/apache/skywalking-banyandb/banyand/tsdb"
	"github.com/apache/skywalking-banyandb/banyand/tsdb/index"
	"github.com/apache/skywalking-banyandb/pkg/logger"
	"github.com/apache/skywalking-banyandb/pkg/query/logical"
	resourceSchema "github.com/apache/skywalking-banyandb/pkg/schema"
	"github.com/apache/skywalking-banyandb/pkg/timestamp"
)

const (
	plainChunkSize = 1 << 20
	intChunkNum    = 120
	intChunkSize   = 4 * 1024
)

type measure struct {
	databaseSupplier       tsdb.Supplier
	l                      *logger.Logger
	schema                 *databasev1.Measure
	indexWriter            *index.Writer
	processorManager       *topNProcessorManager
	name                   string
	group                  string
	indexRules             []*databasev1.IndexRule
	topNAggregations       []*databasev1.TopNAggregation
	maxObservedModRevision int64
	interval               time.Duration
	shardNum               uint32
}

func (s *measure) startSteamingManager(pipeline queue.Queue) error {
	if len(s.topNAggregations) == 0 {
		return nil
	}
	tagMapSpec := logical.TagSpecMap{}
	tagMapSpec.RegisterTagFamilies(s.schema.GetTagFamilies())

	s.processorManager = &topNProcessorManager{
		l:            s.l,
		pipeline:     pipeline,
		m:            s,
		s:            tagMapSpec,
		topNSchemas:  s.topNAggregations,
		processorMap: make(map[*commonv1.Metadata][]*topNStreamingProcessor),
	}

	return s.processorManager.start()
}

func (s *measure) GetSchema() *databasev1.Measure {
	return s.schema
}

func (s *measure) GetMetadata() *commonv1.Metadata {
	return s.schema.Metadata
}

func (s *measure) GetIndexRules() []*databasev1.IndexRule {
	return s.indexRules
}

func (s *measure) GetTopN() []*databasev1.TopNAggregation {
	return s.topNAggregations
}

func (s *measure) MaxObservedModRevision() int64 {
	return s.maxObservedModRevision
}

func (s *measure) Close() error {
	if s.processorManager == nil {
		return nil
	}
	return s.processorManager.Close()
}

func (s *measure) parseSpec() (err error) {
	s.name, s.group = s.schema.GetMetadata().GetName(), s.schema.GetMetadata().GetGroup()
	s.maxObservedModRevision = int64(math.Max(float64(resourceSchema.ParseMaxModRevision(s.indexRules)), float64(resourceSchema.ParseMaxModRevision(s.topNAggregations))))
	if s.schema.Interval != "" {
		s.interval, err = timestamp.ParseDuration(s.schema.Interval)
	}
	return err
}

type measureSpec struct {
	schema           *databasev1.Measure
	indexRules       []*databasev1.IndexRule
	topNAggregations []*databasev1.TopNAggregation
}

func openMeasure(shardNum uint32, db tsdb.Supplier, spec measureSpec, l *logger.Logger, pipeline queue.Queue,
) (*measure, error) {
	m := &measure{
		shardNum:         shardNum,
		schema:           spec.schema,
		indexRules:       spec.indexRules,
		topNAggregations: spec.topNAggregations,
		l:                l,
	}
	if err := m.parseSpec(); err != nil {
		return nil, err
	}
	ctx := context.WithValue(context.Background(), logger.ContextKey, l)

	m.databaseSupplier = db
	m.indexWriter = index.NewWriter(ctx, index.WriterOptions{
		DB:         db,
		ShardNum:   shardNum,
		Families:   spec.schema.TagFamilies,
		IndexRules: spec.indexRules,
	})

	if startErr := m.startSteamingManager(pipeline); startErr != nil {
		l.Err(startErr).Str("measure", spec.schema.GetMetadata().GetName()).
			Msg("fail to start streaming manager")
	}

	return m, nil
}