api/proto/banyandb/database/v1/schema.proto

// Licensed to Apache Software Foundation (ASF) under one or more contributor
// license agreements. See the NOTICE file distributed with
// this work for additional information regarding copyright
// ownership. Apache Software Foundation (ASF) licenses this file to you under
// the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

syntax = "proto3";

package banyandb.database.v1;

import "banyandb/common/v1/common.proto";
import "banyandb/model/v1/common.proto";
import "banyandb/model/v1/query.proto";
import "google/protobuf/timestamp.proto";
import "validate/validate.proto";

option go_package = "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1";
option java_package = "org.apache.skywalking.banyandb.database.v1";

enum TagType {
  TAG_TYPE_UNSPECIFIED = 0;
  TAG_TYPE_STRING = 1;
  TAG_TYPE_INT = 2;
  TAG_TYPE_STRING_ARRAY = 3;
  TAG_TYPE_INT_ARRAY = 4;
  TAG_TYPE_DATA_BINARY = 5;
}

message TagFamilySpec {
  string name = 1 [(validate.rules).string.min_len = 1];
  // tags defines accepted tags
  repeated TagSpec tags = 2 [(validate.rules).repeated.min_items = 1];
}

message TagSpec {
  string name = 1 [(validate.rules).string.min_len = 1];
  TagType type = 2 [(validate.rules).enum.defined_only = true];
  // indexed_only indicates whether the tag is stored
  // True: It's indexed only, but not stored
  // False: it's stored and indexed
  bool indexed_only = 3;
}

// Stream intends to store streaming data, for example, traces or logs
message Stream {
  // metadata is the identity of a trace series
  common.v1.Metadata metadata = 1 [(validate.rules).message.required = true];
  // tag_families
  repeated TagFamilySpec tag_families = 2 [(validate.rules).repeated.min_items = 1];
  // entity indicates how to generate a series and shard a stream
  Entity entity = 3 [(validate.rules).message.required = true];
  // updated_at indicates when the stream is updated
  google.protobuf.Timestamp updated_at = 4;
}

message Entity {
  repeated string tag_names = 1 [(validate.rules).repeated.min_items = 1];
}

message ShardingKey {
  repeated string tag_names = 1 [(validate.rules).repeated.min_items = 1];
}

enum FieldType {
  FIELD_TYPE_UNSPECIFIED = 0;
  FIELD_TYPE_STRING = 1;
  FIELD_TYPE_INT = 2;
  FIELD_TYPE_DATA_BINARY = 3;
  FIELD_TYPE_FLOAT = 4;
}

enum EncodingMethod {
  ENCODING_METHOD_UNSPECIFIED = 0;
  ENCODING_METHOD_GORILLA = 1;
}

enum CompressionMethod {
  COMPRESSION_METHOD_UNSPECIFIED = 0;
  COMPRESSION_METHOD_ZSTD = 1;
}

// FieldSpec is the specification of field
message FieldSpec {
  // name is the identity of a field
  string name = 1 [(validate.rules).string.min_len = 1];
  // field_type denotes the type of field value
  FieldType field_type = 2 [(validate.rules).enum.defined_only = true];
  // encoding_method indicates how to encode data during writing
  EncodingMethod encoding_method = 3 [(validate.rules).enum.defined_only = true];
  // compression_method indicates how to compress data during writing
  CompressionMethod compression_method = 4 [(validate.rules).enum.defined_only = true];
}

// Measure intends to store data point
message Measure {
  // metadata is the identity of a measure
  common.v1.Metadata metadata = 1 [(validate.rules).message.required = true];
  // tag_families are for filter measures
  repeated TagFamilySpec tag_families = 2 [(validate.rules).repeated.min_items = 1];
  // fields denote measure values
  repeated FieldSpec fields = 3;
  // entity indicates which tags will be to generate a series and shard a measure
  Entity entity = 4 [(validate.rules).message.required = true];
  // interval indicates how frequently to send a data point
  // valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h", "d".
  string interval = 5;
  // updated_at indicates when the measure is updated
  google.protobuf.Timestamp updated_at = 6;
  // index_mode specifies whether the data should be stored exclusively in the index,
  // meaning it will not be stored in the data storage system.
  bool index_mode = 7;
  // sharding_key determines the distribution of TopN-related data.
  ShardingKey sharding_key = 8;
}

// TopNAggregation generates offline TopN statistics for a measure's TopN approximation
message TopNAggregation {
  // metadata is the identity of an aggregation
  common.v1.Metadata metadata = 1 [(validate.rules).message.required = true];
  // source_measure denotes the data source of this aggregation
  common.v1.Metadata source_measure = 2 [(validate.rules).message.required = true];
  // field_name is the name of field used for ranking
  string field_name = 3 [(validate.rules).string.min_len = 1];
  // field_value_sort indicates how to sort fields
  // ASC: bottomN
  // DESC: topN
  // UNSPECIFIED: topN + bottomN
  // todo validate plugin exist bug https://github.com/bufbuild/protoc-gen-validate/issues/672
  model.v1.Sort field_value_sort = 4;
  // group_by_tag_names groups data points into statistical counters
  repeated string group_by_tag_names = 5;
  // criteria select partial data points from measure
  model.v1.Criteria criteria = 6;
  // counters_number sets the number of counters to be tracked. The default value is 1000
  int32 counters_number = 7;
  // lru_size defines how much entry is allowed to be maintained in the memory
  int32 lru_size = 8;
  // updated_at indicates when the measure is updated
  google.protobuf.Timestamp updated_at = 9;
}

// IndexRule defines how to generate indices based on tags and the index type
// IndexRule should bind to a subject through an IndexRuleBinding to generate proper indices.
message IndexRule {
  // metadata define the rule's identity
  common.v1.Metadata metadata = 1 [(validate.rules).message.required = true];
  // tags are the combination that refers to an indexed object
  // If the elements in tags are more than 1, the object will generate a multi-tag index
  // Caveat: All tags in a multi-tag MUST have an identical IndexType
  repeated string tags = 2 [(validate.rules).repeated.min_items = 1];
  // Type determine the index structure under the hood
  enum Type {
    TYPE_UNSPECIFIED = 0;
    TYPE_INVERTED = 1;
  }
  // type is the IndexType of this IndexObject.
  Type type = 3 [(validate.rules).enum.defined_only = true];
  // updated_at indicates when the IndexRule is updated
  google.protobuf.Timestamp updated_at = 4;

  // analyzer analyzes tag value to support the full-text searching for TYPE_INVERTED indices.
  // available analyzers are:
  // - "standard" provides grammar based tokenization
  // - "simple" breaks text into tokens at any non-letter character,
  //            such as numbers, spaces, hyphens and apostrophes, discards non-letter characters,
  //            and changes uppercase to lowercase.
  // - "keyword" is a “noop” analyzer which returns the entire input string as a single token.
  // - "url" breaks test into tokens at any non-letter and non-digit character.
  string analyzer = 5;
  // no_sort indicates whether the index is not for sorting.
  bool no_sort = 6;
}

// Subject defines which stream or measure would generate indices
message Subject {
  // catalog is where the subject belongs to
  // todo validate plugin exist bug https://github.com/bufbuild/protoc-gen-validate/issues/672
  common.v1.Catalog catalog = 1;
  // name refers to a stream or measure in a particular catalog
  string name = 2 [(validate.rules).string.min_len = 1];
}

// IndexRuleBinding is a bridge to connect severalIndexRules to a subject
// This binding is valid between begin_at_nanoseconds and expire_at_nanoseconds, that provides flexible strategies
// to control how to generate time series indices.
message IndexRuleBinding {
  // metadata is the identity of this binding
  common.v1.Metadata metadata = 1 [(validate.rules).message.required = true];
  // rules refers to the IndexRule
  repeated string rules = 2 [(validate.rules).repeated.min_items = 1];
  // subject indicates the subject of binding action
  Subject subject = 3 [(validate.rules).message.required = true];
  // begin_at_nanoseconds is the timestamp, after which the binding will be active
  google.protobuf.Timestamp begin_at = 4 [(validate.rules).timestamp.required = true];
  // expire_at_nanoseconds it the timestamp, after which the binding will be inactive
  // expire_at_nanoseconds must be larger than begin_at_nanoseconds
  google.protobuf.Timestamp expire_at = 5 [(validate.rules).timestamp.required = true];
  // updated_at indicates when the IndexRuleBinding is updated
  google.protobuf.Timestamp updated_at = 6;
}

// Property stores the user defined data
message Property {
  // metadata is the identity of a property
  common.v1.Metadata metadata = 1 [(validate.rules).message.required = true];
  // tag stores the content of a property
  repeated TagSpec tags = 2 [(validate.rules).repeated.min_items = 1];
  // updated_at indicates when the property is updated
  google.protobuf.Timestamp updated_at = 6;
}