model.go (364 lines of code) (raw):

package sls import ( "encoding/json" "net/http" "net/url" "strconv" "strings" "github.com/go-kit/kit/log/level" ) // GetLogRequest for GetLogsV2 type GetLogRequest struct { From int64 `json:"from"` // unix time, eg time.Now().Unix() - 900 To int64 `json:"to"` // unix time, eg time.Now().Unix() Topic string `json:"topic"` // @note topic is not used anymore, use __topic__ : xxx in query instead Lines int64 `json:"line"` // max 100; offset, lines and reverse is ignored when use SQL in query Offset int64 `json:"offset"` Reverse bool `json:"reverse"` Query string `json:"query"` PowerSQL bool `json:"powerSql"` FromNsPart int32 `json:"fromNs"` ToNsPart int32 `json:"toNs"` NeedHighlight bool `json:"highlight"` IsAccurate bool `json:"accurate"` } func (glr *GetLogRequest) ToURLParams() url.Values { urlVal := url.Values{} urlVal.Add("type", "log") urlVal.Add("from", strconv.Itoa(int(glr.From))) urlVal.Add("to", strconv.Itoa(int(glr.To))) urlVal.Add("topic", glr.Topic) urlVal.Add("line", strconv.Itoa(int(glr.Lines))) urlVal.Add("offset", strconv.Itoa(int(glr.Offset))) urlVal.Add("reverse", strconv.FormatBool(glr.Reverse)) urlVal.Add("powerSql", strconv.FormatBool(glr.PowerSQL)) urlVal.Add("query", glr.Query) urlVal.Add("fromNs", strconv.Itoa(int(glr.FromNsPart))) urlVal.Add("toNs", strconv.Itoa(int(glr.ToNsPart))) urlVal.Add("highlight", strconv.FormatBool(glr.NeedHighlight)) urlVal.Add("accurate", strconv.FormatBool(glr.IsAccurate)) return urlVal } type GetHistogramRequest struct { Topic string `json:"topic"` From int64 `json:"from"` To int64 `json:"to"` Query string `json:"query"` Interval int32 `json:"interval"` } func (ghr *GetHistogramRequest) ToURLParams() url.Values { urlVal := url.Values{} urlVal.Add("type", "histogram") urlVal.Add("from", strconv.Itoa(int(ghr.From))) urlVal.Add("to", strconv.Itoa(int(ghr.To))) urlVal.Add("topic", ghr.Topic) urlVal.Add("query", ghr.Query) urlVal.Add("interval", strconv.Itoa(int(ghr.Interval))) return urlVal } type PullLogRequest struct { Project string Logstore string ShardID int Cursor string EndCursor string LogGroupMaxCount int Query string // Deprecated: PullMode is not used PullMode string QueryId string CompressType int } func (plr *PullLogRequest) ToURLParams() url.Values { urlVal := url.Values{} urlVal.Add("type", "logs") urlVal.Add("cursor", plr.Cursor) urlVal.Add("count", strconv.Itoa(plr.LogGroupMaxCount)) if plr.EndCursor != "" { urlVal.Add("end_cursor", plr.EndCursor) } if plr.Query != "" { urlVal.Add("query", plr.Query) urlVal.Add("pullMode", "scan_on_stream") if plr.QueryId != "" { urlVal.Add("queryId", plr.QueryId) } } return urlVal } type PullLogMeta struct { NextCursor string Netflow int RawSize int Count int readLastCursor string // int64 string, eg: "1732154287213232020" // these fields are only present when query is set RawSizeBeforeQuery int // processed raw size before query Lines int // result lines after query LinesBeforeQuery int // processed lines before query FailedLines int // failed lines during query DataCountBeforeQuery int // processed logGroup count before query } // GetHistogramsResponse defines response from GetHistograms call type SingleHistogram struct { Progress string `json:"progress"` Count int64 `json:"count"` From int64 `json:"from"` To int64 `json:"to"` } type GetHistogramsResponse struct { Progress string `json:"progress"` Count int64 `json:"count"` Histograms []SingleHistogram `json:"histograms"` } func (resp *GetHistogramsResponse) IsComplete() bool { return strings.ToLower(resp.Progress) == "complete" } // GetLogsResponse defines response from GetLogs call type GetLogsResponse struct { Progress string `json:"progress"` Count int64 `json:"count"` Logs []map[string]string `json:"logs"` Contents string `json:"contents"` HasSQL bool `json:"hasSQL"` Header http.Header `json:"header"` } type MetaTerm struct { Key string `json:"key"` Term string `json:"term"` } type PhraseQueryInfoV3 struct { ScanAll *bool `json:"scanAll,omitempty"` BeginOffset *int64 `json:"beginOffset,omitempty"` EndOffset *int64 `json:"endOffset,omitempty"` EndTime *int64 `json:"endTime,omitempty"` } type GetLogsV3ResponseMeta struct { Progress string `json:"progress"` AggQuery string `json:"aggQuery"` WhereQuery string `json:"whereQuery"` HasSQL bool `json:"hasSQL"` ProcessedRows int64 `json:"processedRows"` ElapsedMillisecond int64 `json:"elapsedMillisecond"` CpuSec float64 `json:"cpuSec"` CpuCores float64 `json:"cpuCores"` Limited int64 `json:"limited"` Count int64 `json:"count"` ProcessedBytes int64 `json:"processedBytes"` TelemetryType string `json:"telementryType"` // telementryType, ignore typo PowerSql bool `json:"powerSql"` InsertedSql string `json:"insertedSQL"` Keys []string `json:"keys,omitempty"` Terms []MetaTerm `json:"terms,omitempty"` Marker *string `json:"marker,omitempty"` Mode *int `json:"mode,omitempty"` PhraseQueryInfo *PhraseQueryInfoV3 `json:"phraseQueryInfo,omitempty"` Shard *int `json:"shard,omitempty"` ScanBytes *int64 `json:"scanBytes,omitempty"` IsAccurate *bool `json:"isAccurate,omitempty"` ColumnTypes []string `json:"columnTypes,omitempty"` Highlights []map[string]string `json:"highlights,omitempty"` } type PhraseQueryInfoV2 struct { ScanAll string `json:"scanAll,omitempty"` BeginOffset string `json:"beginOffset,omitempty"` EndOffset string `json:"endOffset,omitempty"` EndTime string `json:"endTime,omitempty"` } func (s *PhraseQueryInfoV3) toPhraseQueryInfoV2() *PhraseQueryInfoV2 { if s == nil { return nil } return &PhraseQueryInfoV2{ ScanAll: BoolPtrToStringNum(s.ScanAll), BeginOffset: Int64PtrToString(s.BeginOffset), EndOffset: Int64PtrToString(s.EndOffset), EndTime: Int64PtrToString(s.EndTime), } } type QueryInfoV2 struct { Keys []string `json:"keys,omitempty"` Terms [][]string `json:"terms,omitempty"` // [[term, key], [term2, key2]] Limited string `json:"limited,omitempty"` Marker *string `json:"marker,omitempty"` Mode *int `json:"mode,omitempty"` PhraseQueryInfo *PhraseQueryInfoV2 `json:"phraseQueryInfo,omitempty"` Shard *int `json:"shard,omitempty"` ScanBytes *int64 `json:"scanBytes,omitempty"` IsAccurate *int64 `json:"isAccurate,omitempty"` ColumnTypes []string `json:"columnTypes,omitempty"` Highlights []map[string]string `json:"highlight,omitempty"` } func (meta *GetLogsV3ResponseMeta) constructQueryInfo() (string, error) { var terms [][]string for _, term := range meta.Terms { terms = append(terms, []string{term.Term, term.Key}) } var isAccurate *int64 if meta.IsAccurate != nil { res := BoolToInt64(*meta.IsAccurate) isAccurate = &res } limited := "" if meta.Limited != 0 { limited = strconv.FormatInt(meta.Limited, 10) } queryInfo := &QueryInfoV2{ Keys: meta.Keys, Terms: terms, Limited: limited, Marker: meta.Marker, Mode: meta.Mode, PhraseQueryInfo: meta.PhraseQueryInfo.toPhraseQueryInfoV2(), Shard: meta.Shard, ScanBytes: meta.ScanBytes, IsAccurate: isAccurate, ColumnTypes: meta.ColumnTypes, Highlights: meta.Highlights, } contents, err := json.Marshal(queryInfo) if err != nil { return "", err } return string(contents), nil } // GetLogsV3Response defines response from GetLogs call type GetLogsV3Response struct { Meta GetLogsV3ResponseMeta `json:"meta"` Logs []map[string]string `json:"data"` } func (resp *GetLogsV3Response) IsComplete() bool { return strings.ToLower(resp.Meta.Progress) == "complete" } // GetLogLinesResponse defines response from GetLogLines call // note: GetLogLinesResponse.Logs is nil when use GetLogLinesResponse type GetLogLinesResponse struct { GetLogsResponse Lines []json.RawMessage } func (resp *GetLogsResponse) IsComplete() bool { return strings.ToLower(resp.Progress) == "complete" } func (resp *GetLogsResponse) GetKeys() (error, []string) { type Content map[string][]interface{} var content Content err := json.Unmarshal([]byte(resp.Contents), &content) if err != nil { return err, nil } result := []string{} for _, v := range content["keys"] { result = append(result, v.(string)) } return nil, result } type GetContextLogsResponse struct { Progress string `json:"progress"` TotalLines int64 `json:"total_lines"` BackLines int64 `json:"back_lines"` ForwardLines int64 `json:"forward_lines"` Logs []map[string]string `json:"logs"` } func (resp *GetContextLogsResponse) IsComplete() bool { return strings.ToLower(resp.Progress) == "complete" } type JsonKey struct { Type string `json:"type"` Alias string `json:"alias,omitempty"` DocValue bool `json:"doc_value,omitempty"` } // IndexKey ... type IndexKey struct { Token []string `json:"token"` // tokens that split the log line. CaseSensitive bool `json:"caseSensitive"` Type string `json:"type"` // text, long, double DocValue bool `json:"doc_value,omitempty"` Alias string `json:"alias,omitempty"` Chn bool `json:"chn"` // parse chinese or not JsonKeys map[string]*JsonKey `json:"json_keys,omitempty"` } type IndexLine struct { Token []string `json:"token"` CaseSensitive bool `json:"caseSensitive"` IncludeKeys []string `json:"include_keys,omitempty"` ExcludeKeys []string `json:"exclude_keys,omitempty"` Chn bool `json:"chn"` // parse chinese or not } // Index is an index config for a log store. type Index struct { Keys map[string]IndexKey `json:"keys,omitempty"` Line *IndexLine `json:"line,omitempty"` Ttl uint32 `json:"ttl,omitempty"` MaxTextLen uint32 `json:"max_text_len,omitempty"` LogReduce bool `json:"log_reduce"` LogReduceWhiteListDict []string `json:"log_reduce_white_list,omitempty"` LogReduceBlackListDict []string `json:"log_reduce_black_list,omitempty"` } // CreateDefaultIndex return a full text index config func CreateDefaultIndex() *Index { return &Index{ Line: &IndexLine{ Token: []string{" ", "\n", "\t", "\r", ",", ";", "[", "]", "{", "}", "(", ")", "&", "^", "*", "#", "@", "~", "=", "<", ">", "/", "\\", "?", ":", "'", "\""}, CaseSensitive: false, }, } } type GetMeteringModeResponse struct { MeteringMode string `json:"meteringMode"` } const ( CHARGE_BY_FUNCTION = "ChargeByFunction" CHARGE_BY_DATA_INGEST = "ChargeByDataIngest" ) type PostLogStoreLogsRequest struct { LogGroup *LogGroup HashKey *string CompressType int Processor string } type StoreView struct { Name string `json:"name"` StoreType string `json:"storeType"` Stores []*StoreViewStore `json:"stores"` } // storeType of storeView const ( STORE_VIEW_STORE_TYPE_LOGSTORE = "logstore" STORE_VIEW_STORE_TYPE_METRICSTORE = "metricstore" ) type StoreViewStore struct { Project string `json:"project"` StoreName string `json:"storeName"` Query string `json:"query,omitempty"` } type GetStoreViewIndexResponse struct { Indexes []*StoreViewIndex `json:"indexes"` StoreViewErrors []*StoreViewErrors `json:"storeViewErrors"` } type StoreViewIndex struct { ProjectName string `json:"projectName"` LogStore string `json:"logstore"` Index Index `json:"index"` } type StoreViewErrors struct { ProjectName string `json:"projectName"` LogStore string `json:"logstore"` Status string `json:"status"` Message string `json:"message"` } type ListStoreViewsRequest struct { Offset int `json:"offset"` Size int `json:"size"` } type ListStoreViewsResponse struct { Total int `json:"total"` Count int `json:"count"` StoreViews []string `json:"storeviews"` } // If cursor is unknown, returns empty string // If pullLogs with non-empty query or consumer with non-empty query, returns empty string func (l *LogGroup) GetCursor() string { return l.cursor } func (l *LogGroupList) addCursorIfPossible(readLastCursor string) error { lastCursorInt, err := strconv.ParseInt(readLastCursor, 10, 64) if err != nil { if IsDebugLevelMatched(1) { level.Error(Logger).Log("msg", "decode readLastCursor failed", "cursor", readLastCursor, "err", err) } return err } cursor := lastCursorInt - int64(len(l.LogGroups)) + 1 for i := 0; i < len(l.LogGroups); i++ { l.LogGroups[i].cursor = encodeCursor(cursor) cursor++ } return nil }