pkg/approvaltest/approvals.go

// Licensed to Elasticsearch B.V. under one or more contributor
// license agreements. See the NOTICE file distributed with
// this work for additional information regarding copyright
// ownership. Elasticsearch B.V. licenses this file to you under
// the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

// Package approvaltest contains helper functions to compare and assert
// the received content of a test vs the accepted.
package approvaltest

import (
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"testing"

	"github.com/google/go-cmp/cmp"
	"github.com/tidwall/gjson"
	"github.com/tidwall/sjson"

	"github.com/elastic/apm-tools/pkg/espoll"
)

const (
	// ApprovedSuffix signals a file has been reviewed and approved.
	ApprovedSuffix = ".approved.json"

	// ReceivedSuffix signals a file has changed and not yet been approved.
	ReceivedSuffix = ".received.json"
)

// ApproveEvents compares the _source of the search hits with the
// contents of the file in "approvals/<name>.approved.json".
//
// Dynamic fields (@timestamp, observer.id, etc.) are replaced
// with a static string for comparison. Integration tests elsewhere
// use canned data to test fields that we do not cover here.
//
// If the events differ, then the test will fail.
func ApproveEvents(t testing.TB, name string, hits []espoll.SearchHit, dynamic ...string) {
	t.Helper()

	sources := make([][]byte, len(hits))
	for i, hit := range hits {
		sources[i] = hit.RawSource
	}
	rewriteDynamic(t, sources, false, dynamic...)
	// Rewrite dynamic fields and sort them for repeatable diffs.
	sort.Slice(sources, func(i, j int) bool {
		return compareDocumentFields(sources[i], sources[j]) < 0
	})
	approveEventDocs(t, filepath.Join("approvals", name), sources)
}

// ApproveFields compares the fields of the search hits with the
// contents of the file in "approvals/<name>.approved.json".
//
// Dynamic fields (@timestamp, observer.id, etc.) are replaced
// with a static string for comparison. Integration tests elsewhere
// use canned data to test fields that we do not cover here.
//
// TODO(axw) eventually remove ApproveEvents when we have updated
// all calls to use ApproveFields. ApproveFields should be used
// since it includes runtime fields, whereas ApproveEvents only
// looks at _source.
func ApproveFields(t testing.TB, name string, hits []espoll.SearchHit, dynamic ...string) {
	t.Helper()

	fields := make([][]byte, len(hits))
	for i, hit := range hits {
		fields[i] = hit.RawFields
	}
	// Rewrite dynamic fields and sort them for repeatable diffs.
	rewriteDynamic(t, fields, true, dynamic...)
	sort.Slice(fields, func(i, j int) bool {
		return compareDocumentFields(fields[i], fields[j]) < 0
	})
	approveFields(t, filepath.Join("approvals", name), fields)
}

// rewriteDynamic rewrites all dynamic fields to have a known value, so dynamic
// fields don't affect diffs. The flattenedKeys parameter defines how the
// field should be queried in the source, if flattenedKeys is passed as true
// then the source will be queried for the dynamic fields as flattened keys.
func rewriteDynamic(t testing.TB, srcs [][]byte, flattenedKeys bool, dynamic ...string) {
	t.Helper()

	// Fields generated by the server (e.g. observer.*)
	// agent which may change between tests.
	//
	// Ignore their values in comparisons, but compare
	// existence: either the field exists in both, or neither.
	dynamic = append([]string{
		"ecs.version",
		"event.ingested",
		"observer.ephemeral_id",
		"observer.hostname",
		"observer.id",
		"observer.version",
	}, dynamic...)

	for i := range srcs {
		for _, field := range dynamic {
			if flattenedKeys {
				field = strings.ReplaceAll(field, ".", "\\.")
			}
			existing := gjson.GetBytes(srcs[i], field)
			if !existing.Exists() {
				continue
			}

			var v interface{}
			if existing.IsArray() {
				v = []any{"dynamic"}
			} else {
				v = "dynamic"
			}

			var err error
			srcs[i], err = sjson.SetBytes(srcs[i], field, v)
			if err != nil {
				t.Fatal(err)
			}
		}
	}
}

// approveEventDocs compares the given event documents with
// the contents of the file in "<name>.approved.json".
//
// Any specified dynamic fields (e.g. @timestamp, observer.id)
// will be replaced with a static string for comparison.
//
// If the events differ, then the test will fail.
func approveEventDocs(t testing.TB, name string, eventDocs [][]byte) {
	t.Helper()

	events := make([]interface{}, len(eventDocs))
	for i, doc := range eventDocs {

		var event map[string]interface{}
		if err := json.Unmarshal(doc, &event); err != nil {
			t.Fatal(err)
		}
		events[i] = event
	}

	received := map[string]interface{}{"events": events}
	approve(t, name, received)
}

func approveFields(t testing.TB, name string, docs [][]byte) {
	t.Helper()

	// Rewrite all dynamic fields to have a known value,
	// so dynamic fields don't affect diffs.
	decodedDocs := make([]any, len(docs))
	for i, doc := range docs {
		var fields map[string]any
		if err := json.Unmarshal(doc, &fields); err != nil {
			t.Fatal(err)
		}
		decodedDocs[i] = fields
	}

	approve(t, name, decodedDocs)
}

// approve compares the given value with the contents of the file
// "<name>.approved.json".
//
// If the value differs, then the test will fail.
func approve(t testing.TB, name string, received interface{}) {
	t.Helper()

	var approved interface{}
	if err := readApproved(name, &approved); err != nil {
		t.Fatalf("failed to read approved file: %v", err)
	}
	if diff := cmp.Diff(approved, received); diff != "" {
		if err := writeReceived(name, received); err != nil {
			t.Fatalf("failed to write received file: %v", err)
		}
		t.Fatalf("%s\n%s\n\n", diff,
			"Test failed. Run `make check-approvals` to verify the diff.",
		)
	} else {
		// Remove an old *.received.json file if it exists, ignore errors
		_ = removeReceived(name)
	}
}

func readApproved(name string, approved interface{}) error {
	path := name + ApprovedSuffix
	f, err := os.Open(path)
	if err != nil && !os.IsNotExist(err) {
		return fmt.Errorf("failed to open approved file for %s: %w", name, err)
	}
	defer f.Close()
	if os.IsNotExist(err) {
		return nil
	}
	if err := json.NewDecoder(f).Decode(&approved); err != nil {
		return fmt.Errorf("failed to decode approved file for %s: %w", name, err)
	}
	return nil
}

func removeReceived(name string) error {
	return os.Remove(name + ReceivedSuffix)
}

func writeReceived(name string, received interface{}) error {
	fullpath := name + ReceivedSuffix
	if err := os.MkdirAll(filepath.Dir(fullpath), 0755); err != nil {
		return fmt.Errorf("failed to create directories for received file: %w", err)
	}
	f, err := os.Create(fullpath)
	if err != nil {
		return fmt.Errorf("failed to create received file for %s: %w", name, err)
	}
	defer f.Close()
	enc := json.NewEncoder(f)
	enc.SetIndent("", "    ")
	if err := enc.Encode(received); err != nil {
		return fmt.Errorf("failed to encode received file for %s: %w", name, err)
	}
	return nil
}