lib/collections.go

// Licensed to Elasticsearch B.V. under one or more contributor
// license agreements. See the NOTICE file distributed with
// this work for additional information regarding copyright
// ownership. Elasticsearch B.V. licenses this file to you under
// the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

package lib

import (
	"reflect"
	"sort"
	"strings"

	"github.com/google/cel-go/cel"
	"github.com/google/cel-go/common"
	"github.com/google/cel-go/common/ast"
	"github.com/google/cel-go/common/types"
	"github.com/google/cel-go/common/types/ref"
	"github.com/google/cel-go/common/types/traits"
	"github.com/google/cel-go/parser"
)

// Collections returns a cel.EnvOption to configure extended functions for
// handling collections.
//
// As (Macro)
//
// The as macro is syntactic sugar for [val].map(var, function)[0].
//
// Examples:
//
//	{"a":1, "b":2}.as(v, v.a == 1)         // return true
//	{"a":1, "b":2}.as(v, v)                // return {"a":1, "b":2}
//	{"a":1, "b":2}.as(v, v.with({"c":3}))  // return {"a":1, "b":2, "c":3}
//	{"a":1, "b":2}.as(v, [v, v])           // return [{"a":1, "b":2}, {"a":1, "b":2}]
//
// # Collate
//
// Returns a list of values obtained by traversing fields in the receiver with
// the path or paths given as the string parameter. When a list is traversed all
// children are include in the resulting list:
//
//	<list<dyn>>.collate(<string>) -> <list<dyn>>
//	<list<dyn>>.collate(<list<string>>) -> <list<dyn>>
//	<map<string,dyn>>.collate(<string>) -> <list<dyn>>
//	<map<string,dyn>>.collate(<list<string>>) -> <list<dyn>>
//
// Examples:
//
//	Given v:
//	{
//	        "a": [
//	            {"b": 1},
//	            {"b": 2},
//	            {"b": 3}
//	        ],
//	        "b": [
//	            {"b": -1, "c": 10},
//	            {"b": -2, "c": 20},
//	            {"b": -3, "c": 30}
//	        ]
//	}
//
//	v.collate("a")             // return [{"b": 1}, {"b": 2}, {"b": 3}]
//	v.collate("a.b")           // return [1, 2, 3]
//	v.collate(["a.b", "b.b"])  // return [1, 2, 3, -1, -2, -3]
//	v.collate(["a", "b.b"])    // return [{"b": 1 }, {"b": 2 }, {"b": 3 }, -1, -2, -3 ]
//
// If the the path to be dropped includes a dot, it can be escaped with a literal
// backslash. See drop below.
//
// # Drop
//
// Returns the value of the receiver with the object at the given paths remove:
//
//	<list<dyn>>.drop(<string>) -> <list<dyn>>
//	<list<dyn>>.drop(<list<string>>) -> <list<dyn>>
//	<map<string,dyn>>.drop(<string>) -> <map<string,dyn>>
//	<map<string,dyn>>.drop(<list<string>>) -> <map<string,dyn>>
//
// Examples:
//
//	Given v:
//	{
//	        "a": [
//	            {"b": 1},
//	            {"b": 2},
//	            {"b": 3}
//	        ],
//	        "b": [
//	            {"b": -1, "c": 10},
//	            {"b": -2, "c": 20},
//	            {"b": -3, "c": 30}
//	        ]
//	}
//
//	v.drop("a")             // return {"b": [{"b": -1, "c": 10}, {"b": -2, "c": 20}, {"b": -3, "c": 30}]}
//	v.drop("a.b")           // return {"a": [{}, {}, {}], "b": [{"b": -1, "c": 10}, {"b": -2, "c": 20}, {"b": -3, "c": 30}]}
//	v.drop(["a.b", "b.b"])  // return {"a": [{}, {}, {}], "b": [{"c": 10}, {"c": 20}, {"c": 30}]}
//	v.drop(["a", "b.b"])    // return {"b": [{"c": 10}, {"c": 20}, {"c": 30}]}
//
// If the the path to be dropped includes a dot, it can be escaped with a literal
// backslash.
//
// Examples:
//
//	Given v:
//	{
//	        "dotted.path": [
//	            {"b": -1, "c": 10},
//	            {"b": -2, "c": 20},
//	            {"b": -3, "c": 30}
//	        ]
//	}
//
//	v.drop("dotted\\.path.b")  // return {"dotted.path": [{"c": 10}, {"c": 20}, {"c": 30}]}
//
// # Drop Empty
//
// Returns the value of the receiver with all empty lists and maps removed,
// recursively
//
//	<list<dyn>>.drop_empty() -> <list<dyn>>
//	<map<string,dyn>>.drop_empty() -> <map<string,dyn>>
//
// Examples:
//
//	Given v:
//	{
//	        "a": [
//	            {},
//	            {},
//	            {}
//	        ],
//	        "b": [
//	            {"b": -1, "c": 10},
//	            {"b": -2, "c": 20},
//	            {"b": -3, "c": 30}
//	        ]
//	}
//
//	v.drop_empty()  // return {"b":[{"b":-1, "c":10}, {"b":-2, "c":20}, {"b":-3, "c":30}]}
//
// # Flatten
//
// Returns a list of non-list objects resulting from the depth-first
// traversal of a nested list:
//
//	<list<dyn>...>.flatten() -> <list<dyn>>
//
// Examples:
//
//	[[1],[2,3],[[[4]],[5,6]]].flatten()                     // return [1, 2, 3, 4, 5, 6]
//	[[{"a":1,"b":[10, 11]}],[2,3],[[[4]],[5,6]]].flatten()  // return [{"a":1, "b":[10, 11]}, 2, 3, 4, 5, 6]
//
// # Front
//
// Returns the first n elements of a list:
//
//	front(<list<dyn>>, <int>) -> <optional<dyn>>
//
// Examples:
//
//	front([1, 2, 3, 4, 5, 6], 2)  // return [1, 2]
//	front([6], 2)                 // return [6]
//	front([], 2)                  // return []
//
// # Keys
//
// Returns a list of keys from a map:
//
//	keys(<map<dyn,dyn>>) -> <list<dyn>>
//	<map<dyn,dyn>>.keys() -> <list<dyn>>
//
// Examples:
//
//	keys({"a":1, "b":2})   // return ["a", "b"]
//	{1:"a", 2:"b"}.keys()   // return [1, 2]
//
// # Max
//
// Returns the maximum value of a list or pair of comparable objects:
//
//	<list<dyn>>.max() -> <dyn>
//	max(<list<dyn>>) -> <dyn>
//	max(<dyn>, <dyn>) -> <dyn>
//
// Examples:
//
//	[1,2,3,4,5,6,7].max()  // return 7
//	max([1,2,3,4,5,6,7])   // return 7
//	max(1,7)               // return 7
//
// # Min
//
// Returns the minimum value of a list or pair of comparable objects:
//
//	<list<dyn>>.min() -> <dyn>
//	min(<list<dyn>>) -> <dyn>
//	min(<dyn>, <dyn>) -> <dyn>
//
// Examples:
//
//	[1,2,3,4,5,6,7].min()  // return 1
//	min([1,2,3,4,5,6,7])   // return 1
//	min(1,7)               // return 1
//
// # Sum
//
// Returns the sum of a list of int or list of double:
//
//	<list<int>>.sum() -> <int>
//	sum(<list<int>>) -> <int>
//	<list<double>>.sum() -> <double>
//	sum(<list<double>>) -> <double>
//
// sum([]) or sum with a list of mixed int and double will return an error.
//
// Examples:
//
//	[1,2,3,4,5,6,7].sum()  // return 28
//	sum([1,2,3,4,5,6,7])   // return 28
//
// # Tail
//
// Returns the elements of a list after the first element, or if an integer
// second parameter is provided, after that index:
//
//	tail(<list<dyn>>) -> <optional<dyn>>
//	tail(<list<dyn>>, <int>) -> <optional<dyn>>
//
// Examples:
//
//	tail([1, 2, 3, 4, 5, 6])     // return [2, 3, 4, 5, 6]
//	tail([6])                    // return []
//	tail([])                     // return []
//	tail([1, 2, 3, 4, 5, 6], 2)  // return [3, 4, 5, 6]
//	tail([6], 2)                 // return []
//	tail([], 2)                  // return []
//
// The conjugate of the single parameter tail call, getting the first element,
// can be achieved directly using list indexing, for example if a is
// [1, 2, 3, 4, 5, 6] and b is []:
//
//	a[?0]  // return 1
//	b[?0]  // return optional.none
//
// The conjugate of the two parameter tail call, getting the first n elements,
// can be achieved by using the front function.
//
// # Values
//
// Returns a list of values from a map:
//
//	values(<map<dyn,dyn>>) -> <list<dyn>>
//	<map<dyn,dyn>>.values() -> <list<dyn>>
//
// Examples:
//
//	values({"a":1, "b":2})   // return [1, 2]
//	{1:"a", 2:"b"}.values()   // return ["a", "b"]
//
// # With
//
// Returns the receiver's value with the value of the parameter updating
// or adding fields:
//
//	<map<K,V>>.with(<map<K,V>>) -> <map<K,V>>
//
// Examples:
//
//	{"a":1, "b":2}.with({"a":10, "c":3})  // return {"a":10, "b":2, "c":3}
//
// # With Replace
//
// Returns the receiver's value with the value of the parameter replacing
// existing fields:
//
//	<map<K,V>>.with_replace(<map<K,V>>) -> <map<K,V>>
//
// Examples:
//
//	{"a":1, "b":2}.with_replace({"a":10, "c":3})  // return {"a":10, "b":2}
//
// # With Update
//
// Returns the receiver's value with the value of the parameter updating
// the map without replacing any existing fields:
//
//	<map<K,V>>.with_update(<map<K,V>>) -> <map<K,V>>
//
// Examples:
//
//	{"a":1, "b":2}.with_update({"a":10, "c":3})  // return {"a":1, "b":2, "c":3}
//
// # Zip
//
// Returns a map keyed on elements of a list with values from another equally
// sized list:
//
//	zip(<list<K>>, <list<V>>) -> <map<K,V>>
//	<list<K>>.zip(<list<V>>) -> <map<K,V>>
//
// Examples:
//
//	zip(["a", "b"], [1, 2])  // return {"a":1, "b":2}
//	["a", "b"].zip([1, 2])   // return {"a":1, "b":2}
func Collections() cel.EnvOption {
	return cel.Lib(collectionsLib{})
}

type collectionsLib struct{}

func (collectionsLib) CompileOptions() []cel.EnvOption {
	return []cel.EnvOption{
		cel.Macros(parser.NewReceiverMacro("as", 2, makeAs)),

		cel.Function("collate",
			cel.MemberOverload(
				"list_collate_string",
				[]*cel.Type{listDyn, cel.StringType},
				listDyn,
				cel.BinaryBinding(catch(collateFields)),
			),
			cel.MemberOverload(
				"list_collate_list_string",
				[]*cel.Type{listDyn, listString},
				listDyn,
				cel.BinaryBinding(catch(collateFields)),
			),
			cel.MemberOverload(
				"map_collate_string",
				[]*cel.Type{mapStringDyn, cel.StringType},
				listDyn,
				cel.BinaryBinding(catch(collateFields)),
			),
			cel.MemberOverload(
				"map_collate_list_string",
				[]*cel.Type{mapStringDyn, listString},
				listDyn,
				cel.BinaryBinding(catch(collateFields)),
			),
		),

		cel.Function("drop",
			cel.MemberOverload(
				"list_drop_string",
				[]*cel.Type{listV, cel.StringType},
				listV,
				cel.BinaryBinding(catch(dropFields)),
			),
			cel.MemberOverload(
				"list_drop_list_string",
				[]*cel.Type{listV, listString},
				listV,
				cel.BinaryBinding(catch(dropFields)),
			),
			cel.MemberOverload(
				"map_drop_string",
				[]*cel.Type{mapKV, cel.StringType},
				mapKV,
				cel.BinaryBinding(catch(dropFields)),
			),
			cel.MemberOverload(
				"map_drop_list_string",
				[]*cel.Type{mapKV, listString},
				mapKV,
				cel.BinaryBinding(catch(dropFields)),
			),
		),

		cel.Function("drop_empty",
			cel.MemberOverload(
				"list_drop_empty",
				[]*cel.Type{listV},
				listV,
				cel.UnaryBinding(catch(dropEmpty)),
			),
			cel.MemberOverload(
				"map_drop_empty",
				[]*cel.Type{mapKV},
				mapKV,
				cel.UnaryBinding(catch(dropEmpty)),
			),
		),

		cel.Function("flatten",
			cel.MemberOverload(
				"list_flatten",
				[]*cel.Type{listDyn},
				listDyn,
				cel.UnaryBinding(catch(flatten)),
			),
		),

		cel.Function("front",
			cel.Overload(
				"front_list_int",
				[]*cel.Type{listV, cel.IntType},
				listV,
				cel.BinaryBinding(catch(frontN)),
			),
		),

		cel.Function("keys",
			cel.MemberOverload(
				"map_keys",
				[]*cel.Type{mapKV},
				listK,
				cel.UnaryBinding(catch(mapKeys)),
			),
			cel.Overload(
				"keys_map",
				[]*cel.Type{mapKV},
				listK,
				cel.UnaryBinding(catch(mapKeys)),
			),
		),

		cel.Function("max",
			cel.MemberOverload(
				"list_max",
				[]*cel.Type{listV},
				typeV,
				cel.UnaryBinding(catch(maxList)),
			),
			cel.Overload(
				"max_list",
				[]*cel.Type{listV},
				typeV,
				cel.UnaryBinding(catch(maxList)),
			),
			cel.Overload(
				"max_pair",
				[]*cel.Type{typeV, typeV},
				typeV,
				cel.BinaryBinding(maxDiadic),
			),
		),

		cel.Function("min",
			cel.MemberOverload(
				"list_min",
				[]*cel.Type{listV},
				typeV,
				cel.UnaryBinding(catch(minList)),
			),
			cel.Overload(
				"min_list",
				[]*cel.Type{listV},
				typeV,
				cel.UnaryBinding(catch(minList)),
			),
			cel.Overload(
				"min_pair",
				[]*cel.Type{typeV, typeV},
				typeV,
				cel.BinaryBinding(minDiadic),
			),
		),

		cel.Function("sum",
			cel.MemberOverload(
				"list_sum",
				[]*cel.Type{listV},
				typeV,
				cel.UnaryBinding(catch(sumList)),
			),
			cel.Overload(
				"sum_list",
				[]*cel.Type{listV},
				typeV,
				cel.UnaryBinding(catch(sumList)),
			),
		),

		cel.Function("tail",
			cel.Overload(
				"tail_list",
				[]*cel.Type{listV},
				listV,
				cel.UnaryBinding(catch(tail)),
			),
			cel.Overload(
				"tail_list_int",
				[]*cel.Type{listV, cel.IntType},
				listV,
				cel.BinaryBinding(catch(tailN)),
			),
		),

		cel.Function("values",
			cel.MemberOverload(
				"map_values",
				[]*cel.Type{mapKV},
				listK,
				cel.UnaryBinding(catch(mapValues)),
			),
			cel.Overload(
				"values_map",
				[]*cel.Type{mapKV},
				listK,
				cel.UnaryBinding(catch(mapValues)),
			),
		),

		cel.Function("with",
			cel.MemberOverload(
				"map_with_map",
				[]*cel.Type{mapKV, mapKV},
				mapKV,
				cel.BinaryBinding(catch(withAll)),
			),
		),

		cel.Function("with_update",
			cel.MemberOverload(
				"map_with_update_map",
				[]*cel.Type{mapKV, mapKV},
				mapKV,
				cel.BinaryBinding(catch(withUpdate)),
			),
		),

		cel.Function("with_replace",
			cel.MemberOverload(
				"map_with_replace_map",
				[]*cel.Type{mapKV, mapKV},
				mapKV,
				cel.BinaryBinding(catch(withReplace)),
			),
		),

		cel.Function("zip",
			cel.MemberOverload(
				"list_zip",
				[]*cel.Type{listK, listV},
				mapKV,
				cel.BinaryBinding(catch(zipLists)),
			),
			cel.Overload(
				"zip_list",
				[]*cel.Type{listK, listV},
				mapKV,
				cel.BinaryBinding(catch(zipLists)),
			),
		),
	}
}

func (collectionsLib) ProgramOptions() []cel.ProgramOption { return nil }

func frontN(arg0, arg1 ref.Val) ref.Val {
	obj := arg0
	l, ok := obj.(traits.Lister)
	if !ok {
		return types.ValOrErr(obj, "no such overload")
	}
	until, ok := arg1.(types.Int)
	if !ok {
		return types.ValOrErr(until, "no such overload")
	}
	if l.Size() == types.IntZero {
		return arg0
	}
	t := make([]ref.Val, 0, until)
	it := l.Iterator()
	for it.HasNext() == types.True && until > 0 {
		t = append(t, it.Next())
		until--
	}
	return types.NewRefValList(types.DefaultTypeAdapter, t)
}

func tail(arg ref.Val) ref.Val {
	obj := arg
	l, ok := obj.(traits.Lister)
	if !ok {
		return types.ValOrErr(obj, "no such overload")
	}
	if l.Size() == types.IntZero {
		return arg
	}
	n := l.Size().(types.Int)
	t := make([]ref.Val, 0, n-1)
	it := l.Iterator()
	head := true
	for it.HasNext() == types.True {
		if head {
			it.Next()
			head = false
			continue
		}
		t = append(t, it.Next())
	}
	return types.NewRefValList(types.DefaultTypeAdapter, t)
}

func tailN(arg0, arg1 ref.Val) ref.Val {
	obj := arg0
	l, ok := obj.(traits.Lister)
	if !ok {
		return types.ValOrErr(obj, "no such overload")
	}
	after, ok := arg1.(types.Int)
	if !ok {
		return types.ValOrErr(after, "no such overload")
	}
	n := l.Size().(types.Int)
	if n == 0 {
		return arg0
	}
	if n <= after {
		return types.NewRefValList(types.DefaultTypeAdapter, []ref.Val{})
	}
	t := make([]ref.Val, 0, n-after)
	it := l.Iterator()
	for it.HasNext() == types.True {
		if after > 0 {
			it.Next()
			after--
			continue
		}
		t = append(t, it.Next())
	}
	return types.NewRefValList(types.DefaultTypeAdapter, t)
}

func flatten(arg ref.Val) ref.Val {
	obj := arg
	l, ok := obj.(traits.Lister)
	if !ok {
		return types.ValOrErr(obj, "no such overload")
	}
	dst := types.NewMutableList(types.DefaultTypeAdapter)
	flattenParts(dst, l)
	return dst.ToImmutableList()
}

func flattenParts(dst traits.MutableLister, val traits.Lister) {
	it := val.Iterator()
	for it.HasNext().Value().(bool) {
		if _, ok := it.Next().(traits.Lister); !ok {
			dst.Add(val)
			return
		}
	}
	it = val.Iterator()
	for it.HasNext() == types.True {
		flattenParts(dst, it.Next().(traits.Lister))
	}
}

func withAll(dst, src ref.Val) ref.Val {
	new, other, err := with(dst, src)
	if err != nil {
		return err
	}
	for k, v := range other {
		new[k] = v
	}
	return types.NewRefValMap(types.DefaultTypeAdapter, new)
}

func withUpdate(dst, src ref.Val) ref.Val {
	new, other, err := with(dst, src)
	if err != nil {
		return err
	}
	for k, v := range other {
		if _, ok := new[k]; ok {
			continue
		}
		new[k] = v
	}
	return types.NewRefValMap(types.DefaultTypeAdapter, new)
}

func withReplace(dst, src ref.Val) ref.Val {
	new, other, err := with(dst, src)
	if err != nil {
		return err
	}
	for k, v := range other {
		if _, ok := new[k]; !ok {
			continue
		}
		new[k] = v
	}
	return types.NewRefValMap(types.DefaultTypeAdapter, new)
}

var refValMap = reflect.TypeOf(map[ref.Val]ref.Val(nil))

func with(dst, src ref.Val) (res, other map[ref.Val]ref.Val, maybe ref.Val) {
	obj, ok := dst.(traits.Mapper)
	if !ok {
		return nil, nil, types.ValOrErr(obj, "no such overload")
	}
	val, ok := src.(traits.Mapper)
	if !ok {
		return nil, nil, types.ValOrErr(src, "unsupported src type")
	}

	new := make(map[ref.Val]ref.Val)
	m, err := obj.ConvertToNative(refValMap)
	if err != nil {
		return nil, nil, types.NewErr("unable to convert dst to native: %v", err)
	}
	for k, v := range m.(map[ref.Val]ref.Val) {
		new[k] = v
	}
	m, err = val.ConvertToNative(refValMap)
	if err != nil {
		return nil, nil, types.NewErr("unable to convert src to native: %v", err)
	}
	return new, m.(map[ref.Val]ref.Val), nil
}

// TODO: Make this configurable to allow map, list and string emptiness and null.
func dropEmpty(val ref.Val) ref.Val {
	obj, ok := val.(iterator)
	if !ok || !hasEmpty(obj) {
		return val
	}

	switch obj := val.(type) {
	case traits.Lister:
		new := make([]ref.Val, 0, obj.Size().Value().(int64))
		it := obj.Iterator()
		for it.HasNext() == types.True {
			elem := it.Next()
			switch val := elem.(type) {
			case iterator:
				if val.Size() != types.IntZero {
					res := dropEmpty(val)
					if v, ok := res.(traits.Sizer); ok {
						if v.Size() != types.IntZero {
							new = append(new, res)
						}
					} else {
						new = append(new, res)
					}
				}
			default:
				new = append(new, val)
			}
		}
		return types.NewRefValList(types.DefaultTypeAdapter, new)

	case traits.Mapper:
		new := make(map[ref.Val]ref.Val)
		m, err := obj.ConvertToNative(refValMap)
		if err != nil {
			return types.NewErr("unable to convert map to native: %v", err)
		}
		for k, v := range m.(map[ref.Val]ref.Val) {
			switch val := v.(type) {
			case iterator:
				if val.Size() != types.IntZero {
					res := dropEmpty(v)
					if v, ok := res.(traits.Sizer); ok {
						if v.Size() != types.IntZero {
							new[k] = res
						}
					} else {
						new[k] = res
					}
				}
			default:
				new[k] = v
			}
		}
		return types.NewRefValMap(types.DefaultTypeAdapter, new)

	default:
		// This should never happen since non-iterator
		// types will have been returned in the preamble.
		return val
	}
}

// hasEmpty returns whether val is a map or a list that has any zero-sized
// map or list elements recursively. Zero sized strings are not considered
// to be empty.
func hasEmpty(val iterator) bool {
	it := val.Iterator()
	switch val := val.(type) {
	case traits.Lister:
		for it.HasNext() == types.True {
			elem := it.Next()
			iter, ok := elem.(iterator)
			if !ok {
				continue
			}
			if iter.Size() == types.IntZero || hasEmpty(iter) {
				return true
			}
		}
	case traits.Mapper:
		for it.HasNext() == types.True {
			elem := val.Get(it.Next())
			iter, ok := elem.(iterator)
			if !ok {
				continue
			}
			if iter.Size() == types.IntZero || hasEmpty(iter) {
				return true
			}
		}
	}
	return false
}

// iterator is the common interface for lists and maps required for dropEmpty.
type iterator interface {
	ref.Val
	traits.Iterable
	traits.Sizer
}

func dropFields(obj, fields ref.Val) ref.Val {
	switch fields := fields.(type) {
	case types.String:
		return dropFieldPath(obj, fields)
	case traits.Lister:
		it := fields.Iterator()
		for it.HasNext() == types.True {
			obj = dropFieldPath(obj, it.Next().ConvertToType(types.StringType).(types.String))
		}
		return obj
	}
	return types.NewErr("invalid parameter type for drop: %v", fields.Type())
}

func dropFieldPath(arg ref.Val, path types.String) (val ref.Val) {
	defer func() {
		switch err := recover().(type) {
		case *types.Err:
			val = err
		}
	}()
	if !hasFieldPath(arg, path) {
		return arg
	}

	switch obj := arg.(type) {
	case traits.Lister:
		new := make([]ref.Val, 0, obj.Size().Value().(int64))
		it := obj.Iterator()
		for it.HasNext() == types.True {
			elem := it.Next()
			new = append(new, dropFieldPath(elem, path))
		}
		return types.NewRefValList(types.DefaultTypeAdapter, new)

	case traits.Mapper:
		dotIdx, escaped := pathSepIndex(string(path))
		switch {
		case dotIdx == 0, dotIdx == len(path)-1:
			return types.NewErr("invalid parameter path for drop: %s", path)

		case dotIdx < 0:
			new := make(map[ref.Val]ref.Val)
			m, err := obj.ConvertToNative(refValMap)
			if err != nil {
				return types.NewErr("unable to convert map to native: %v", err)
			}
			for k, v := range m.(map[ref.Val]ref.Val) {
				if k.Equal(path) == types.False {
					new[k] = v
				}
			}
			return types.NewRefValMap(types.DefaultTypeAdapter, new)

		default:
			new := make(map[ref.Val]ref.Val)
			m, err := obj.ConvertToNative(refValMap)
			if err != nil {
				return types.NewErr("unable to convert map to native: %v", err)
			}
			head := path[:dotIdx]
			if escaped {
				head = types.String(strings.ReplaceAll(string(head), `\.`, "."))
			}
			tail := path[dotIdx+1:]
			for k, v := range m.(map[ref.Val]ref.Val) {
				if k.Equal(head) == types.True {
					new[head] = dropFieldPath(v, tail)
				} else {
					new[k] = v
				}
			}
			return types.NewRefValMap(types.DefaultTypeAdapter, new)
		}

	default:
		return obj
	}
}

func hasFieldPath(arg ref.Val, path types.String) bool {
	switch obj := arg.(type) {
	case traits.Lister:
		it := obj.Iterator()
		for it.HasNext() == types.True {
			if hasFieldPath(it.Next(), path) {
				return true
			}
		}
		return false

	case traits.Mapper:
		dotIdx, escaped := pathSepIndex(string(path))
		switch {
		case dotIdx == 0, dotIdx == len(path)-1:
			panic(types.NewErr("invalid parameter path for drop: %s", path))

		case dotIdx < 0:
			m, err := obj.ConvertToNative(refValMap)
			if err != nil {
				panic(types.NewErr("unable to convert map to native: %v", err))
			}
			for k := range m.(map[ref.Val]ref.Val) {
				if k.Equal(path) == types.True {
					return true
				}
			}
			return false

		default:
			m, err := obj.ConvertToNative(refValMap)
			if err != nil {
				panic(types.NewErr("unable to convert map to native: %v", err))
			}
			head := path[:dotIdx]
			if escaped {
				head = types.String(strings.ReplaceAll(string(head), `\.`, "."))
			}
			tail := path[dotIdx+1:]
			for k, v := range m.(map[ref.Val]ref.Val) {
				if k.Equal(head) == types.True {
					return hasFieldPath(v, tail)
				}
			}
			return false
		}

	default:
		return false
	}
}

func collateFields(arg, fields ref.Val) (vals ref.Val) {
	defer func() {
		switch err := recover().(type) {
		case *types.Err:
			vals = err
		}
	}()
	switch fields := fields.(type) {
	case types.String:
		return types.NewRefValList(types.DefaultTypeAdapter, collateFieldPath(arg, fields))
	case traits.Lister:
		var elems []ref.Val
		it := fields.Iterator()
		for it.HasNext() == types.True {
			switch field := it.Next().(type) {
			case types.String:
				elems = append(elems, collateFieldPath(arg, field.ConvertToType(types.StringType).(types.String))...)
			default:
				return types.NewErr("invalid parameter type for collate fields: %v", field.Type())
			}
		}
		return types.NewRefValList(types.DefaultTypeAdapter, elems)
	}
	return types.NewErr("invalid parameter type for collate: %v", fields.Type())
}

func collateFieldPath(arg ref.Val, path types.String) []ref.Val {
	var collation []ref.Val
	switch obj := arg.(type) {
	case traits.Lister:
		it := obj.Iterator()
		for it.HasNext() == types.True {
			elem := it.Next()
			collation = append(collation, collateFieldPath(elem, path)...)
		}
		return collation

	case traits.Mapper:
		dotIdx, escaped := pathSepIndex(string(path))
		switch {
		case dotIdx == 0, dotIdx == len(path)-1:
			panic(types.NewErr("invalid parameter path for drop: %s", path))

		case dotIdx < 0:
			m, err := obj.ConvertToNative(refValMap)
			if err != nil {
				panic(types.NewErr("unable to convert map to native: %v", err))
			}
			for k, v := range m.(map[ref.Val]ref.Val) {
				if k.Equal(path) == types.True {
					switch v := v.(type) {
					case traits.Lister:
						it := v.Iterator()
						for it.HasNext() == types.True {
							collation = append(collation, it.Next())
						}
					default:
						collation = append(collation, v)
					}
				}
			}

		default:
			m, err := obj.ConvertToNative(refValMap)
			if err != nil {
				panic(types.NewErr("unable to convert map to native: %v", err))
			}
			head := path[:dotIdx]
			if escaped {
				head = types.String(strings.ReplaceAll(string(head), `\.`, "."))
			}
			tail := path[dotIdx+1:]
			for k, v := range m.(map[ref.Val]ref.Val) {
				if k.Equal(head) == types.True {
					collation = append(collation, collateFieldPath(v, tail)...)
				}
			}
		}

	default:
		if path == "" {
			collation = []ref.Val{obj}
		}
	}

	return collation
}

func sumList(arg ref.Val) ref.Val {
	list, ok := arg.(traits.Lister)
	if !ok {
		return types.NoSuchOverloadErr()
	}
	if list.Size() == types.IntZero {
		return types.NewErr("no sum of empty list")
	}

	var (
		iSum             int
		fSum             float64
		hasInt, hasFloat bool
	)
	it := list.Iterator()
	for i := 0; it.HasNext() == types.True; i++ {
		elem := it.Next()
		switch elem := elem.(type) {
		case types.Double:
			if hasInt {
				return types.NewErr("no sum of mixed int and double: first mismatch at index %d", i)
			}
			hasFloat = true
			fSum += float64(elem)
		case types.Int:
			if hasFloat {
				return types.NewErr("no sum of mixed int and double: first mismatch at index %d", i)
			}
			hasInt = true
			iSum += int(elem)
		default:
			return types.NewErr("no sum of list containing %T", elem)
		}
	}
	if hasInt {
		return types.Int(iSum)
	}
	return types.Double(fSum)
}

type comparer interface {
	ref.Val
	traits.Comparer
}

func minDiadic(arg0, arg1 ref.Val) ref.Val {
	return compareDiadic(arg0, arg1, -1)
}

func maxDiadic(arg0, arg1 ref.Val) ref.Val {
	return compareDiadic(arg0, arg1, 1)
}

func compareDiadic(arg0, arg1 ref.Val, cmp types.Int) ref.Val {
	a, ok := arg0.(comparer)
	if !ok {
		return types.NoSuchOverloadErr()
	}
	b, ok := arg1.(comparer)
	if !ok {
		return types.NoSuchOverloadErr()
	}
	if a.Compare(b) == cmp {
		return a
	}
	return b
}

func minList(arg ref.Val) ref.Val {
	return compare(arg, -1)
}

func maxList(arg ref.Val) ref.Val {
	return compare(arg, 1)
}

func compare(arg ref.Val, cmp types.Int) ref.Val {
	list, ok := arg.(traits.Lister)
	if !ok {
		return types.NoSuchOverloadErr()
	}
	if list.Size() == types.IntZero {
		return types.NewErr("no extremum of empty list")
	}

	var min comparer
	it := list.Iterator()
	for it.HasNext() == types.True {
		elem, ok := it.Next().(comparer)
		if !ok {
			return types.NoSuchOverloadErr()
		}
		if min == nil || elem.Compare(min) == cmp {
			min = elem
		}
	}
	return min
}

func zipLists(arg0, arg1 ref.Val) ref.Val {
	keys, ok := arg0.(traits.Lister)
	if !ok {
		return types.NoSuchOverloadErr()
	}
	vals, ok := arg1.(traits.Lister)
	if !ok {
		return types.NoSuchOverloadErr()
	}
	if keys.Size() != vals.Size() {
		return types.NewErr("zip: size(keys) != size(vals): %d != %d", keys.Size(), vals.Size())
	}
	n, _ := keys.Size().(types.Int)
	m := make(map[ref.Val]ref.Val, n)
	for i := types.Int(0); i < n; i++ {
		m[keys.Get(i)] = vals.Get(i)
	}
	return types.NewRefValMap(types.DefaultTypeAdapter, m)
}

func mapKeys(val ref.Val) ref.Val {
	mapK, ok := val.(traits.Mapper)
	if !ok {
		return types.ValOrErr(mapK, "no such overload")
	}
	n, ok := mapK.Size().(types.Int)
	if !ok {
		return types.NewErr("unable to get size of map")
	}
	keys := make([]ref.Val, 0, n)
	if mapK.Size() != types.IntZero {
		canSort := true
		it := mapK.Iterator()
		for it.HasNext() == types.True {
			k := it.Next()
			keys = append(keys, k)
			_, ok := k.(traits.Comparer)
			if !ok {
				canSort = false
			}
		}
		if canSort {
			sort.Slice(keys, func(i, j int) bool {
				return keys[i].(traits.Comparer).Compare(keys[j]) == types.Int(-1)
			})
		}
	}
	return types.NewRefValList(types.DefaultTypeAdapter, keys)
}

func mapValues(val ref.Val) ref.Val {
	mapK, ok := val.(traits.Mapper)
	if !ok {
		return types.ValOrErr(mapK, "no such overload")
	}
	n, ok := mapK.Size().(types.Int)
	if !ok {
		return types.NewErr("unable to get size of map")
	}
	values := make([]ref.Val, 0, n)
	type valComparer interface {
		ref.Val
		traits.Comparer
	}
	type kv struct {
		Key   valComparer
		Value ref.Val
	}
	if mapK.Size() != types.IntZero {
		canSort := true
		it := mapK.Iterator()
		ss := make([]kv, 0, n)
		for it.HasNext() == types.True {
			k := it.Next()
			v := mapK.Get(k)
			ck, ok := k.(valComparer)
			if !ok {
				canSort = false
			}
			ss = append(ss, kv{ck, v})
		}
		if canSort {
			sort.Slice(ss, func(i, j int) bool {
				return ss[i].Key.Compare(ss[j].Key) == types.Int(-1)
			})
		}
		for _, kv := range ss {
			values = append(values, kv.Value)
		}
	}
	return types.NewRefValList(types.DefaultTypeAdapter, values)
}

func makeAs(mef cel.MacroExprFactory, target ast.Expr, args []ast.Expr) (ast.Expr, *cel.Error) {
	init := target
	ident := args[0]
	expr := args[1]
	if ident.Kind() != ast.IdentKind {
		return nil, &common.Error{Message: "argument is not an identifier"}
	}
	label := ident.AsIdent()

	const unused = "_"
	return mef.NewComprehension(
		mef.NewList(),
		unused,
		label,
		init,
		mef.NewLiteral(types.False),
		mef.NewIdent(label),
		expr,
	), nil
}

// pathSepIndex returns the offset to a non-escaped dot path separator and
// whether the path element before the separator contains a backslash-escaped
// path separator.
func pathSepIndex(s string) (off int, escaped bool) {
	for {
		idx := strings.IndexByte(s[off:], '.')
		if idx == -1 {
			return -1, escaped
		}
		off += idx
		if idx == 0 || s[off-1] != '\\' {
			return off, escaped
		}
		off++
		escaped = true
	}
}