in packages/datastream.go [307:345]
func (d *DataStream) validateRequiredFields(fs PackageFileSystem) error {
fieldsDirPath := path.Join(d.BasePath, "fields")
// Collect fields from all files
fieldsFiles, err := fs.Glob(path.Join(fieldsDirPath, "*"))
if err != nil {
return err
}
var allFields []util.MapStr
for _, path := range fieldsFiles {
body, err := ReadAll(fs, path)
if err != nil {
return fmt.Errorf("reading file failed (path: %s): %w", path, err)
}
var m []util.MapStr
err = yamlv2.Unmarshal(body, &m)
if err != nil {
return fmt.Errorf("unmarshaling file failed (path: %s): %w", path, err)
}
allFields = append(allFields, m...)
}
if err != nil {
return fmt.Errorf("walking through fields files failed: %w", err)
}
// Flatten all fields
for i, fields := range allFields {
allFields[i] = fields.Flatten()
}
// Verify required keys
err = requireField(allFields, "data_stream.type", "constant_keyword", err)
err = requireField(allFields, "data_stream.dataset", "constant_keyword", err)
err = requireField(allFields, "data_stream.namespace", "constant_keyword", err)
err = requireField(allFields, "@timestamp", "date", err)
return err
}