aucoalesce/coalesce.go (681 lines of code) (raw):

// Licensed to Elasticsearch B.V. under one or more contributor // license agreements. See the NOTICE file distributed with // this work for additional information regarding copyright // ownership. Elasticsearch B.V. licenses this file to you under // the Apache License, Version 2.0 (the "License"); you may // not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, // software distributed under the License is distributed on an // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. // Package aucoalesce provides functions to coalesce compound audit messages // into a single event and normalize all message types with some common fields. package aucoalesce import ( "errors" "fmt" "os" "strconv" "strings" "time" "github.com/elastic/go-libaudit/v2/auparse" ) // modeBlockDevice is the file mode bit representing block devices. This OS // package does not have a constant defined for this. const modeBlockDevice = 0o60000 // ECSEvent contains ECS-specific categorization fields type ECSEvent struct { Kind string `json:"kind,omitempty" yaml:"kind,omitempty"` Category []string `json:"category,omitempty" yaml:"category,omitempty"` Type []string `json:"type,omitempty" yaml:"type,omitempty"` Outcome string `json:"outcome,omitempty" yaml:"outcome,omitempty"` } type ECSEntityData struct { Name string `json:"name,omitempty" yaml:"name,omitempty"` ID string `json:"id,omitempty" yaml:"id,omitempty"` } type ECSEntity struct { ECSEntityData `yaml:",inline"` Effective ECSEntityData `json:"effective" yaml:"effective"` Target ECSEntityData `json:"target" yaml:"target"` Changes ECSEntityData `json:"changes" yaml:"changes"` } type ECSFields struct { Event ECSEvent `json:"event" yaml:"event"` User ECSEntity `json:"user" yaml:"user"` Group ECSEntityData `json:"group" yaml:"group"` } type Event struct { Timestamp time.Time `json:"@timestamp" yaml:"timestamp"` Sequence uint32 `json:"sequence" yaml:"sequence"` Category AuditEventType `json:"category" yaml:"category"` Type auparse.AuditMessageType `json:"record_type" yaml:"record_type"` Result string `json:"result,omitempty" yaml:"result,omitempty"` Session string `json:"session" yaml:"session"` Tags []string `json:"tags,omitempty" yaml:"tags,omitempty"` Summary Summary `json:"summary" yaml:"summary"` User User `json:"user" yaml:"user"` Process Process `json:"process,omitempty" yaml:"process,omitempty"` File *File `json:"file,omitempty" yaml:"file,omitempty"` Source *Address `json:"source,omitempty" yaml:"source,omitempty"` Dest *Address `json:"destination,omitempty" yaml:"destination,omitempty"` Net *Network `json:"network,omitempty" yaml:"network,omitempty"` Data map[string]string `json:"data,omitempty" yaml:"data,omitempty"` Paths []map[string]string `json:"paths,omitempty" yaml:"paths,omitempty"` ECS ECSFields `json:"ecs" yaml:"ecs"` Warnings []error `json:"-" yaml:"-"` } type Summary struct { Actor Actor `json:"actor" yaml:"actor"` Action string `json:"action,omitempty" yaml:"action,omitempty"` Object Object `json:"object,omitempty" yaml:"object,omitempty"` How string `json:"how,omitempty" yaml:"how,omitempty"` } type Actor struct { Primary string `json:"primary,omitempty" yaml:"primary,omitempty"` Secondary string `json:"secondary,omitempty" yaml:"secondary,omitempty"` } type Process struct { PID string `json:"pid,omitempty" yaml:"pid,omitempty"` PPID string `json:"ppid,omitempty" yaml:"ppid,omitempty"` Title string `json:"title,omitempty" yaml:"title,omitempty"` Name string `json:"name,omitempty" yaml:"name,omitempty"` // Comm Exe string `json:"exe,omitempty" yaml:"exe,omitempty"` CWD string `json:"cwd,omitempty" yaml:"cwd,omitempty"` Args []string `json:"args,omitempty" yaml:"args,omitempty"` } func (p Process) IsEmpty() bool { return p.PID == "" && p.PPID == "" && p.Title == "" && p.Name == "" && p.Exe == "" && p.CWD == "" && len(p.Args) == 0 } type User struct { IDs map[string]string `json:"ids,omitempty" yaml:"ids,omitempty"` // Identifying data like auid, uid, euid, suid, fsuid, gid, egid, sgid, fsgid. Names map[string]string `json:"names,omitempty" yaml:"names,omitempty"` // Mappings of ID to name (auid -> "root"). SELinux map[string]string `json:"selinux,omitempty" yaml:"selinux,omitempty"` // SELinux labels. } type File struct { Path string `json:"path,omitempty" yaml:"path,omitempty"` Device string `json:"device,omitempty" yaml:"device,omitempty"` Inode string `json:"inode,omitempty" yaml:"inode,omitempty"` Mode string `json:"mode,omitempty" yaml:"mode,omitempty"` // Permissions UID string `json:"uid,omitempty" yaml:"uid,omitempty"` GID string `json:"gid,omitempty" yaml:"gid,omitempty"` Owner string `json:"owner,omitempty" yaml:"owner,omitempty"` Group string `json:"group,omitempty" yaml:"group,omitempty"` SELinux map[string]string `json:"selinux,omitempty" yaml:"selinux,omitempty"` // SELinux labels. } type Direction uint8 const ( IncomingDir Direction = iota + 1 OutgoingDir ) func (d Direction) String() string { switch d { case IncomingDir: return "ingress" case OutgoingDir: return "egress" } return "unknown" } func (d Direction) MarshalText() ([]byte, error) { return []byte(d.String()), nil } type Network struct { Direction Direction `json:"direction" yaml:"direction"` } type Address struct { Hostname string `json:"hostname,omitempty" yaml:"hostname,omitempty"` // Hostname. IP string `json:"ip,omitempty" yaml:"ip,omitempty"` // IPv4 or IPv6 address. Port string `json:"port,omitempty" yaml:"port,omitempty"` // Port number. Path string `json:"path,omitempty" yaml:"path,omitempty"` // Unix socket path. } type Object struct { Type string `json:"type,omitempty" yaml:"type,omitempty"` Primary string `json:"primary,omitempty" yaml:"primary,omitempty"` Secondary string `json:"secondary,omitempty" yaml:"secondary,omitempty"` } // CoalesceMessages combines the given messages into a single event. It assumes // that all the messages in the slice have the same timestamp and sequence // number. An error is returned is msgs is empty or nil or only contains and EOE // (end-of-event) message. func CoalesceMessages(msgs []*auparse.AuditMessage) (*Event, error) { msgs = filterEOE(msgs) var event *Event var err error switch len(msgs) { case 0: return nil, errors.New("messages is empty") case 1: event, err = normalizeSimple(msgs[0]) default: event, err = normalizeCompound(msgs) } if event != nil { applyNormalization(event) addProcess(event) } return event, err } // filterEOE returns a slice (backed by the given msgs slice) that does not // contain EOE (end-of-event) messages. EOE messages are sentinel messages used // to signal the completion of an event, but they carry no data. func filterEOE(msgs []*auparse.AuditMessage) []*auparse.AuditMessage { if len(msgs) > 0 && msgs[len(msgs)-1].RecordType == auparse.AUDIT_EOE { return msgs[:len(msgs)-1] } return msgs } func normalizeSimple(msg *auparse.AuditMessage) (*Event, error) { return newEvent(msg, nil), nil } func normalizeCompound(msgs []*auparse.AuditMessage) (*Event, error) { var special, syscall *auparse.AuditMessage for i, msg := range msgs { if i == 0 && msg.RecordType != auparse.AUDIT_SYSCALL { special = msg continue } if msg.RecordType == auparse.AUDIT_SYSCALL { syscall = msg break } } if syscall == nil { // All compound records have syscall messages. return nil, errors.New("missing syscall message in compound event") } event := newEvent(special, syscall) for _, msg := range msgs { switch msg.RecordType { case auparse.AUDIT_SYSCALL: delete(event.Data, "items") case auparse.AUDIT_PATH: addPathRecord(msg, event) case auparse.AUDIT_SOCKADDR: addSockaddrRecord(msg, event) case auparse.AUDIT_EXECVE: addExecveRecord(msg, event) default: addFieldsToEventData(msg, event) } } return event, nil } func newEvent(msg, syscall *auparse.AuditMessage) *Event { if msg == nil { msg = syscall } event := &Event{ Timestamp: msg.Timestamp, Sequence: msg.Sequence, Category: GetAuditEventType(msg.RecordType), Type: msg.RecordType, Data: make(map[string]string, 10), } if syscall != nil { msg = syscall } data, err := msg.Data() if err != nil { event.Warnings = append(event.Warnings, err) return event } if result, found := data["result"]; found { event.Result = result delete(data, "result") } else { event.Result = "unknown" } if ses, found := data["ses"]; found { event.Session = ses delete(data, "ses") } if auid, found := data["auid"]; found { event.Summary.Actor.Primary = auid } if uid, found := data["uid"]; found { event.Summary.Actor.Secondary = uid } // Ignore error because msg.Data() would have produced the same error. event.Tags, _ = msg.Tags() for k, v := range data { if strings.HasSuffix(k, "uid") || strings.HasSuffix(k, "gid") { addSubjectAttribute(k, v, event) } else if strings.HasPrefix(k, "subj_") { addSubjectSELinuxLabel(k[5:], v, event) } else { event.Data[k] = v } } return event } func addSubjectAttribute(key, value string, event *Event) { if event.User.IDs == nil { event.User.IDs = map[string]string{} } event.User.IDs[key] = value } func addSubjectSELinuxLabel(key, value string, event *Event) { if event.User.SELinux == nil { event.User.SELinux = map[string]string{} } event.User.SELinux[key] = value } func addSockaddrRecord(sockaddr *auparse.AuditMessage, event *Event) { data, err := sockaddr.Data() if err != nil { event.Warnings = append(event.Warnings, fmt.Errorf( "failed to parse SOCKADDR message: %w", err)) return } syscall, found := event.Data["syscall"] if !found { event.Warnings = append(event.Warnings, errors.New( "failed to add SOCKADDR data because syscall is unknown")) return } for k, v := range data { event.Data["socket_"+k] = v } switch syscall { case "recvfrom", "recvmsg", "accept", "accept4": addAddress(data, &event.Source) event.Net = &Network{Direction: IncomingDir} case "connect", "sendto", "sendmsg": addAddress(data, &event.Dest) event.Net = &Network{Direction: OutgoingDir} default: // These are the other syscalls that contain SOCKADDR, but they // have no clear source or destination: // bind, listen, getpeername, getsockname return } } func addAddress(sockaddr map[string]string, addr **Address) { var ( ip = sockaddr["addr"] port = sockaddr["port"] path = sockaddr["path"] ) if ip != "" || port != "" || path != "" { *addr = &Address{ IP: ip, Port: port, Path: path, } } } func addPathRecord(path *auparse.AuditMessage, event *Event) { data, err := path.Data() if err != nil { event.Warnings = append(event.Warnings, fmt.Errorf( "failed to parse PATH message: %w", err)) return } event.Paths = append(event.Paths, data) } func addProcess(event *Event) { event.Process.PID = event.Data["pid"] delete(event.Data, "pid") event.Process.PPID = event.Data["ppid"] delete(event.Data, "ppid") event.Process.Title = event.Data["proctitle"] delete(event.Data, "proctitle") event.Process.Name = event.Data["comm"] delete(event.Data, "comm") event.Process.Exe = event.Data["exe"] delete(event.Data, "exe") event.Process.CWD = event.Data["cwd"] delete(event.Data, "cwd") } func addExecveRecord(execve *auparse.AuditMessage, event *Event) { data, err := execve.Data() if err != nil { event.Warnings = append(event.Warnings, fmt.Errorf( "failed to parse EXECVE message: %w", err)) return } argc, found := data["argc"] if !found { event.Warnings = append(event.Warnings, errors.New("argc key not found in EXECVE message")) return } event.Data["argc"] = argc count, err := strconv.ParseUint(argc, 10, 32) if err != nil { event.Warnings = append(event.Warnings, fmt.Errorf( "failed to convert argc='%v' to number: %w", argc, err)) return } var args []string for i := 0; i < int(count); i++ { key := "a" + strconv.Itoa(i) arg, found := data[key] if !found { event.Warnings = append(event.Warnings, fmt.Errorf( "failed to find arg %v", key)) return } delete(data, key) args = append(args, arg) } event.Process.Args = args } func addFieldsToEventData(msg *auparse.AuditMessage, event *Event) { data, err := msg.Data() if err != nil { event.Warnings = append(event.Warnings, fmt.Errorf("failed to parse message: %w", err)) return } for k, v := range data { if _, found := event.Data[k]; found { event.Warnings = append(event.Warnings, fmt.Errorf( "duplicate key (%v) from %v message", k, msg.RecordType)) continue } event.Data[k] = v } } func applyNormalization(event *Event) { setHowDefaults(event) var syscallNorm *Normalization if syscall, ok := event.Data["syscall"]; ok { syscallNorm, ok = syscallNorms[syscall] if !ok { syscallNorm = syscallNorms["*"] // get the default to add some basic categorization } } var norm *Normalization if event.Type == auparse.AUDIT_SYSCALL { norm = syscallNorm } else { norms := recordTypeNorms[event.Type.String()] switch len(norms) { case 0: // No normalization found. case 1: norm = norms[0] default: nextNorm: for _, n := range norms { // Select normalization if all 'has_fields' are present. for _, f := range n.HasFields.Values { if _, found := event.Data[f]; !found { continue nextNorm } } norm = n } } } if norm == nil { event.Warnings = append(event.Warnings, errors.New("no normalization found for event")) return } event.ECS.Event.Kind = norm.ECS.Kind event.ECS.Event.Category = norm.ECS.Category.Values event.ECS.Event.Type = norm.ECS.Type.Values // we check to see if the non-AUDIT_SYSCALL event has an associated syscall // from another part of the auditd message log, if it does and we have normalizations // for that syscall, we merge the ECS categorization and type information so that // the event has both enrichment for the record type itself and for the syscall it // captures hasAdditionalNormalization := syscallNorm != nil && syscallNorm != norm if hasAdditionalNormalization { event.ECS.Event.Category = append(event.ECS.Event.Category, syscallNorm.ECS.Category.Values...) event.ECS.Event.Type = append(event.ECS.Event.Type, syscallNorm.ECS.Type.Values...) if event.Result == "fail" { event.ECS.Event.Outcome = "failure" } } event.Summary.Action = norm.Action switch norm.ObjectWhat { case "file", "filesystem": event.Summary.Object.Type = norm.ObjectWhat if len(event.Paths) > 0 { if err := setFileObject(event, norm.ObjectPathIndex); err != nil { event.Warnings = append(event.Warnings, fmt.Errorf("failed to set file object: %w", err)) } } case "socket": event.Summary.Object.Type = norm.ObjectWhat setSocketObject(event) default: event.Summary.Object.Type = norm.ObjectWhat } if len(norm.SubjectPrimaryFieldName.Values) > 0 { var err error for _, subjKey := range norm.SubjectPrimaryFieldName.Values { if err = setSubjectPrimary(subjKey, event); err == nil { break } } if err != nil { event.Warnings = append(event.Warnings, fmt.Errorf("failed to set subject primary using keys=%v because they were not found", norm.SubjectPrimaryFieldName.Values)) } } if len(norm.SubjectSecondaryFieldName.Values) > 0 { var err error for _, subjKey := range norm.SubjectSecondaryFieldName.Values { if err = setSubjectSecondary(subjKey, event); err == nil { break } } if err != nil { event.Warnings = append(event.Warnings, fmt.Errorf("failed to set subject secondary using keys=%v because they were not found", norm.SubjectSecondaryFieldName.Values)) } } if len(norm.ObjectPrimaryFieldName.Values) > 0 { var err error for _, objKey := range norm.ObjectPrimaryFieldName.Values { if err = setObjectPrimary(objKey, event); err == nil { break } } if err != nil { event.Warnings = append(event.Warnings, fmt.Errorf("failed to set object primary using keys=%v because they were not found", norm.ObjectPrimaryFieldName.Values)) } } if len(norm.ObjectSecondaryFieldName.Values) > 0 { var err error for _, objKey := range norm.ObjectSecondaryFieldName.Values { if err = setObjectSecondary(objKey, event); err == nil { break } } if err != nil { event.Warnings = append(event.Warnings, fmt.Errorf("failed to set object secondary using keys=%v because they were not found", norm.ObjectSecondaryFieldName.Values)) } } if len(norm.How.Values) > 0 { var err error for _, howKey := range norm.How.Values { if err = setHow(howKey, event); err == nil { break } } if err != nil { event.Warnings = append(event.Warnings, fmt.Errorf("failed to set how using keys=%v because they were not found", norm.How.Values)) } } if event.Source == nil && len(norm.SourceIP.Values) > 0 { var err error for _, sourceIPKey := range norm.SourceIP.Values { if err = setSourceIP(sourceIPKey, event); err == nil { break } } if err != nil { event.Warnings = append(event.Warnings, fmt.Errorf("failed to "+ "set source IP using keys=%v because they were not found", norm.SourceIP.Values)) } } // Populate ECS fields from `mappings` section. for _, mapping := range norm.ECS.Mappings { if mapping.To != nil && mapping.From != nil { mapping.To(event, mapping.From(event)) } } } func getValue(key string, event *Event) (string, bool) { value, found := event.Data[key] if !found { // Fallback to user IDs. value, found = event.User.IDs[key] } return value, found } func setSourceIP(key string, event *Event) error { value, found := event.Data[key] if !found { return fmt.Errorf("failed to set source IP value: key '%v' not found", key) } delete(event.Data, key) event.Source = &Address{ IP: value, } event.Net = &Network{ Direction: IncomingDir, } return nil } func setHow(key string, event *Event) error { value, found := getValue(key, event) if !found { return fmt.Errorf("failed to set how value: key '%v' not found", key) } event.Summary.How = value return nil } func setSubjectPrimary(key string, event *Event) error { value, found := getValue(key, event) if !found { return fmt.Errorf("failed to set subject primary value: key '%v' not found", key) } event.Summary.Actor.Primary = value return nil } func setSubjectSecondary(key string, event *Event) error { value, found := getValue(key, event) if !found { return fmt.Errorf("failed to set subject secondary value: key '%v' not found", key) } event.Summary.Actor.Secondary = value return nil } func setObjectPrimary(key string, event *Event) error { value, found := getValue(key, event) if !found { return fmt.Errorf("failed to set object primary value: key '%v' not found", key) } event.Summary.Object.Primary = value return nil } func setObjectSecondary(key string, event *Event) error { value, found := getValue(key, event) if !found { return fmt.Errorf("failed to set object secondary value: key '%v' not found", key) } event.Summary.Object.Secondary = value return nil } func setFileObject(event *Event, pathIndexHint int) error { if len(event.Paths) == 0 { return errors.New("path message not found") } var pathIndex int if len(event.Paths) > pathIndexHint { pathIndex = pathIndexHint } path := event.Paths[pathIndex] for _, p := range event.Paths[pathIndex:] { // Skip over PARENT and UNKNOWN types in case the path index was wrong. if nametype := p["nametype"]; nametype != "PARENT" && nametype != "UNKNOWN" { path = p break } } event.File = &File{} if value, found := path["name"]; found { event.Summary.Object.Primary = value event.File.Path = value } if value, found := path["inode"]; found { event.File.Inode = value } if value, found := path["rdev"]; found { event.File.Device = value } if value, found := path["mode"]; found { mode, err := strconv.ParseUint(value, 8, 64) if err != nil { return fmt.Errorf("failed to parse file mode: %w", err) } m := os.FileMode(mode) event.File.Mode = fmt.Sprintf("%04o", 0o7777&m) switch { case m.IsRegular(): event.Summary.Object.Type = "file" case m.IsDir(): event.Summary.Object.Type = "directory" case m&os.ModeCharDevice != 0: event.Summary.Object.Type = "character-device" case m&modeBlockDevice != 0: event.Summary.Object.Type = "block-device" case m&os.ModeNamedPipe != 0: event.Summary.Object.Type = "named-pipe" case m&os.ModeSymlink != 0: event.Summary.Object.Type = "symlink" case m&os.ModeSocket != 0: event.Summary.Object.Type = "socket" } } if value, found := path["ouid"]; found { event.File.UID = value } if value, found := path["ogid"]; found { event.File.GID = value } for k, v := range path { if strings.HasPrefix(k, "obj_") { addFileSELinuxLabel(k[4:], v, event) } } return nil } func addFileSELinuxLabel(key, value string, event *Event) { if event.File.SELinux == nil { event.File.SELinux = map[string]string{} } event.File.SELinux[key] = value } func setSocketObject(event *Event) { value, found := event.Data["socket_addr"] if found { event.Summary.Object.Primary = value } else { value, found = event.Data["socket_path"] if found { event.Summary.Object.Primary = value } } value, found = event.Data["socket_port"] if found { event.Summary.Object.Secondary = value } } func setHowDefaults(event *Event) { exe, found := event.Data["exe"] if !found { // Fallback to comm. exe, found = event.Data["comm"] if !found { return } } event.Summary.How = exe switch { case strings.HasPrefix(exe, "/usr/bin/python"), strings.HasPrefix(exe, "/usr/bin/sh"), strings.HasPrefix(exe, "/usr/bin/bash"), strings.HasPrefix(exe, "/usr/bin/perl"): default: return } // It's probably some kind of interpreted script so use "comm". comm, found := event.Data["comm"] if !found { return } event.Summary.How = comm } func (e *ECSEntityData) set(value string) { if value == "" || value == "unset" || value == "4294967295" || value == "-1" { *e = ECSEntityData{ID: "unset"} return } // This could be called using an UID or a name if _, err := strconv.ParseUint(value, 10, 64); err == nil { e.ID = value } else { e.Name = value } } func (e *ECSEntityData) lookup(cache *EntityCache) { if (e.ID == "") == (e.Name == "") { return } if e.ID != "" { e.Name = cache.LookupID(e.ID) } else { e.ID = cache.LookupName(e.Name) } } func (e *ECSEntity) lookup(cache *EntityCache) { e.ECSEntityData.lookup(cache) e.Effective.lookup(cache) e.Target.lookup(cache) e.Changes.lookup(cache) }