in x-pack/filebeat/module/o365/audit/config/pipeline.js [743:1086]
function AuditProcessor(tenant_names, debug) {
var builder = new PipelineBuilder("o365.audit", debug);
var unsetIPValues = {"null": true, "<null>": true, "": true};
builder.Add("cleanupNulls", function(event) {
[
"o365audit.ClientIP",
"o365audit.ClientIPAddress",
"o365audit.ActorIpAddress",
"o365audit.OriginatingServer"
].forEach(function(field) {
if (event.Get(field) in unsetIPValues) event.Delete(field);
});
});
builder.Add("convertCommonAuditRecordFields", new processor.Convert({
fields: [
{from: "o365audit.Id", to: "event.id"},
{from: "o365audit.ClientIP", to: "client.address"},
{from: "o365audit.ClientIPAddress", to: "client.address"},
{from: "o365audit.ActorIpAddress", to: "client.address"},
{from: "o365audit.UserId", to: "user.id", type: "string"},
{from: "o365audit.Workload", to: "event.provider"},
{from: "o365audit.Operation", to: "event.action"},
{from: "o365audit.OrganizationId", to: "organization.id"},
// Extra common fields:
{from: "o365audit.UserAgent", to: "user_agent.original"},
],
ignore_missing: true,
fail_on_error: false
}));
builder.Add("mapEventType", makeMapper({
from: 'o365audit.RecordType',
to: 'event.code',
// Keep original RecordType for unknown mappings.
default: function(recordType) {
return recordType;
},
mappings: {
1: 'ExchangeAdmin', // Events from the Exchange admin audit log.
2: 'ExchangeItem', // Events from an Exchange mailbox audit log for actions that are performed on a single item, such as creating or receiving an email message.
3: 'ExchangeItemGroup', // Events from an Exchange mailbox audit log for actions that can be performed on multiple items, such as moving or deleted one or more email messages.
4: 'SharePoint', // SharePoint events.
6: 'SharePointFileOperation', // SharePoint file operation events.
8: 'AzureActiveDirectory', // Azure Active Directory events.
7: 'OneDrive', // OneDrive for Business events.
9: 'AzureActiveDirectoryAccountLogon', // Azure Active Directory OrgId logon events (deprecating).
10: 'DataCenterSecurityCmdlet', // Data Center security cmdlet events.
11: 'ComplianceDLPSharePoint', // Data loss protection (DLP) events in SharePoint and OneDrive for Business.
12: 'Sway', // Events from the Sway service and clients.
13: 'ComplianceDLPExchange', // Data loss protection (DLP) events in Exchange, when configured via Unified DLP Policy. DLP events based on Exchange Transport Rules are not supported.
14: 'SharePointSharingOperation', // SharePoint sharing events.
15: 'AzureActiveDirectoryStsLogon', // Secure Token Service (STS) logon events in Azure Active Directory.
16: 'SkypeForBusinessPSTNUsage', // Public Switched Telephone Network (PSTN) events from Skype for Business.
17: 'SkypeForBusinessUsersBlocked', // Blocked user events from Skype for Business.
18: 'SecurityComplianceCenterEOPCmdlet', // Admin actions from the Security & Compliance Center.
19: 'ExchangeAggregatedOperation', // Aggregated Exchange mailbox auditing events.
20: 'PowerBIAudit', // Power BI events.
21: 'CRM', // Microsoft CRM events.
22: 'Yammer', // Yammer events.
23: 'SkypeForBusinessCmdlets', // Skype for Business events.
24: 'Discovery', // Events for eDiscovery activities performed by running content searches and managing eDiscovery cases in the Security & Compliance Center.
25: 'MicrosoftTeams', // Events from Microsoft Teams.
28: 'ThreatIntelligence', // Phishing and malware events from Exchange Online Protection and Office 365 Advanced Threat Protection.
29: 'MailSubmission', // Submission events from Exchange Online Protection and Microsoft Defender for Office 365.
30: 'MicrosoftFlow', // Microsoft Power Automate (formerly called Microsoft Flow) events.
31: 'AeD', // Advanced eDiscovery events.
32: 'MicrosoftStream', // Microsoft Stream events.
33: 'ComplianceDLPSharePointClassification', // Events related to DLP classification in SharePoint.
34: 'ThreatFinder', // Campaign-related events from Microsoft Defender for Office 365.
35: 'Project', // Microsoft Project events.
36: 'SharePointListOperation', // SharePoint List events.
37: 'SharePointCommentOperation', // SharePoint comment events.
38: 'DataGovernance', // Events related to retention policies and retention labels in the Security & Compliance Center
39: 'Kaizala', // Kaizala events.
40: 'SecurityComplianceAlerts', // Security and compliance alert signals.
41: 'ThreatIntelligenceUrl', // Safe links time-of-block and block override events from Office 365 Advanced Threat Protection.
42: 'SecurityComplianceInsights', // Events related to insights and reports in the Office 365 security and compliance center.
43: 'MIPLabel', // Events related to the detection in the Transport pipeline of email messages that have been tagged (manually or automatically) with sensitivity labels.
44: 'WorkplaceAnalytics', // Workplace Analytics events.
45: 'PowerAppsApp', // Power Apps events.
46: 'PowerAppsPlan', // Subscription plan events for Power Apps.
47: 'ThreatIntelligenceAtpContent', // Phishing and malware events for files in SharePoint, OneDrive for Business, and Microsoft Teams from Office 365 Advanced Threat Protection.
48: 'LabelContentExplorer', // Events related to data classification content explorer.
49: 'TeamsHealthcare', // Events related to the Patients application in Microsoft Teams for Healthcare.
50: 'ExchangeItemAggregated', // Events related to the MailItemsAccessed mailbox auditing action.
51: 'HygieneEvent', // Events related to outbound spam protection.
52: 'DataInsightsRestApiAudit', // Data Insights REST API events.
53: 'InformationBarrierPolicyApplication', // Events related to the application of information barrier policies.
54: 'SharePointListItemOperation', // SharePoint list item events.
55: 'SharePointContentTypeOperation', // SharePoint list content type events.
56: 'SharePointFieldOperation', // SharePoint list field events.
57: 'MicrosoftTeamsAdmin', // Teams admin events.
58: 'HRSignal', // Events related to HR data signals that support the Insider risk management solution.
59: 'MicrosoftTeamsDevice', // Teams device events.
60: 'MicrosoftTeamsAnalytics', // Teams analytics events.
61: 'InformationWorkerProtection', // Events related to compromised user alerts.
62: 'Campaign', // Email campaign events from Microsoft Defender for Office 365.
63: 'DLPEndpoint', // Endpoint DLP events.
64: 'AirInvestigation', // Automated incident response (AIR) events.
65: 'Quarantine', // Quarantine events.
66: 'MicrosoftForms', // Microsoft Forms events.
67: 'ApplicationAudit', // Application audit events.
68: 'ComplianceSupervisionExchange', // Events tracked by the Communication compliance offensive language model.
69: 'CustomerKeyServiceEncryption', // Events related to the customer key encryption service.
70: 'OfficeNative', // Events related to sensitivity labels applied to Office documents.
71: 'MipAutoLabelSharePointItem', // Auto-labeling events in SharePoint.
72: 'MipAutoLabelSharePointPolicyLocation', // Auto-labeling policy events in SharePoint.
73: 'MicrosoftTeamsShifts', // Teams Shifts events.
75: 'MipAutoLabelExchangeItem', // Auto-labeling events in Exchange.
76: 'CortanaBriefing', // Briefing email events.
78: 'WDATPAlerts', // Events related to alerts generated by Windows Defender for Endpoint.
82: 'SensitivityLabelPolicyMatch', // Events generated when the file labeled with a sensitivity label is opened or renamed.
83: 'SensitivityLabelAction', // Event generated when sensitivity labels are applied, updated, or removed from a file.
84: 'SensitivityLabeledFileAction', // Events generated when a file labeled with a sensitivity label is opened or renamed.
85: 'AttackSim', // Attack simulator events.
86: 'AirManualInvestigation', // Events related to manual investigations in Automated investigation and response (AIR).
87: 'SecurityComplianceRBAC', // Security and compliance RBAC events.
88: 'UserTraining', // Attack simulator training events in Microsoft Defender for Office 365.
89: 'AirAdminActionInvestigation', // Events related to admin actions in Automated investigation and response (AIR).
90: 'MSTIC', // Threat intelligence events in Microsoft Defender for Office 365.
91: 'PhysicalBadgingSignal', // Events related to physical badging signals that support the Insider risk management solution.
93: 'AipDiscover', // Azure Information Protection (AIP) scanner events.
94: 'AipSensitivityLabelAction', // AIP sensitivity label events.
95: 'AipProtectionAction', // AIP protection events.
96: 'AipFileDeleted', // AIP file deletion events.
97: 'AipHeartBeat', // AIP heartbeat events.
98: 'MCASAlerts', // Events corresponding to alerts triggered by Microsoft Cloud App Security.
99: 'OnPremisesFileShareScannerDlp', // Events related to scanning for sensitive data on file shares.
100: 'OnPremisesSharePointScannerDlp', // Events related to scanning for sensitive data in SharePoint.
101: 'ExchangeSearch', // Events related to using Outlook on the web (OWA) to search for mailbox items.
102: 'SharePointSearch', // Events related to searching an organization's SharePoint home site.
103: 'PrivacyInsights', // Privacy insight events.
105: 'MyAnalyticsSettings', // MyAnalytics events.
106: 'SecurityComplianceUserChange', // Events related to modifying or deleting a user.
107: 'ComplianceDLPExchangeClassification', // Exchange DLP classification events.
109: 'MipExactDataMatch', // Exact Data Match (EDM) classification events.
113: 'MS365DCustomDetection', // Events related to custom detection actions in Microsoft 365 Defender.
147: 'CoreReportingSettings', // Reports settings events.
148: 'ComplianceConnector', // Events related to importing non-Microsoft data using data connectors in the Microsoft Purview compliance portal.
174: 'DataShareOperation', // Events related to sharing of data ingested via SystemSync.
181: 'EduDataLakeDownloadOperation', // Events related to the export of SystemSync ingested data from the lake.
},
}));
builder.Add("setEventFields", new processor.AddFields({
target: 'event',
fields: {
kind: 'event',
type: 'info',
// Not so sure about web as a default category:
category: 'web',
},
}));
builder.Add("mapEventOutcome", makeMapper({
from: 'o365audit.ResultStatus',
to: 'event.outcome',
lowercase: true,
default: 'success',
mappings: {
'success': 'success', // This one is necessary to map Success
'succeeded': 'success',
'partiallysucceeded': 'success',
'true': 'success',
'failed': 'failure',
'false': 'failure',
},
}));
builder.Add("makeParametersDict", makeObjFromNameValuePairArray({
from: 'o365audit.Parameters',
to: 'o365audit.Parameters',
}));
builder.Add("makeExtendedPropertiesDict", makeObjFromNameValuePairArray({
from: 'o365audit.ExtendedProperties',
to: 'o365audit.ExtendedProperties',
}));
builder.Add("makeModifiedPropertyDict", makeDictFromModifiedPropertyArray({
from: 'o365audit.ModifiedProperties',
to: 'o365audit.ModifiedProperties',
}));
// Turn AlertLinks into an array of keyword instead of array of objects.
builder.Add("alertLinks", function (evt) {
var list = evt.Get("o365audit.AlertLinks");
if (list == null || !(list instanceof Array)) return;
var links = [];
for (var i=0; i<list.length; i++) {
var link = list[i].AlertLinkHref;
if (link != null && typeof link === "string" && link.length > 0) {
links.push(link);
}
}
switch (links.length) {
case 0:
evt.Delete('o365audit.AlertLinks');
break;
case 1:
evt.Put("o365audit.AlertLinks", links[0]);
break;
default:
evt.Put("o365audit.AlertLinks", links);
}
});
// Populate event specific fields.
var dlp = dataLossPreventionSchema(debug);
builder.Add("productSpecific", makeConditional({
condition: function(event) {
return event.Get("event.code");
},
'ExchangeAdmin': exchangeAdminSchema(debug).Run,
'ExchangeItem': exchangeMailboxSchema(debug).Run,
'AzureActiveDirectory': azureADSchema(debug).Run,
'AzureActiveDirectoryStsLogon': azureADLogonSchema(debug).Run,
'SharePointFileOperation': sharePointFileOperationSchema(debug).Run,
'SecurityComplianceAlerts': securityComplianceAlertsSchema(debug).Run,
'ComplianceDLPSharePoint': dlp.Run,
'ComplianceDLPExchange': dlp.Run,
'Yammer': yammerSchema(debug).Run,
'MicrosoftTeams': teamsSchema(debug).Run,
}));
builder.Add("extractClientIPPortBrackets", new processor.Dissect({
tokenizer: '[%{_ip}]:%{_port}',
field: 'client.address',
target_prefix: 'client',
'when.and': [
{'not.has_fields': ['client._ip', 'client._port']},
{'contains.client.address': ']:'},
],
}));
builder.Add("extractClientIPv4Port", new processor.Dissect({
tokenizer: '%{_ip}:%{_port}',
field: 'client.address',
target_prefix: 'client',
'when.and': [
{'not.has_fields': ['client._ip', 'client._port']},
{'contains.client.address': '.'},
{'contains.client.address': ':'},
// Best effort to avoid parsing IPv6-mapped IPv4 as ip:port.
// Won't succeed if IPv6 address is not shortened.
{'not.contains.client.address': '::'},
],
}));
// Copy the client/server.address to .ip fields if they are valid IPs.
builder.Add("convertIPs", new processor.Convert({
fields: [
{from: "client.address", to: "client.ip", type: "ip"},
{from: "server.address", to: "server.ip", type: "ip"},
{from: "client._ip", to: "client.ip", type: "ip"},
{from: "client._port", to: "client.port", type: "long"},
],
ignore_missing: true,
fail_on_error: false
}));
builder.Add("removeTempIP", function (evt) {
evt.Delete("client._ip");
evt.Delete("client._port");
});
builder.Add("setSrcDstFields", new processor.Convert({
fields: [
{from: "client.ip", to: "source.ip"},
{from: "client.port", to: "source.port"},
{from: "server.ip", to: "destination.ip"},
],
ignore_missing: true,
fail_on_error: false
}));
[
'user',
'user.target',
'source.user',
'destination.user',
].forEach(function (prefix) {
builder.Add('setFromID' + prefix, splitEmailUserID(prefix));
})
builder.Add("setNetworkType", function(event) {
var ip = event.Get("client.ip");
if (ip == null) return;
event.Put("network.type", ip.indexOf(".") !== -1? "ipv4" : "ipv6");
});
builder.Add("setRelatedIP", appendFields({
fields: [
"client.ip",
"server.ip",
],
to: 'related.ip'
}));
builder.Add("setRelatedUser", appendFields({
fields: [
"user.name",
"user.target.name",
"file.owner",
],
to: 'related.user'
}));
// Set user-agent from an alternative location.
builder.Add("altUserAgent", function(evt) {
var ext = evt.Get("o365audit.ExtendedProperties.UserAgent");
if (ext != null) evt.Put("user_agent.original", ext);
});
// Set host.name to the O365 tenant. This is necessary to aggregate events
// in SIEM app based on the tenant instead of the host where Filebeat is
// running.
builder.Add("setHostName", function(evt) {
var value;
if ((value=evt.Get("organization.id"))!=null) {
value = value.toLowerCase();
evt.Put("host.id", value);
// Use tenant name provided in the configuration.
if (value in tenant_names && value !== "") {
evt.Put("organization.name", value);
evt.Put("host.name", tenant_names[value]);
return;
}
}
if ((value=evt.Get("organization.name"))!=null ||
(value=evt.Get("user.domain")) != null ) {
evt.Put("host.name", value);
}
});
builder.Add("saveRaw", new processor.Convert({
fields: [
{from: "o365audit", to: "o365.audit"},
],
mode: "rename"
}));
var chain = builder.Build();
return {
process: chain.Run
};
}