m365_defender: Improve ECS Mappings for Detection Rules

packages/m365_defender/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.14.0"
+  changes:
+    - description: Improve ECS mappings
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/10179
 - version: "2.13.0"
   changes:
     - description: Removed import_mappings. Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template.

packages/m365_defender/data_stream/event/_dev/test/pipeline/test-device.log

@@ -12,6 +12,8 @@
 {"Tenant":"DefaultTenant","category":"AdvancedHunting-DeviceProcessEvents","operationName":"Publish","properties":{"AccountDomain":"testmachine6","AccountName":"administrator1","AccountObjectId":null,"AccountSid":"S-1-5-21-1874808502-2282282112-3464708742-500","AccountUpn":null,"ActionType":"ProcessCreated","AdditionalFields":"[]","AppGuardContainerId":null,"DeviceId":"999b6fd7c532534ba50b3232fa992c38a273d4fb","DeviceName":"testmachine6","FileName":"smartscreen.exe","FileSize":2387456,"FolderPath":"C:\\Windows\\System32\\smartscreen.exe","InitiatingProcessAccountDomain":"nt authority","InitiatingProcessAccountName":"system","InitiatingProcessAccountObjectId":null,"InitiatingProcessAccountSid":"S-1-5-18","InitiatingProcessAccountUpn":null,"InitiatingProcessCommandLine":"svchost.exe -k DcomLaunch -p","InitiatingProcessCreationTime":"2022-11-09T17:39:34.1193719Z","InitiatingProcessFileName":"svchost.exe","InitiatingProcessFileSize":55320,"InitiatingProcessFolderPath":"c:\\windows\\system32\\svchost.exe","NetworkAdapterName":"en01","InitiatingProcessId":996,"InitiatingProcessIntegrityLevel":"System","InitiatingProcessLogonId":999,"InitiatingProcessMD5":"b7f884c1b74a263f746ee12a5f7c9f6a","InitiatingProcessParentCreationTime":"2022-11-09T17:39:33.8279942Z","InitiatingProcessParentFileName":"services.exe","InitiatingProcessParentId":852,"InitiatingProcessSHA1":"1bc5066ddf693fc034d6514618854e26a84fd0d1","InitiatingProcessSHA256":"add683a6910abbbf0e28b557fad0ba998166394932ae2aca069d9aa19ea8fe88","InitiatingProcessSignatureStatus":"Valid","InitiatingProcessSignerType":"OsVendor","InitiatingProcessTokenElevation":"TokenElevationTypeDefault","InitiatingProcessVersionInfoCompanyName":"Microsoft Corporation","InitiatingProcessVersionInfoFileDescription":"Host Process for Windows Services","InitiatingProcessVersionInfoInternalFileName":"svchost.exe","InitiatingProcessVersionInfoOriginalFileName":"svchost.exe","InitiatingProcessVersionInfoProductName":"Microsoft® Windows® Operating System","InitiatingProcessVersionInfoProductVersion":"10.0.19041.1806","LogonId":1443318,"MD5":"b9d697df9e883f0d99720b0430448cb1","MachineGroup":"UnassignedGroup","ProcessCommandLine":"smartscreen.exe -Embedding","ProcessCreationTime":"2022-11-09T17:59:52.0344972Z","ProcessId":6412,"ProcessIntegrityLevel":"High","ProcessTokenElevation":"TokenElevationTypeDefault","ProcessVersionInfoCompanyName":"Microsoft Corporation","ProcessVersionInfoFileDescription":"Windows Defender SmartScreen","ProcessVersionInfoInternalFileName":"smartscreen.exe","ProcessVersionInfoOriginalFileName":"smartscreen.exe","ProcessVersionInfoProductName":"Microsoft® Windows® Operating System","ProcessVersionInfoProductVersion":"10.0.19041.2251","ReportId":4824,"SHA1":"9dec87de894f5228033f87cf874441502bfa4f97","SHA256":"8011a5f4ac65d85cbe593bdad886449e3807d950b234e77c675a0f7ca3b7c781","Timestamp":"2022-11-09T17:59:52.6265786Z"},"tenantId":"12345af3-bc0e-4f36-b08e-27759e912345","time":"2022-11-09T18:03:21.9948950Z"}
 {"Tenant":"DefaultTenant","category":"AdvancedHunting-DeviceRegistryEvents","operationName":"Publish","properties":{"ActionType":"RegistryValueSet","AppGuardContainerId":null,"DeviceId":"999b6fd7c532534ba50b3232fa992c38a273d4fb","DeviceName":"testmachine6","InitiatingProcessAccountDomain":"nt authority","InitiatingProcessAccountName":"system","InitiatingProcessAccountObjectId":null,"InitiatingProcessAccountSid":"S-1-5-18","InitiatingProcessAccountUpn":null,"InitiatingProcessCommandLine":"powershell.exe -ExecutionPolicy AllSigned -NoProfile -NonInteractive","InitiatingProcessCreationTime":"2022-11-09T19:17:20.4156553Z","InitiatingProcessFileName":"powershell.exe","InitiatingProcessFileSize":452608,"InitiatingProcessFolderPath":"c:\\windows\\system32\\windowspowershell\\v1.0\\powershell.exe","InitiatingProcessId":5900,"InitiatingProcessIntegrityLevel":"System","InitiatingProcessMD5":"04029e121a0cfa5991749937dd22a1d9","InitiatingProcessParentCreationTime":"2022-11-09T19:16:54.9433819Z","InitiatingProcessParentFileName":"SenseIR.exe","InitiatingProcessParentId":5668,"InitiatingProcessSHA1":"f43d9bb316e30ae1a3494ac5b0624f6bea1bf054","InitiatingProcessSHA256":"9f914d42706fe215501044acd85a32d58aaef1419d404fddfa5d3b48f66ccd9f","InitiatingProcessTokenElevation":"TokenElevationTypeDefault","InitiatingProcessVersionInfoCompanyName":"Microsoft Corporation","InitiatingProcessVersionInfoFileDescription":"Windows PowerShell","InitiatingProcessVersionInfoInternalFileName":"POWERSHELL","InitiatingProcessVersionInfoOriginalFileName":"PowerShell.EXE","InitiatingProcessVersionInfoProductName":"Microsoft® Windows® Operating System","InitiatingProcessVersionInfoProductVersion":"10.0.19041.546","MachineGroup":"UnassignedGroup","PreviousRegistryKey":null,"PreviousRegistryValueData":null,"PreviousRegistryValueName":"Blob","RegistryKey":"HKEY_LOCAL_MACHINE\\SOFTWARE\\Microsoft\\SystemCertificates\\Windows Live ID Token Issuer\\Certificates\\B68D8F953E551914324E557E6164D68B9926650C","RegistryValueData":null,"RegistryValueName":"Blob","RegistryValueType":"Binary","ReportId":6571,"Timestamp":"2022-11-09T19:17:43.5752234Z"},"tenantId":"12345af3-bc0e-4f36-b08e-27759e912345","time":"2022-11-09T19:23:21.8925266Z"}
 {"Tenant":"DefaultTenant","category":"AdvancedHunting-DeviceRegistryEvents","operationName":"Publish","properties":{"ActionType":"RegistryValueSet","AppGuardContainerId":"","DeviceId":"2af9e3da2eb7bd1b6c1fccb55ab5cd4cdec1e583","DeviceName":"desktop-device3","InitiatingProcessAccountDomain":"nt authority","InitiatingProcessAccountName":"system","InitiatingProcessAccountObjectId":null,"InitiatingProcessAccountSid":"S-1-5-18","InitiatingProcessAccountUpn":null,"InitiatingProcessCommandLine":"\"MsSense.exe\"","InitiatingProcessCreationTime":"2024-05-06T11:55:32.2214858Z","InitiatingProcessFileName":"mssense.exe","InitiatingProcessFileSize":522184,"InitiatingProcessFolderPath":"c:\\program files\\windows defender advanced threat protection\\mssense.exe","InitiatingProcessId":4688,"InitiatingProcessIntegrityLevel":"System","InitiatingProcessMD5":"71fc679ef0665dde1cbb72c95cecf894","InitiatingProcessParentCreationTime":"2024-05-06T11:48:52.81722Z","InitiatingProcessParentFileName":"services.exe","InitiatingProcessParentId":688,"InitiatingProcessSHA1":"d608e39caae86429f9f45b7f9a1f0417222cf641","InitiatingProcessSHA256":"1b32190da2ba5be59c35fa659cc063d1dd98a9f87d0b0a716f99fbc1c8433022","InitiatingProcessTokenElevation":"TokenElevationTypeDefault","InitiatingProcessVersionInfoCompanyName":"Microsoft Corporation","InitiatingProcessVersionInfoFileDescription":"Windows Defender Advanced Threat Protection Service Executable","InitiatingProcessVersionInfoInternalFileName":"MsSense.exe","InitiatingProcessVersionInfoOriginalFileName":"MsSense.exe","InitiatingProcessVersionInfoProductName":"Microsoft® Windows® Operating System","InitiatingProcessVersionInfoProductVersion":"10.8737.26020.1018","MachineGroup":null,"PreviousRegistryKey":"","PreviousRegistryValueData":null,"PreviousRegistryValueName":"782655b2-0575-4aa2-82b8-7fd560afeff6","RegistryKey":"HKEY_LOCAL_MACHINE\\SYSTEM\\ControlSet001\\Control\\WMI\\Security","RegistryValueData":null,"RegistryValueName":"782655b2-0575-4aa2-82b8-7fd560afeff6","RegistryValueType":"Binary","ReportId":21669,"Timestamp":"2024-05-08T15:23:15.8225851Z"},"tenantId":"12345af3-bc0e-4f36-b08e-27759e912345","time":"2024-05-08T15:27:56.0452290Z"}
+{"Tenant":"DefaultTenant","_TimeReceivedBySvc":"2024-06-19T01:07:05.1053450Z","category":"AdvancedHunting-DeviceRegistryEvents","operationName":"Publish","properties":{"ActionType":"RegistryKeyDeleted","AppGuardContainerId":"","DeviceId":"2af9e3da2eb7bd1b6c1fccb55ab5cd4cdec1e583","DeviceName":"desktop-user","InitiatingProcessAccountDomain":"nt authority","InitiatingProcessAccountName":"system","InitiatingProcessAccountObjectId":null,"InitiatingProcessAccountSid":"S-1-5-18","InitiatingProcessAccountUpn":null,"InitiatingProcessCommandLine":"svchost.exe -k netsvcs -p -s wlidsvc","InitiatingProcessCreationTime":"2024-06-19T01:06:24.8543864Z","InitiatingProcessFileName":"svchost.exe","InitiatingProcessFileSize":57528,"InitiatingProcessFolderPath":"c:\\windows\\system32\\svchost.exe","InitiatingProcessId":3176,"InitiatingProcessIntegrityLevel":"System","InitiatingProcessMD5":"7469cc568ad6821fd9d925542730a7d8","InitiatingProcessParentCreationTime":"2024-06-18T16:30:41.690549Z","InitiatingProcessParentFileName":"services.exe","InitiatingProcessParentId":728,"InitiatingProcessRemoteSessionDeviceName":null,"InitiatingProcessRemoteSessionIP":null,"InitiatingProcessSHA1":"e4e3f6bbad17b41a42687b3d75ade4a10b0870ec","InitiatingProcessSHA256":"6fc3bf1fdfd76860be782554f8d25bd32f108db934d70f4253f1e5f23522e503","InitiatingProcessSessionId":0,"InitiatingProcessTokenElevation":"TokenElevationTypeDefault","InitiatingProcessVersionInfoCompanyName":"Microsoft Corporation","InitiatingProcessVersionInfoFileDescription":"Host Process for Windows Services","InitiatingProcessVersionInfoInternalFileName":"svchost.exe","InitiatingProcessVersionInfoOriginalFileName":"svchost.exe","InitiatingProcessVersionInfoProductName":"Microsoft® Windows® Operating System","InitiatingProcessVersionInfoProductVersion":"10.0.19041.4355","IsInitiatingProcessRemoteSession":false,"MachineGroup":null,"PreviousRegistryKey":"HKEY_LOCAL_MACHINE\\SOFTWARE\\Microsoft\\SystemCertificates\\Windows Live ID Token Issuer\\Certificates\\B68D8F953E551914324E557E6164D68B9926649C","PreviousRegistryValueData":null,"PreviousRegistryValueName":null,"RegistryKey":"","RegistryValueData":null,"RegistryValueName":null,"RegistryValueType":"None","ReportId":7857,"Timestamp":"2024-06-19T01:06:24.9112589Z"},"tenantId":"12345af3-bc0e-4f36-b08e-27759e912345","time":"2024-06-19T01:09:19.9778014Z"}
+{"Tenant":"DefaultTenant","_TimeReceivedBySvc":"2024-06-19T07:33:10.2684381Z","category":"AdvancedHunting-DeviceRegistryEvents","operationName":"Publish","properties":{"ActionType":"RegistryValueSet","AppGuardContainerId":"","DeviceId":"2af9e3da2eb7bd1b6c1fccb55ab5cd4cdec1e583","DeviceName":"desktop-user","InitiatingProcessAccountDomain":"nt authority","InitiatingProcessAccountName":"system","InitiatingProcessAccountObjectId":null,"InitiatingProcessAccountSid":"S-1-5-18","InitiatingProcessAccountUpn":null,"InitiatingProcessCommandLine":"\"MsSense.exe\"","InitiatingProcessCreationTime":"2024-06-18T16:30:43.2552366Z","InitiatingProcessFileName":"mssense.exe","InitiatingProcessFileSize":522200,"InitiatingProcessFolderPath":"c:\\program files\\windows defender advanced threat protection\\mssense.exe","InitiatingProcessId":3144,"InitiatingProcessIntegrityLevel":"System","InitiatingProcessMD5":"c311a5744bc0f42b2dbea2c68d1cbd06","InitiatingProcessParentCreationTime":"2024-06-18T16:30:41.690549Z","InitiatingProcessParentFileName":"services.exe","InitiatingProcessParentId":728,"InitiatingProcessRemoteSessionDeviceName":null,"InitiatingProcessRemoteSessionIP":null,"InitiatingProcessSHA1":"9e30598eded8386d8050f409ebd86b1fa5ec474e","InitiatingProcessSHA256":"6ea1404c4e81bc30f75d6a202c32485ca1e331b41bf3fd019bfffcc7425707b6","InitiatingProcessSessionId":0,"InitiatingProcessTokenElevation":"TokenElevationTypeDefault","InitiatingProcessVersionInfoCompanyName":"Microsoft Corporation","InitiatingProcessVersionInfoFileDescription":"Windows Defender Advanced Threat Protection Service Executable","InitiatingProcessVersionInfoInternalFileName":"MsSense.exe","InitiatingProcessVersionInfoOriginalFileName":"MsSense.exe","InitiatingProcessVersionInfoProductName":"Microsoft® Windows® Operating System","InitiatingProcessVersionInfoProductVersion":"10.8750.27558.1004","IsInitiatingProcessRemoteSession":false,"MachineGroup":null,"PreviousRegistryKey":"","PreviousRegistryValueData":"133632450540305655","PreviousRegistryValueName":"CrashHeartbeat","RegistryKey":"HKEY_LOCAL_MACHINE\\SOFTWARE\\Microsoft\\Windows Advanced Threat Protection","RegistryValueData":"133632558540099193","RegistryValueName":"CrashHeartbeat","RegistryValueType":"Qword","ReportId":9370,"Timestamp":"2024-06-19T07:30:54.2606584Z"},"tenantId":"12345af3-bc0e-4f36-b08e-27759e912345","time":"2024-06-19T07:35:47.9510563Z"}
 {"Tenant":"DefaultTenant","category":"AdvancedHunting-DeviceNetworkEvents","operationName":"Publish","properties":{"Timestamp": "2023-07-19T12:17:42.7782364Z","DeviceId": "22bb10ffe3104214b20fc7de339a2b053e915e5c","DeviceName": "janeslaptop1.corporatedomain","ActionType": "ConnectionFailed","RemoteIP": "175.16.199.0","RemotePort": 80,"RemoteUrl": "subdomain.domain.tld","LocalIP": "89.160.20.112","LocalPort": 50258,"Protocol": "Tcp","LocalIPType": "Private","RemoteIPType": "Public","InitiatingProcessSHA1": "3e44b0d0319d24fa51b472de23062b10c0c32ec3","InitiatingProcessSHA256": "fe0ddd41ed02f1faa59526c53178c8366d9c90a777619eaaf7b7e5656f3ea4cb","InitiatingProcessMD5": "df9b3bee634a5578481a8c7cf4f614a3","InitiatingProcessFileName": "msedgewebview2.exe","InitiatingProcessFileSize": 3657056,"InitiatingProcessVersionInfoCompanyName": "Microsoft Corporation","InitiatingProcessVersionInfoProductName": "Microsoft Edge WebView2","InitiatingProcessVersionInfoProductVersion": "114.0.1823.79","InitiatingProcessVersionInfoInternalFileName": "msedgewebview2_exe","InitiatingProcessVersionInfoOriginalFileName": "msedgewebview2.exe","InitiatingProcessVersionInfoFileDescription": "Microsoft Edge WebView2","InitiatingProcessId": 17916,"InitiatingProcessCommandLine": "\"msedgewebview2.exe\" --type=utility --utility-sub-type=network.mojom.NetworkService --lang=en-US --service-sandbox-type=none --noerrdialogs --user-data-dir=\"C:\\Users\\username\\AppData\\Local\\Citrix\\SelfService\\CitrixWebControlCache\\EBWebView\" --webview-exe-name=SelfService.exe --webview-exe-version=22.3.1.22 --embedded-browser-webview=1 --embedded-browser-webview-dpi-awareness=2 --mojo-platform-channel-handle=3456 --field-trial-handle=1824,i --enable-features=msSingleSignOnOSForPrimaryAccountIsShared --disable-features=MojoIpcz /prefetch:3 /pfhostedapp:1234","InitiatingProcessCreationTime": "2023-08-09T18:43:00.0810399Z","InitiatingProcessFolderPath": "c:\\program files (x86)\\microsoft\\edgewebview\\application\\114.0.1823.79\\msedgewebview2.exe","InitiatingProcessParentFileName": "msedgewebview2.exe","InitiatingProcessParentId": 17808,"InitiatingProcessParentCreationTime": "2023-08-09T18:42:58.8197327Z","InitiatingProcessAccountDomain": "corporatedomain","InitiatingProcessAccountName": "username","InitiatingProcessAccountSid": "S-1-5-21-57989841-2025429265-839522115-329672","InitiatingProcessAccountUpn": "email@domain","InitiatingProcessAccountObjectId": "3600a12b-9d66-4dc3-9e2a-956c3623d0e4","InitiatingProcessIntegrityLevel": "Medium","InitiatingProcessTokenElevation": "TokenElevationTypeDefault","ReportId": 110313,"AppGuardContainerId":null,"AdditionalFields":null},"tenantId":"12345af3-bc0e-4f36-b08e-27759e912345","time":"2023-07-19T18:03:21.9948950Z"}
 {"Tenant":"DefaultTenant","category":"AdvancedHunting-DeviceNetworkEvents","operationName":"Publish","properties":{"Timestamp": "2023-07-19T12:16:10.7489034Z","DeviceId": "22bb10ffe3104214b20fc7de339a2b053e915e5c","DeviceName": "janeslaptop1.corporatedomain","ActionType": "DnsConnectionInspected","RemoteIP": "175.16.199.0","RemotePort": 53,"RemoteUrl":null,"LocalIP": "89.160.20.112","LocalPort": 54125,"Protocol": "Udp","LocalIPType":null,"RemoteIPType":null,"InitiatingProcessSHA1":null,"InitiatingProcessSHA256":null,"InitiatingProcessMD5":null,"InitiatingProcessFileName":null,"InitiatingProcessFileSize":null,"InitiatingProcessVersionInfoCompanyName":null,"InitiatingProcessVersionInfoProductName":null,"InitiatingProcessVersionInfoProductVersion":null,"InitiatingProcessVersionInfoInternalFileName":null,"InitiatingProcessVersionInfoOriginalFileName":null,"InitiatingProcessVersionInfoFileDescription":null,"InitiatingProcessId": 0,"InitiatingProcessCommandLine":null,"InitiatingProcessCreationTime":null,"InitiatingProcessFolderPath":null,"InitiatingProcessParentFileName":null,"InitiatingProcessParentId": 0,"InitiatingProcessParentCreationTime":null,"InitiatingProcessAccountDomain":null,"InitiatingProcessAccountName":null,"InitiatingProcessAccountSid":null,"InitiatingProcessAccountUpn":null,"InitiatingProcessAccountObjectId":null,"InitiatingProcessIntegrityLevel":null,"InitiatingProcessTokenElevation": "None","ReportId": 19542,"AppGuardContainerId":null,"AdditionalFields": {  "direction": "Out",  "trans_id": "18296",  "rtt": "0.05926012992858887",  "query": "janeslaptop1.corporatedomain",  "qclass": "1",  "qclass_name": "C_INTERNET",  "qtype": "1",  "qtype_name": "A",  "rcode": "0",  "uid": "CpeJkh3698EpWwy4Z9",  "rcode_name": "NOERROR",  "AA": "true",  "TC": "false",  "RD": "true",  "RA": "true",  "answers": "[\"89.160.20.112\"]",  "TTLs": "[1200.0]",  "rejected": "false",  "ts": "133370937691236740"}},"tenantId":"12345af3-bc0e-4f36-b08e-27759e912345","time":"2023-07-19T18:03:22.9948950Z"}
 {"Tenant":"DefaultTenant","category":"AdvancedHunting-DeviceNetworkEvents","operationName":"Publish","properties":{"Timestamp": "2023-07-19T12:16:28.6231143Z","DeviceId": "22bb10ffe3104214b20fc7de339a2b053e915e5c","DeviceName": "janeslaptop1.corporatedomain","ActionType": "NtlmAuthenticationInspected","RemoteIP": "175.16.199.0","RemotePort": 135,"RemoteUrl":null,"LocalIP": "89.160.20.112","LocalPort": 55514,"Protocol": "Tcp","LocalIPType":null,"RemoteIPType":null,"InitiatingProcessSHA1":null,"InitiatingProcessSHA256":null,"InitiatingProcessMD5":null,"InitiatingProcessFileName":null,"InitiatingProcessFileSize":null,"InitiatingProcessVersionInfoCompanyName":null,"InitiatingProcessVersionInfoProductName":null,"InitiatingProcessVersionInfoProductVersion":null,"InitiatingProcessVersionInfoInternalFileName":null,"InitiatingProcessVersionInfoOriginalFileName":null,"InitiatingProcessVersionInfoFileDescription":null,"InitiatingProcessId": 0,"InitiatingProcessCommandLine":null,"InitiatingProcessCreationTime":null,"InitiatingProcessFolderPath":null,"InitiatingProcessParentFileName":null,"InitiatingProcessParentId": 0,"InitiatingProcessParentCreationTime":null,"InitiatingProcessAccountDomain":null,"InitiatingProcessAccountName":null,"InitiatingProcessAccountSid":null,"InitiatingProcessAccountUpn":null,"InitiatingProcessAccountObjectId":null,"InitiatingProcessIntegrityLevel":null,"InitiatingProcessTokenElevation": "None","ReportId": 33108,"AppGuardContainerId":null,"AdditionalFields": {  "direction": "In",  "server_nb_computer_name": "hostname",  "server_nb_domain_name": "corporatedomain",  "server_dns_computer_name": "janeslaptop1.corporatedomain",  "server_dns_domain_name": "corporatedomain",  "server_tree_name": "corporatedomain",  "uid": "Cd6CKC1yC7AvYHXnq",  "server_version": "10.0 22621 15",  "ts": "133370931234950000"}},"tenantId":"12345af3-bc0e-4f36-b08e-27759e912345","time":"2023-07-19T18:03:23.9948950Z"}