elastic · taylor-swanson · Dec 13, 2021 · Oct 6, 2021 · Nov 22, 2021 · Nov 23, 2021
diff --git a/packages/aws/_dev/build/docs/firewall.md b/packages/aws/_dev/build/docs/firewall.md
@@ -0,0 +1,20 @@
+# AWS Network Firewall
+
+This integration is used to fetch logs and metrics from [AWS Network Firewall](https://aws.amazon.com/network-firewall/).
+
+## Logs
+
+The `firewall_logs` dataset collects AWS Network Firewall logs. Users can use these logs to
+monitor network activity.
+
+{{event "firewall_logs" }}
+
+{{fields "firewall_logs"}}
+
+## Metrics
+
+The `firewall_metrics` dataset collects AWS Network Firewall metrics.
+
+{{event "firewall_metrics" }}
+
+{{fields "firewall_metrics"}}
diff --git a/packages/aws/changelog.yml b/packages/aws/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.6.0"
+  changes:
+    - description: Add integration for AWS Network Firewall
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/2199
 - version: "1.5.0"
   changes:
     - description: Support Kibana 8.0

diff --git a/packages/aws/data_stream/firewall_logs/_dev/test/pipeline/test-common-config.yml b/packages/aws/data_stream/firewall_logs/_dev/test/pipeline/test-common-config.yml
@@ -0,0 +1,5 @@
+dynamic_fields:
+  event.ingested: ".*"
+fields:
+  tags:
+    - preserve_original_event
diff --git a/packages/aws/data_stream/firewall_logs/_dev/test/pipeline/test-firewall.log b/packages/aws/data_stream/firewall_logs/_dev/test/pipeline/test-firewall.log
@@ -0,0 +1,2 @@
+{"firewall_name":"AWSNetworkFirewall","availability_zone":"us-east-2a","event_timestamp":"1636381332","event":{"timestamp":"2021-11-08T14:22:12.637611+0000","flow_id":706471429191862,"event_type":"alert","src_ip":"81.2.69.143","src_port":51254,"dest_ip":"216.160.83.57","dest_port":80,"proto":"TCP","alert":{"action":"blocked","signature_id":1000003,"rev":1,"signature":"Deny all other TCP traffic","category":"","severity":3},"http":{"hostname":"216.160.83.57","url":"/","http_user_agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36","http_method":"GET","protocol":"HTTP/1.1","length":0},"app_proto":"http"}}
+{"firewall_name":"AWSNetworkFirewall","availability_zone":"us-east-2a","event_timestamp":"1636395643","event":{"timestamp":"2021-11-08T18:20:43.324542+0000","flow_id":625409144331688,"event_type":"netflow","src_ip":"216.160.83.61","src_port":61953,"dest_ip":"89.160.20.156","dest_port":5060,"proto":"TCP","netflow":{"pkts":1,"bytes":40,"start":"2021-11-08T18:18:11.990632+0000","end":"2021-11-08T18:18:11.990632+0000","age":0,"min_ttl":221,"max_ttl":221},"tcp":{"tcp_flags":"02","syn":true}}}
diff --git a/packages/aws/data_stream/firewall_logs/_dev/test/pipeline/test-firewall.log-expected.json b/packages/aws/data_stream/firewall_logs/_dev/test/pipeline/test-firewall.log-expected.json
@@ -0,0 +1,224 @@
+{
+    "expected": [
+        {
+            "destination": {
+                "geo": {
+                    "continent_name": "North America",
+                    "region_iso_code": "US-ID",
+                    "city_name": "Salmon",
+                    "country_iso_code": "US",
+                    "country_name": "United States",
+                    "region_name": "Idaho",
+                    "location": {
+                        "lon": -113.8784,
+                        "lat": 45.1571
+                    }
+                },
+                "as": {
+                    "number": 209,
+                    "organization": {
+                        "name": "CenturyLink Communications, LLC"
+                    }
+                },
+                "address": "216.160.83.57",
+                "port": 80,
+                "ip": "216.160.83.57",
+                "domain": "216.160.83.57"
+            },
+            "rule": {
+                "name": "Deny all other TCP traffic",
+                "id": "1000003"
+            },
+            "source": {
+                "geo": {
+                    "continent_name": "Europe",
+                    "region_iso_code": "GB-OXF",
+                    "city_name": "Abingdon",
+                    "country_iso_code": "GB",
+                    "country_name": "United Kingdom",
+                    "region_name": "Oxfordshire",
+                    "location": {
+                        "lon": -1.3614,
+                        "lat": 51.7095
+                    }
+                },
+                "as": {
+                    "number": 20712,
+                    "organization": {
+                        "name": "Andrews \u0026 Arnold Ltd"
+                    }
+                },
+                "address": "81.2.69.143",
+                "port": 51254,
+                "ip": "81.2.69.143"
+            },
+            "message": "",
+            "url": {
+                "path": "/",
+                "original": "/"
+            },
+            "tags": [
+                "preserve_original_event"
+            ],
+            "network": {
+                "protocol": "http",
+                "transport": "tcp"
+            },
+            "cloud": {
+                "availability_zone": "us-east-2a"
+            },
+            "observer": {
+                "name": "AWSNetworkFirewall"
+            },
+            "@timestamp": "2021-11-08T14:22:12.637Z",
+            "ecs": {
+                "version": "1.12.0"
+            },
+            "related": {
+                "ip": [
+                    "81.2.69.143",
+                    "216.160.83.57"
+                ]
+            },
+            "http": {
+                "request": {
+                    "method": "GET"
+                },
+                "version": "1.1"
+            },
+            "event": {
+                "severity": 3,
+                "ingested": "2021-11-23T19:58:40.222546600Z",
+                "original": "{\"firewall_name\":\"AWSNetworkFirewall\",\"availability_zone\":\"us-east-2a\",\"event_timestamp\":\"1636381332\",\"event\":{\"timestamp\":\"2021-11-08T14:22:12.637611+0000\",\"flow_id\":706471429191862,\"event_type\":\"alert\",\"src_ip\":\"81.2.69.143\",\"src_port\":51254,\"dest_ip\":\"216.160.83.57\",\"dest_port\":80,\"proto\":\"TCP\",\"alert\":{\"action\":\"blocked\",\"signature_id\":1000003,\"rev\":1,\"signature\":\"Deny all other TCP traffic\",\"category\":\"\",\"severity\":3},\"http\":{\"hostname\":\"216.160.83.57\",\"url\":\"/\",\"http_user_agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36\",\"http_method\":\"GET\",\"protocol\":\"HTTP/1.1\",\"length\":0},\"app_proto\":\"http\"}}",
+                "category": [
+                    "network"
+                ],
+                "type": [
+                    "connection",
+                    "denied"
+                ],
+                "kind": "alert"
+            },
+            "aws": {
+                "firewall": {
+                    "flow": {
+                        "id": "706471429191862"
+                    }
+                }
+            },
+            "user_agent": {
+                "name": "Chrome",
+                "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36",
+                "os": {
+                    "name": "Mac OS X",
+                    "version": "10.15.7",
+                    "full": "Mac OS X 10.15.7"
+                },
+                "device": {
+                    "name": "Mac"
+                },
+                "version": "95.0.4638.69"
+            }
+        },
+        {
+            "destination": {
+                "geo": {
+                    "continent_name": "Europe",
+                    "region_iso_code": "SE-AB",
+                    "city_name": "Tumba",
+                    "country_iso_code": "SE",
+                    "country_name": "Sweden",
+                    "region_name": "Stockholm",
+                    "location": {
+                        "lon": 17.8167,
+                        "lat": 59.2
+                    }
+                },
+                "as": {
+                    "number": 29518,
+                    "organization": {
+                        "name": "Bredband2 AB"
+                    }
+                },
+                "address": "89.160.20.156",
+                "port": 5060,
+                "ip": "89.160.20.156"
+            },
+            "source": {
+                "geo": {
+                    "continent_name": "North America",
+                    "region_iso_code": "US-ID",
+                    "city_name": "Salmon",
+                    "country_iso_code": "US",
+                    "country_name": "United States",
+                    "region_name": "Idaho",
+                    "location": {
+                        "lon": -113.8784,
+                        "lat": 45.1571
+                    }
+                },
+                "as": {
+                    "number": 209,
+                    "organization": {
+                        "name": "CenturyLink Communications, LLC"
+                    }
+                },
+                "address": "216.160.83.61",
+                "port": 61953,
+                "ip": "216.160.83.61"
+            },
+            "tags": [
+                "preserve_original_event"
+            ],
+            "network": {
+                "protocol": "unknown",
+                "transport": "tcp"
+            },
+            "cloud": {
+                "availability_zone": "us-east-2a"
+            },
+            "observer": {
+                "name": "AWSNetworkFirewall"
+            },
+            "@timestamp": "2021-11-08T18:20:43.324Z",
+            "ecs": {
+                "version": "1.12.0"
+            },
+            "related": {
+                "ip": [
+                    "216.160.83.61",
+                    "89.160.20.156"
+                ]
+            },
+            "event": {
+                "ingested": "2021-11-23T19:58:40.222557800Z",
+                "original": "{\"firewall_name\":\"AWSNetworkFirewall\",\"availability_zone\":\"us-east-2a\",\"event_timestamp\":\"1636395643\",\"event\":{\"timestamp\":\"2021-11-08T18:20:43.324542+0000\",\"flow_id\":625409144331688,\"event_type\":\"netflow\",\"src_ip\":\"216.160.83.61\",\"src_port\":61953,\"dest_ip\":\"89.160.20.156\",\"dest_port\":5060,\"proto\":\"TCP\",\"netflow\":{\"pkts\":1,\"bytes\":40,\"start\":\"2021-11-08T18:18:11.990632+0000\",\"end\":\"2021-11-08T18:18:11.990632+0000\",\"age\":0,\"min_ttl\":221,\"max_ttl\":221},\"tcp\":{\"tcp_flags\":\"02\",\"syn\":true}}}",
+                "category": [
+                    "network"
+                ],
+                "type": [
+                    "connection"
+                ],
+                "kind": "event"
+            },
+            "aws": {
+                "firewall": {
+                    "tcp_flags_array": [
+                        "syn"
+                    ],
+                    "tcp_flags": "02",
+                    "flow": {
+                        "min_ttl": 221,
+                        "max_ttl": 221,
+                        "bytes": 40,
+                        "start": "2021-11-08T18:18:11.990632+0000",
+                        "end": "2021-11-08T18:18:11.990632+0000",
+                        "id": "625409144331688",
+                        "pkts": 1,
+                        "age": 0
+                    }
+                }
+            }
+        }
+    ]
+}
diff --git a/packages/aws/data_stream/firewall_logs/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/firewall_logs/agent/stream/aws-s3.yml.hbs
@@ -0,0 +1,48 @@
+queue_url: {{queue_url}}
+{{#if credential_profile_name}}
+credential_profile_name: {{credential_profile_name}}
+{{/if}}
+{{#if shared_credential_file}}
+shared_credential_file: {{shared_credential_file}}
+{{/if}}
+{{#if visibility_timeout}}
+visibility_timeout: {{visibility_timeout}}
+{{/if}}
+{{#if api_timeout}}
+api_timeout: {{api_timeout}}
+{{/if}}
+{{#if endpoint}}
+endpoint: {{endpoint}}
+{{/if}}
+{{#if access_key_id}}
+access_key_id: {{access_key_id}}
+{{/if}}
+{{#if secret_access_key}}
+secret_access_key: {{secret_access_key}}
+{{/if}}
+{{#if session_token}}
+session_token: {{session_token}}
+{{/if}}
+{{#if role_arn}}
+role_arn: {{role_arn}}
+{{/if}}
+{{#if fips_enabled}}
+fips_enabled: {{fips_enabled}}
+{{/if}}
+{{#if proxy_url }}
+proxy_url: {{proxy_url}}
+{{/if}}
+tags:
+{{#if preserve_original_event}}
+  - preserve_original_event
+{{/if}}
+{{#each tags as |tag i|}}
+  - {{tag}}
+{{/each}}
+{{#contains "forwarded" tags}}
+publisher_pipeline.disable_host: true
+{{/contains}}
+{{#if processors}}
+processors:
+{{processors}}
+{{/if}}
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		{"firewall_name":"AWSNetworkFirewall","availability_zone":"us-east-2a","event_timestamp":"1636381332","event":{"timestamp":"2021-11-08T14:22:12.637611+0000","flow_id":706471429191862,"event_type":"alert","src_ip":"81.2.69.143","src_port":51254,"dest_ip":"216.160.83.57","dest_port":80,"proto":"TCP","alert":{"action":"blocked","signature_id":1000003,"rev":1,"signature":"Deny all other TCP traffic","category":"","severity":3},"http":{"hostname":"216.160.83.57","url":"/","http_user_agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36","http_method":"GET","protocol":"HTTP/1.1","length":0},"app_proto":"http"}}
		{"firewall_name":"AWSNetworkFirewall","availability_zone":"us-east-2a","event_timestamp":"1636395643","event":{"timestamp":"2021-11-08T18:20:43.324542+0000","flow_id":625409144331688,"event_type":"netflow","src_ip":"216.160.83.61","src_port":61953,"dest_ip":"89.160.20.156","dest_port":5060,"proto":"TCP","netflow":{"pkts":1,"bytes":40,"start":"2021-11-08T18:18:11.990632+0000","end":"2021-11-08T18:18:11.990632+0000","age":0,"min_ttl":221,"max_ttl":221},"tcp":{"tcp_flags":"02","syn":true}}}