feat: improve s3 access log parsing along with documentation updates

Kavindu-Dodan · Kavindu-Dodan · commit 40313afccab2 · 2025-09-12T11:10:57.000-07:00
Signed-off-by: Kavindu Dodanduwa &lt;kavindu.dodanduwa@elastic.co&gt;
diff --git a/packages/awsfirehose/_dev/build/docs/README.md b/packages/awsfirehose/_dev/build/docs/README.md
@@ -52,11 +52,15 @@ This is a current limitation in Firehose, which we are working with AWS to resol
 
     To effectively utilize your data, you need to install the necessary integrations. Log in to Kibana, navigate to **Management** > **Integrations** in the sidebar.
 
-    - First Install the **Amazon Data Firehose Integration**. This integration will receive and route your Firehose data to various backend datasets. Find it by searching or browse the catalog.
+    - First Install **Amazon Data Firehose Integration** assets. Assets installed through this integration will receive and route your Firehose data to various backend datasets. Find it by searching or browse the catalog.
     
     ![Amazon Data Firehose](../img/amazonfirehose.png)
-    
-    - Second install the **AWS integration**. This integration provides assets such as index templates, ingest pipelines, and dashboards that will serve as the destination points for the data routed by the Firehose Integration.
+
+     Navigate to the **Settings** tab and click **Install Amazon Data Firehose**. Confirm by clicking **Install Amazon Data Firehose** in the popup.
+
+    ![Install Firehose assets](../img/firehose_settings.png)
+
+    - Then install the **AWS integration** assets. This integration provides assets such as index templates, ingest pipelines, and dashboards that will serve as the destination points for the data routed by the Firehose Integration.
     Find the **AWS** integration by searching or browsing the catalog.
     
     ![AWS integration](../img/aws.png)
diff --git a/packages/awsfirehose/changelog.yml b/packages/awsfirehose/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.9.0"
+  changes:
+    - description: Improve access log classification and improve documentation
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/15309
 - version: "1.8.2"
   changes:
     - description: Remove regex from ingest pipeline
diff --git a/packages/awsfirehose/data_stream/logs/_dev/test/pipeline/test-s3access-log.json b/packages/awsfirehose/data_stream/logs/_dev/test/pipeline/test-s3access-log.json
@@ -29,6 +29,21 @@
             "data_stream.dataset": "awsfirehose",
             "aws.kinesis.name": "firehose-s3access-logs-to-elastic",
             "event.id": "37670326805251200781477669690942747782212394134076063744"
+        },
+        {
+            "cloud.region": "us-east-1",
+            "aws.firehose.arn": "arn:aws:firehose:us-east-2:123456:deliverystream/firehose-s3access-logs-to-elastic",
+            "data_stream.namespace": "default",
+            "message": "7cd47ef2be amzn-s3-demo-bucket [06/Feb/2019:00:00:38 +0000] 192.0.2.3 79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be e5042925-b524-4b3b-a869-f3881e78ff3a S3.COMPUTE.OBJECT.CHECKSUM example-object - - - - 1048576 - - - - -bPf7qjG4XwYdPgDQTl72GW/uotRhdPz2UryEyAFLDSRmKrakUkJCYLtAw6fdANcrsUYc1M/kIulXM1u5vZQT5g== - - - - - -",
+            "aws.kinesis.type": "deliverystream",
+            "data_stream.type": "logs",
+            "aws.firehose.request_id": "971ae05f-a128-4a7f-b623-30f9bc513e55",
+            "cloud.provider": "aws",
+            "@timestamp": "2023-07-25T21:04:35Z",
+            "cloud.account.id": "123456",
+            "data_stream.dataset": "awsfirehose",
+            "aws.kinesis.name": "firehose-s3access-logs-to-elastic",
+            "event.id": "checksum-test"
         }
     ]
 }
diff --git a/packages/awsfirehose/data_stream/logs/_dev/test/pipeline/test-s3access-log.json-expected.json b/packages/awsfirehose/data_stream/logs/_dev/test/pipeline/test-s3access-log.json-expected.json
@@ -41,6 +41,27 @@
             },
             "event.id": "37670326805251200781477669690942747782212394134076063744",
             "message": "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [01/Aug/2019:00:24:41 +0000] 89.160.20.156 arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9 44EE8651683CB4DA REST.GET.LOCATION - \"GET /test-s3-ks/?location&aws-account=627959692251 HTTP/1.1\" 200 - 142 - 17 - \"-\" \"AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation\" - BsCfJedfuSnds2QFoxi+E/O7M6OEWzJnw4dUaes/2hyA363sONRJKzB7EOY+Bt9DTHYUn+HoHxI= SigV4 ECDHE-RSA-AES128-SHA AuthHeader - TLSv1.2"
+        },
+        {
+            "@timestamp": "2023-07-25T21:04:35Z",
+            "aws.firehose.arn": "arn:aws:firehose:us-east-2:123456:deliverystream/firehose-s3access-logs-to-elastic",
+            "aws.firehose.request_id": "971ae05f-a128-4a7f-b623-30f9bc513e55",
+            "aws.kinesis.name": "firehose-s3access-logs-to-elastic",
+            "aws.kinesis.type": "deliverystream",
+            "cloud.account.id": "123456",
+            "cloud.provider": "aws",
+            "cloud.region": "us-east-1",
+            "data_stream.dataset": "aws.s3access",
+            "data_stream.namespace": "default",
+            "data_stream.type": "logs",
+            "ecs": {
+                "version": "8.11.0"
+            },
+            "event": {
+                "dataset": "aws.s3access"
+            },
+            "event.id": "checksum-test",
+            "message": "7cd47ef2be amzn-s3-demo-bucket [06/Feb/2019:00:00:38 +0000] 192.0.2.3 79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be e5042925-b524-4b3b-a869-f3881e78ff3a S3.COMPUTE.OBJECT.CHECKSUM example-object - - - - 1048576 - - - - -bPf7qjG4XwYdPgDQTl72GW/uotRhdPz2UryEyAFLDSRmKrakUkJCYLtAw6fdANcrsUYc1M/kIulXM1u5vZQT5g== - - - - - -"
         }
     ]
 }
diff --git a/packages/awsfirehose/data_stream/logs/elasticsearch/ingest_pipeline/default.yml b/packages/awsfirehose/data_stream/logs/elasticsearch/ingest_pipeline/default.yml
@@ -66,6 +66,7 @@ processors:
         // AWS S3 Access Logs, CloudFront, and ELB Logs
         else {
             // Inlined logic for splitting tokens
+            // Note - Tokenization split Time field into two sections, hence token count after index 3 (0 based) is increased by 1
             def tokens_result = new ArrayList();
             StringBuilder currentToken = new StringBuilder();
             boolean insideQuotes = false;
@@ -89,16 +90,21 @@ processors:
             def tokens = tokens_result.toArray(new String[0]);
             def tokenCount = tokens.length;
         
-            // Check for S3 Access logs first using a more reliable token count
+            // Check for S3 Access logs first using a more reliable token content check
             if (tokenCount >= 24) {
                 def hostHeader = tokens[23];
                 if (hostHeader.contains('s3') && hostHeader.contains('amazonaws.com')) {
                     ctx.event.dataset = 'aws.s3access';
+                    return;
+                } 
+
+                // Check for operation field content. Refer - https://docs.aws.amazon.com/AmazonS3/latest/userguide/LogFormat.html#log-record-fields
+                def opField = tokens[7];
+                if (opField.startsWith("SOAP.") || opField.startsWith("REST.") || opField.startsWith("BATCH.") || opField.startsWith("WEBSITE.") || opField.startsWith("S3.")) {
+                     ctx.event.dataset = 'aws.s3access';
+                     return;
                 }
             }
-            if (tokenCount == 25) {
-                ctx.event.dataset = 'aws.s3access';
-            }
         
             // Fallback to CloudFront and ELB if S3 check fails
             if (ctx.event.dataset == null) {
diff --git a/packages/awsfirehose/docs/README.md b/packages/awsfirehose/docs/README.md
@@ -52,11 +52,15 @@ This is a current limitation in Firehose, which we are working with AWS to resol
 
     To effectively utilize your data, you need to install the necessary integrations. Log in to Kibana, navigate to **Management** > **Integrations** in the sidebar.
 
-    - First Install the **Amazon Data Firehose Integration**. This integration will receive and route your Firehose data to various backend datasets. Find it by searching or browse the catalog.
+    - First Install **Amazon Data Firehose Integration** assets. Assets installed through this integration will receive and route your Firehose data to various backend datasets. Find it by searching or browse the catalog.
     
     ![Amazon Data Firehose](../img/amazonfirehose.png)
-    
-    - Second install the **AWS integration**. This integration provides assets such as index templates, ingest pipelines, and dashboards that will serve as the destination points for the data routed by the Firehose Integration.
+
+     Navigate to the **Settings** tab and click **Install Amazon Data Firehose**. Confirm by clicking **Install Amazon Data Firehose** in the popup.
+
+    ![Install Firehose assets](../img/firehose_settings.png)
+
+    - Then install the **AWS integration** assets. This integration provides assets such as index templates, ingest pipelines, and dashboards that will serve as the destination points for the data routed by the Firehose Integration.
     Find the **AWS** integration by searching or browsing the catalog.
     
     ![AWS integration](../img/aws.png)
diff --git a/packages/awsfirehose/img/firehose_settings.png b/packages/awsfirehose/img/firehose_settings.png
diff --git a/packages/awsfirehose/manifest.yml b/packages/awsfirehose/manifest.yml
@@ -1,7 +1,7 @@
 format_version: "3.3.0"
 name: awsfirehose
 title: Amazon Data Firehose
-version: 1.8.2
+version: 1.9.0
 description: Stream logs and metrics from Amazon Data Firehose into Elastic Cloud.
 type: integration
 categories:
