[Github] - Added support for gcs and azure-blob-storage inputs for github audit data stream

packages/github/_dev/deploy/docker/docker-compose.yml

@@ -12,3 +12,17 @@ services:
       - http-server
       - --addr=:8080
       - --config=/files/config.yml
+  azure-blob-storage-emulator:
+    image: mcr.microsoft.com/azure-storage/azurite:latest
+    command: azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --skipApiVersionCheck --disableProductStyleUrl
+    ports:
+      - "10000/tcp"
+  uploader:
+    image: mcr.microsoft.com/azure-cli
+    depends_on:
+      - azure-blob-storage-emulator
+    volumes:
+      - ./sample_logs:/sample_logs
+    entrypoint: >
+      sh -c " sleep 5 && export AZURE_STORAGE_CONNECTION_STRING='DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azure-blob-storage-emulator:10000/devstoreaccount1;' && az storage container create --name test-container && az storage blob upload --container-name test-container --file /sample_logs/cloud-storage-data.log --name cloud-storage-data.log"
+

packages/github/_dev/deploy/docker/sample_logs/cloud-storage-data.log

@@ -0,0 +1,3 @@
+{"@timestamp": 1698579600000, "action": "user.login", "active": true, "actor": "john_doe", "actor_id": 12345, "actor_location": {"country_name": "USA", "ip": "192.168.1.1"}, "org_id": 67890, "org": "tech-corp", "user_id": 12345, "business_id": 56789, "business": "tech-enterprise", "message": "User logged in successfully.", "name": "John Doe", "device": "laptop", "login_method": "password"}
+{"actor":"github-actor","org":"Example-Org","action":"organization_default_label.create","created_at":1583364251067}
+{"actor":"github-actor","org":"Example-Org","created_at":1608939056939,"action":"org.oauth_app_access_approved","actor_location":{"country_code":"US"}}

packages/github/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.15.0"
+  changes:
+    - description: Added support for abs and gcs inputs in audit data stream.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/15303
 - version: "2.14.0"
   changes:
     - description: Add links panel widget in dashboards.

packages/github/data_stream/audit/_dev/test/system/test-abs-config.yml

@@ -0,0 +1,16 @@
+deployer: docker
+service: azure-blob-storage-emulator
+input: azure-blob-storage
+vars:
+data_stream:
+  vars:
+    account_name: devstoreaccount1
+    service_account_key: "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
+    storage_url: "http://{{Hostname}}:{{Port}}/devstoreaccount1/"
+    number_of_workers: 3
+    poll: true
+    poll_interval: 15s
+    containers: |
+      - name: test-container
+assert:
+  hit_count: 3

packages/github/data_stream/audit/agent/stream/abs.yml.hbs

@@ -0,0 +1,60 @@
+{{#if account_name}}
+account_name: {{account_name}}
+{{/if}}
+{{#if oauth2}}
+auth.oauth2:
+  client_id: {{client_id}}
+  client_secret: {{client_secret}}
+  tenant_id: {{tenant_id}}
+{{/if}}
+{{#if service_account_key}}
+auth.shared_credentials.account_key: {{service_account_key}}
+{{/if}}
+{{#if service_account_uri}}
+auth.connection_string.uri: {{service_account_uri}}
+{{/if}}
+{{#if storage_url}}
+storage_url: {{storage_url}}
+{{/if}}
+{{#if number_of_workers}}
+max_workers: {{number_of_workers}}
+{{/if}}
+{{#if poll}}
+poll: {{poll}}
+{{/if}}
+{{#if poll_interval}}
+poll_interval: {{poll_interval}}
+{{/if}}
+
+{{#if containers}}
+containers:
+{{containers}}
+{{/if}}
+{{#if file_selectors}}
+file_selectors:
+{{file_selectors}}
+{{/if}}
+{{#if timestamp_epoch}}
+timestamp_epoch: {{timestamp_epoch}}
+{{/if}}
+{{#if expand_event_list_from_field}}
+expand_event_list_from_field: {{expand_event_list_from_field}}
+{{/if}}
+
+tags:
+{{#if preserve_original_event}}
+  - preserve_original_event
+{{/if}}
+{{#if preserve_duplicate_custom_fields}}
+  - preserve_duplicate_custom_fields
+{{/if}}
+{{#each tags as |tag|}}
+  - {{tag}}
+{{/each}}
+{{#contains "forwarded" tags}}
+publisher_pipeline.disable_host: true
+{{/contains}}
+{{#if processors}}
+processors:
+{{processors}}
+{{/if}}

packages/github/data_stream/audit/agent/stream/gcs.yml.hbs

@@ -0,0 +1,53 @@
+{{#if project_id}}
+project_id: {{project_id}}
+{{/if}}
+{{#if service_account_key}}
+auth.credentials_json.account_key: {{service_account_key}}
+{{/if}}
+{{#if service_account_file}}
+auth.credentials_file.path: {{service_account_file}}
+{{/if}}
+{{#if number_of_workers}}
+max_workers: {{number_of_workers}}
+{{/if}}
+{{#if poll}}
+poll: {{poll}}
+{{/if}}
+{{#if poll_interval}}
+poll_interval: {{poll_interval}}
+{{/if}}
+{{#if buckets}}
+buckets:
+{{buckets}}
+{{/if}}
+{{#if file_selectors}}
+file_selectors:
+{{file_selectors}}
+{{/if}}
+{{#if timestamp_epoch}}
+timestamp_epoch: {{timestamp_epoch}}
+{{/if}}
+{{#if expand_event_list_from_field}}
+expand_event_list_from_field: {{expand_event_list_from_field}}
+{{/if}}
+{{#if alternative_host}}
+alternative_host: {{alternative_host}}
+{{/if}}
+
+tags:
+{{#if preserve_original_event}}
+  - preserve_original_event
+{{/if}}
+{{#if preserve_duplicate_custom_fields}}
+  - preserve_duplicate_custom_fields
+{{/if}}
+{{#each tags as |tag|}}
+  - {{tag}}
+{{/each}}
+{{#contains "forwarded" tags}}
+publisher_pipeline.disable_host: true
+{{/contains}}
+{{#if processors}}
+processors:
+{{processors}}
+{{/if}}

packages/github/data_stream/audit/fields/beats.yml

@@ -19,3 +19,15 @@
 - name: log.offset
   type: long
   description: Log offset.
+- name: azure.storage
+  type: group
+  fields:
+    - name: container.name
+      type: keyword
+      description: The name of the Azure Blob Storage container
+    - name: blob.name
+      type: keyword
+      description: The name of the Azure Blob Storage blob object
+    - name: blob.content_type
+      type: keyword
+      description: The content type of the Azure Blob Storage blob object