From 05fdeaca8e6e5102b21e7fadbadfeb24d6a1c333 Mon Sep 17 00:00:00 2001 From: Shourie Ganguly Date: Fri, 12 Sep 2025 15:00:36 +0530 Subject: [PATCH 01/18] added support for azure-blob-storage input along with relevant system tests --- .../_dev/deploy/docker/docker-compose.yml | 17 ++ .../docker/sample_logs/cloud-storage-data.log | 3 + packages/github/changelog.yml | 5 + .../_dev/test/system/test-abs-config.yml | 16 ++ .../audit/agent/stream/abs.yml.hbs | 60 +++++++ .../github/data_stream/audit/fields/beats.yml | 12 ++ .../github/data_stream/audit/manifest.yml | 153 ++++++++++++++++++ packages/github/manifest.yml | 5 +- 8 files changed, 270 insertions(+), 1 deletion(-) create mode 100644 packages/github/_dev/deploy/docker/sample_logs/cloud-storage-data.log create mode 100644 packages/github/data_stream/audit/_dev/test/system/test-abs-config.yml create mode 100644 packages/github/data_stream/audit/agent/stream/abs.yml.hbs diff --git a/packages/github/_dev/deploy/docker/docker-compose.yml b/packages/github/_dev/deploy/docker/docker-compose.yml index 4bffa205686..74fdc511569 100644 --- a/packages/github/_dev/deploy/docker/docker-compose.yml +++ b/packages/github/_dev/deploy/docker/docker-compose.yml @@ -12,3 +12,20 @@ services: - http-server - --addr=:8080 - --config=/files/config.yml + azure-blob-storage-emulator: + image: mcr.microsoft.com/azure-storage/azurite:latest + command: azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --skipApiVersionCheck --disableProductStyleUrl + ports: + - "10000/tcp" + uploader: + image: mcr.microsoft.com/azure-cli + depends_on: + - azure-blob-storage-emulator + volumes: + - ./sample_logs:/sample_logs + entrypoint: > + sh -c " + sleep 5 && + export AZURE_STORAGE_CONNECTION_STRING='DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azure-blob-storage-emulator:10000/devstoreaccount1;' && + az storage container create --name test-container && + az storage blob upload --container-name test-container --file /sample_logs/cloud-storage-data.log --name cloud-storage-data.log" diff --git a/packages/github/_dev/deploy/docker/sample_logs/cloud-storage-data.log b/packages/github/_dev/deploy/docker/sample_logs/cloud-storage-data.log new file mode 100644 index 00000000000..1fd019d2515 --- /dev/null +++ b/packages/github/_dev/deploy/docker/sample_logs/cloud-storage-data.log @@ -0,0 +1,3 @@ +{"@timestamp": 1698579600000, "action": "user.login", "active": true, "actor": "john_doe", "actor_id": 12345, "actor_location": {"country_name": "USA", "ip": "192.168.1.1"}, "org_id": 67890, "org": "tech-corp", "user_id": 12345, "business_id": 56789, "business": "tech-enterprise", "message": "User logged in successfully.", "name": "John Doe", "device": "laptop", "login_method": "password"} +{"actor":"github-actor","org":"Example-Org","action":"organization_default_label.create","created_at":1583364251067} +{"actor":"github-actor","org":"Example-Org","created_at":1608939056939,"action":"org.oauth_app_access_approved","actor_location":{"country_code":"US"}} diff --git a/packages/github/changelog.yml b/packages/github/changelog.yml index 550578f2dff..0878dda935f 100644 --- a/packages/github/changelog.yml +++ b/packages/github/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "2.15.0" + changes: + - description: Added support for abs and gcs inputs in audit data stream. + type: enhancement + link: https://github.com/elastic/integrations/pull/1111 - version: "2.14.0" changes: - description: Add links panel widget in dashboards. diff --git a/packages/github/data_stream/audit/_dev/test/system/test-abs-config.yml b/packages/github/data_stream/audit/_dev/test/system/test-abs-config.yml new file mode 100644 index 00000000000..c2de315a68c --- /dev/null +++ b/packages/github/data_stream/audit/_dev/test/system/test-abs-config.yml @@ -0,0 +1,16 @@ +deployer: docker +service: azure-blob-storage-emulator +input: azure-blob-storage +vars: +data_stream: + vars: + account_name: devstoreaccount1 + service_account_key: "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" + storage_url: "http://{{Hostname}}:{{Port}}/devstoreaccount1/" + number_of_workers: 3 + poll: true + poll_interval: 15s + containers: | + - name: test-container +assert: + hit_count: 3 diff --git a/packages/github/data_stream/audit/agent/stream/abs.yml.hbs b/packages/github/data_stream/audit/agent/stream/abs.yml.hbs new file mode 100644 index 00000000000..683822fa2ad --- /dev/null +++ b/packages/github/data_stream/audit/agent/stream/abs.yml.hbs @@ -0,0 +1,60 @@ +{{#if account_name}} +account_name: {{account_name}} +{{/if}} +{{#if oauth2}} +auth.oauth2: + client_id: {{client_id}} + client_secret: {{client_secret}} + tenant_id: {{tenant_id}} +{{/if}} +{{#if service_account_key}} +auth.shared_credentials.account_key: {{service_account_key}} +{{/if}} +{{#if service_account_uri}} +auth.connection_string.uri: {{service_account_uri}} +{{/if}} +{{#if storage_url}} +storage_url: {{storage_url}} +{{/if}} +{{#if number_of_workers}} +max_workers: {{number_of_workers}} +{{/if}} +{{#if poll}} +poll: {{poll}} +{{/if}} +{{#if poll_interval}} +poll_interval: {{poll_interval}} +{{/if}} + +{{#if containers}} +containers: +{{containers}} +{{/if}} +{{#if file_selectors}} +file_selectors: +{{file_selectors}} +{{/if}} +{{#if timestamp_epoch}} +timestamp_epoch: {{timestamp_epoch}} +{{/if}} +{{#if expand_event_list_from_field}} +expand_event_list_from_field: {{expand_event_list_from_field}} +{{/if}} + +tags: +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} +{{#if preserve_duplicate_custom_fields}} + - preserve_duplicate_custom_fields +{{/if}} +{{#each tags as |tag|}} + - {{tag}} +{{/each}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} +{{#if processors}} +processors: +{{processors}} +{{/if}} diff --git a/packages/github/data_stream/audit/fields/beats.yml b/packages/github/data_stream/audit/fields/beats.yml index b024cda7f40..dd41757b759 100644 --- a/packages/github/data_stream/audit/fields/beats.yml +++ b/packages/github/data_stream/audit/fields/beats.yml @@ -19,3 +19,15 @@ - name: log.offset type: long description: Log offset. +- name: azure.storage + type: group + fields: + - name: container.name + type: keyword + description: The name of the Azure Blob Storage container + - name: blob.name + type: keyword + description: The name of the Azure Blob Storage blob object + - name: blob.content_type + type: keyword + description: The content type of the Azure Blob Storage blob object \ No newline at end of file diff --git a/packages/github/data_stream/audit/manifest.yml b/packages/github/data_stream/audit/manifest.yml index 86468a21966..778bb2f3a1f 100644 --- a/packages/github/data_stream/audit/manifest.yml +++ b/packages/github/data_stream/audit/manifest.yml @@ -424,3 +424,156 @@ streams: # yvgJ38BRsFOtkRuAGSf6ZUwTO8JJRRIFnpUzXflAnGivK9M13D5GEQMmIl6U9Pvk # sxSmbIUfc2SGJGCJD4I= # -----END CERTIFICATE----- + - input: azure-blob-storage + title: GitHub Audit Logs + description: Collect GitHub audit logs from Azure Blob Storage + template_path: abs.yml.hbs + enabled: false + vars: + - name: account_name + type: text + title: Account Name + description: | + This attribute is required for various internal operations with respect to authentication, creating service clients and blob clients which are used internally for various processing purposes. + required: true + show_user: true + - name: client_id + type: text + title: Client ID (OAuth2) + description: Client ID of Azure Account. This is required if 'Collect logs using OAuth2 authentication' is enabled. + required: false + show_user: true + secret: true + - name: client_secret + type: password + title: Client Secret (OAuth2) + description: Client Secret of Azure Account. This is required if 'Collect logs using OAuth2 authentication' is enabled. + required: false + show_user: true + secret: true + - name: tenant_id + type: text + title: Tenant ID (OAuth2) + description: Tenant ID of Azure Account. This is required if 'Collect logs using OAuth2 authentication' is enabled. + multi: false + required: false + show_user: true + - name: service_account_key + type: password + title: Service Account Key + description: | + This attribute contains the access key, found under the Access keys section on Azure Cloud, under the respective storage account. A single storage account can contain multiple containers, and they will all use this common access key. + required: false + show_user: true + secret: true + - name: service_account_uri + type: text + title: Service Account URI + description: | + This attribute contains the connection string, found under the Access keys section on Azure Cloud, under the respective storage account. A single storage account can contain multiple containers, and they will all use this common connection string. + required: false + show_user: false + - name: storage_url + type: text + title: Storage URL + description: | + Use this attribute to specify a custom storage URL if required. By default it points to azure cloud storage. Only use this if there is a specific need to connect to a different environment where blob storage is available. + URL format : {{protocol}}://{{account_name}}.{{storage_uri}}. + required: false + show_user: false + - name: number_of_workers + type: integer + title: Maximum number of workers + multi: false + required: false + show_user: true + default: 3 + description: Determines how many workers are spawned per container. + - name: poll + type: bool + title: Polling + multi: false + required: false + show_user: true + default: true + description: Determines if the container will be continuously polled for new documents. + - name: poll_interval + type: text + title: Polling interval + multi: false + required: false + show_user: true + default: 15s + description: Determines the time interval between polling operations. + - name: containers + type: yaml + title: Containers + description: "This attribute contains the details about a specific container like, name, number_of_workers, poll, poll_interval etc. \nThe attribute 'name' is specific to a container as it describes the container name, while the fields number_of_workers, poll, poll_interval can exist both at the container level and at the global level. \nIf you have already defined the attributes globally, then you can only specify the container name in this yaml config. \nIf you want to override any specific attribute for a container, then, you can define it here. \nAny attribute defined in the yaml will override the global definitions. Please see the relevant [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-azure-blob-storage.html#attrib-containers) for further information.\n" + required: true + show_user: true + default: | + #- name: azure-container1 + # max_workers: 3 + # poll: true + # poll_interval: 15s + #- name: azure-container2 + # max_workers: 3 + # poll: true + # poll_interval: 10s + - name: file_selectors + type: yaml + title: File Selectors + multi: false + required: false + show_user: false + default: | + # - regex: "event/" + description: "If the container will have events that correspond to files that this integration shouldn’t process, file_selectors can be used to limit the files that are downloaded. This is a list of selectors which is made up of regex patters. \nThe regex should match the container filepath. Regexes use [RE2 syntax](https://pkg.go.dev/regexp/syntax). Files that don’t match one of the regexes will not be processed.\n" + - name: timestamp_epoch + type: integer + title: Timestamp Epoch + multi: false + required: false + description: "This attribute can be used to filter out files/blobs which have a timestamp older than the specified value. The value of this attribute should be in unix epoch (seconds) format. \nThis process happens locally on the host hence it is an expensive operation. \nIt is recommended to use this attribute only if there is a specific need to filter out files locally.\n" + show_user: false + - name: expand_event_list_from_field + type: text + title: Expand Event List From Field + multi: false + required: false + show_user: false + description: "If the file-set using this input expects to receive multiple messages bundled under a specific field or an array of objects then the config option for 'expand_event_list_from_field' can be specified. This setting will be able to split the messages under the group value into separate events. \nThis can be specified at the global level or at the container level. \nFor more info please refer to the [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-azure-blob-storage.html#attrib-expand_event_list_from_field)." + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original`. + type: bool + multi: false + default: false + - name: preserve_duplicate_custom_fields + required: true + show_user: false + title: Preserve duplicate custom fields + description: Preserve github.audit fields that were copied to Elastic Common Schema (ECS) fields. + type: bool + multi: false + default: false + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: | + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + - name: tags + type: text + title: Tags + description: Tags to include in the published event. + required: true + default: + - forwarded + - github.audit + multi: true + show_user: false diff --git a/packages/github/manifest.yml b/packages/github/manifest.yml index 56a22bda6fd..047b2441d6d 100644 --- a/packages/github/manifest.yml +++ b/packages/github/manifest.yml @@ -1,6 +1,6 @@ name: github title: GitHub -version: "2.14.0" +version: "2.15.0" description: Collect logs from GitHub with Elastic Agent. type: integration format_version: "3.4.0" @@ -68,6 +68,9 @@ policy_templates: - type: azure-eventhub title: "Collect GitHub logs from Azure Event Hub" description: "Collect GitHub logs from Azure Event Hub" + - type: azure-blob-storage + title: Collect GitHub logs from Azure Blob Storage + description: Collect GitHub logs from Azure Blob Storage - type: cel title: Collect GitHub Security Advisories data via API description: Collect GitHub Security Advisories data via API. From 985cb9097255831b96fac2060d7d8dbef88760fa Mon Sep 17 00:00:00 2001 From: Shourie Ganguly Date: Fri, 12 Sep 2025 15:30:14 +0530 Subject: [PATCH 02/18] elastic-package format --- packages/github/_dev/deploy/docker/docker-compose.yml | 7 ++----- packages/github/data_stream/audit/fields/beats.yml | 2 +- packages/github/data_stream/audit/manifest.yml | 8 ++++---- packages/github/docs/README.md | 3 +++ 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/packages/github/_dev/deploy/docker/docker-compose.yml b/packages/github/_dev/deploy/docker/docker-compose.yml index 74fdc511569..c546224c013 100644 --- a/packages/github/_dev/deploy/docker/docker-compose.yml +++ b/packages/github/_dev/deploy/docker/docker-compose.yml @@ -24,8 +24,5 @@ services: volumes: - ./sample_logs:/sample_logs entrypoint: > - sh -c " - sleep 5 && - export AZURE_STORAGE_CONNECTION_STRING='DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azure-blob-storage-emulator:10000/devstoreaccount1;' && - az storage container create --name test-container && - az storage blob upload --container-name test-container --file /sample_logs/cloud-storage-data.log --name cloud-storage-data.log" + sh -c " sleep 5 && export AZURE_STORAGE_CONNECTION_STRING='DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azure-blob-storage-emulator:10000/devstoreaccount1;' && az storage container create --name test-container && az storage blob upload --container-name test-container --file /sample_logs/cloud-storage-data.log --name cloud-storage-data.log" + diff --git a/packages/github/data_stream/audit/fields/beats.yml b/packages/github/data_stream/audit/fields/beats.yml index dd41757b759..ee5180577d7 100644 --- a/packages/github/data_stream/audit/fields/beats.yml +++ b/packages/github/data_stream/audit/fields/beats.yml @@ -30,4 +30,4 @@ description: The name of the Azure Blob Storage blob object - name: blob.content_type type: keyword - description: The content type of the Azure Blob Storage blob object \ No newline at end of file + description: The content type of the Azure Blob Storage blob object diff --git a/packages/github/data_stream/audit/manifest.yml b/packages/github/data_stream/audit/manifest.yml index 778bb2f3a1f..f59ce634d27 100644 --- a/packages/github/data_stream/audit/manifest.yml +++ b/packages/github/data_stream/audit/manifest.yml @@ -508,7 +508,7 @@ streams: - name: containers type: yaml title: Containers - description: "This attribute contains the details about a specific container like, name, number_of_workers, poll, poll_interval etc. \nThe attribute 'name' is specific to a container as it describes the container name, while the fields number_of_workers, poll, poll_interval can exist both at the container level and at the global level. \nIf you have already defined the attributes globally, then you can only specify the container name in this yaml config. \nIf you want to override any specific attribute for a container, then, you can define it here. \nAny attribute defined in the yaml will override the global definitions. Please see the relevant [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-azure-blob-storage.html#attrib-containers) for further information.\n" + description: "This attribute contains the details about a specific container like, name, number_of_workers, poll, poll_interval etc. The attribute 'name' is specific to a container as it describes the container name, while the fields number_of_workers, poll, poll_interval can exist both at the container level and at the global level. \nIf you have already defined the attributes globally, then you can only specify the container name in this yaml config. If you want to override any specific attribute for a container, then, you can define it here. Any attribute defined in the yaml will override the global definitions. \nPlease see the relevant [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-azure-blob-storage.html#attrib-containers) for further information.\n" required: true show_user: true default: | @@ -528,13 +528,13 @@ streams: show_user: false default: | # - regex: "event/" - description: "If the container will have events that correspond to files that this integration shouldn’t process, file_selectors can be used to limit the files that are downloaded. This is a list of selectors which is made up of regex patters. \nThe regex should match the container filepath. Regexes use [RE2 syntax](https://pkg.go.dev/regexp/syntax). Files that don’t match one of the regexes will not be processed.\n" + description: "If the container will have events that correspond to files that this integration shouldn’t process, file_selectors can be used to limit the files that are downloaded. \nThis is a list of selectors which is made up of regex patters. The regex should match the container filepath. Regexes use [RE2 syntax](https://pkg.go.dev/regexp/syntax). \nFiles that don’t match one of the regexes will not be processed.\n" - name: timestamp_epoch type: integer title: Timestamp Epoch multi: false required: false - description: "This attribute can be used to filter out files/blobs which have a timestamp older than the specified value. The value of this attribute should be in unix epoch (seconds) format. \nThis process happens locally on the host hence it is an expensive operation. \nIt is recommended to use this attribute only if there is a specific need to filter out files locally.\n" + description: "This attribute can be used to filter out files/blobs which have a timestamp older than the specified value. The value of this attribute should be in unix epoch (seconds) format. \nThis process happens locally on the host hence it is an expensive operation. It is recommended to use this attribute only if there is a specific need to filter out files locally.\n" show_user: false - name: expand_event_list_from_field type: text @@ -542,7 +542,7 @@ streams: multi: false required: false show_user: false - description: "If the file-set using this input expects to receive multiple messages bundled under a specific field or an array of objects then the config option for 'expand_event_list_from_field' can be specified. This setting will be able to split the messages under the group value into separate events. \nThis can be specified at the global level or at the container level. \nFor more info please refer to the [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-azure-blob-storage.html#attrib-expand_event_list_from_field)." + description: "If the file-set using this input expects to receive multiple messages bundled under a specific field or an array of objects then the config option for 'expand_event_list_from_field' can be specified. This setting will be able to split the messages under the group value into separate events. \nThis can be specified at the global level or at the container level. For more info please refer to the [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-azure-blob-storage.html#attrib-expand_event_list_from_field).\n" - name: preserve_original_event required: true show_user: true diff --git a/packages/github/docs/README.md b/packages/github/docs/README.md index 10cd3e9447b..a5ab4c6ae61 100644 --- a/packages/github/docs/README.md +++ b/packages/github/docs/README.md @@ -44,6 +44,9 @@ To collect audit log events from AWS S3 or AWS SQS, follow the [guide](https://d | aws.s3.bucket.arn | The AWS S3 bucket ARN. | keyword | | aws.s3.bucket.name | The AWS S3 bucket name. | keyword | | aws.s3.object.key | The AWS S3 Object key. | keyword | +| azure.storage.blob.content_type | The content type of the Azure Blob Storage blob object | keyword | +| azure.storage.blob.name | The name of the Azure Blob Storage blob object | keyword | +| azure.storage.container.name | The name of the Azure Blob Storage container | keyword | | data_stream.dataset | Data stream dataset name. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | From 3f0bc68725547e778b2a1969c4c655b49a604981 Mon Sep 17 00:00:00 2001 From: Shourie Ganguly Date: Fri, 12 Sep 2025 19:04:46 +0530 Subject: [PATCH 03/18] added gcs input support with formatting to existing manifest --- .../audit/agent/stream/gcs.yml.hbs | 53 +++++++ .../github/data_stream/audit/manifest.yml | 132 +++++++++++++++++- packages/github/manifest.yml | 7 +- 3 files changed, 189 insertions(+), 3 deletions(-) create mode 100644 packages/github/data_stream/audit/agent/stream/gcs.yml.hbs diff --git a/packages/github/data_stream/audit/agent/stream/gcs.yml.hbs b/packages/github/data_stream/audit/agent/stream/gcs.yml.hbs new file mode 100644 index 00000000000..08c4239e968 --- /dev/null +++ b/packages/github/data_stream/audit/agent/stream/gcs.yml.hbs @@ -0,0 +1,53 @@ +{{#if project_id}} +project_id: {{project_id}} +{{/if}} +{{#if service_account_key}} +auth.credentials_json.account_key: {{service_account_key}} +{{/if}} +{{#if service_account_file}} +auth.credentials_file.path: {{service_account_file}} +{{/if}} +{{#if number_of_workers}} +max_workers: {{number_of_workers}} +{{/if}} +{{#if poll}} +poll: {{poll}} +{{/if}} +{{#if poll_interval}} +poll_interval: {{poll_interval}} +{{/if}} +{{#if buckets}} +buckets: +{{buckets}} +{{/if}} +{{#if file_selectors}} +file_selectors: +{{file_selectors}} +{{/if}} +{{#if timestamp_epoch}} +timestamp_epoch: {{timestamp_epoch}} +{{/if}} +{{#if expand_event_list_from_field}} +expand_event_list_from_field: {{expand_event_list_from_field}} +{{/if}} +{{#if alternative_host}} +alternative_host: {{alternative_host}} +{{/if}} + +tags: +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} +{{#if preserve_duplicate_custom_fields}} + - preserve_duplicate_custom_fields +{{/if}} +{{#each tags as |tag|}} + - {{tag}} +{{/each}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} +{{#if processors}} +processors: +{{processors}} +{{/if}} \ No newline at end of file diff --git a/packages/github/data_stream/audit/manifest.yml b/packages/github/data_stream/audit/manifest.yml index f59ce634d27..9d198042129 100644 --- a/packages/github/data_stream/audit/manifest.yml +++ b/packages/github/data_stream/audit/manifest.yml @@ -528,7 +528,7 @@ streams: show_user: false default: | # - regex: "event/" - description: "If the container will have events that correspond to files that this integration shouldn’t process, file_selectors can be used to limit the files that are downloaded. \nThis is a list of selectors which is made up of regex patters. The regex should match the container filepath. Regexes use [RE2 syntax](https://pkg.go.dev/regexp/syntax). \nFiles that don’t match one of the regexes will not be processed.\n" + description: "If the container will have events that correspond to files that this integration shouldn’t process, file_selectors can be used to limit the files that are downloaded. \nThis is a list of selectors which is made up of regex patters. The regex should match the container filepath. Regexes use [RE2 syntax](https://pkg.go.dev/regexp/syntax). \nFiles that don’t match one of the regexes will not be processed. \nThis process happens locally on the host hence it is an expensive operation. It is recommended to use this attribute only if there is a specific need to filter out files locally.\n" - name: timestamp_epoch type: integer title: Timestamp Epoch @@ -577,3 +577,133 @@ streams: - github.audit multi: true show_user: false + - input: gcs + title: GitHub Audit Logs + description: Collect GitHub audit logs from Google Cloud Storage. + template_path: gcs.yml.hbs + enabled: false + vars: + - name: project_id + type: text + title: Project Id + description: | + This attribute is required for various internal operations with respect to authentication, creating service clients and bucket clients which are used internally for various processing purposes. + multi: false + required: true + show_user: true + default: my-project-id + - name: service_account_key + type: password + title: Credentials JSON Key + description: | + This attribute contains the json service account credentials string, which can be generated from the google cloud console, ref[Service Account Keys](https://cloud.google.com/iam/docs/creating-managing-service-account-keys). + Required if a Service Account File is not provided. + multi: false + required: false + show_user: true + secret: true + - name: service_account_file + type: text + title: Credentials File Path + description: | + This attribute contains the service account credentials file, which can be generated from the google cloud console, ref [Service Account Keys](https://cloud.google.com/iam/docs/creating-managing-service-account-keys). + Required if a Service Account Key is not provided. + multi: false + required: false + show_user: false + - name: number_of_workers + type: integer + title: Maximum number of workers + multi: false + required: false + show_user: true + default: 3 + description: Determines how many workers are spawned per bucket. + - name: poll + type: bool + title: Polling + multi: false + required: false + show_user: true + default: true + description: Determines if the bucket will be continuously polled for new documents. + - name: poll_interval + type: text + title: Polling Interval + multi: false + required: false + show_user: true + default: 15s + description: Determines the time interval between polling operations. + - name: buckets + type: yaml + title: Buckets + description: >- + This attribute contains the details about a specific bucket like, name, max_workers, poll and poll_interval. The attribute 'name' is specific to a bucket as it describes the bucket name, while the fields max_workers, poll and poll_interval can exist both at the bucket level and at the global level. If you have already defined the attributes globally, then you can only specify the name in this yaml config. If you want to override any specific attribute for a specific bucket, then, you can define it here. Any attribute defined in the yaml will override the global definitions. Please see the relevant[Documentation](https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-gcs#attrib-buckets) for further information. + required: true + show_user: true + default: | + # You can define as many buckets as you want here. + #- name: gcs_bucket_1 + #- name: gcs_bucket_2 + # The config below is an example of how to override the global config. + #- name: gcs_bucket_3 + # max_workers: 3 + # poll: true + # poll_interval: 10s + - name: file_selectors + type: yaml + title: File Selectors + multi: false + required: false + show_user: false + default: | + # - regex: "event/" + description: "If the bucket will have events that correspond to files that this integration shouldn’t process, file_selectors can be used to limit the files that are processed. \nThis is a list of selectors which is made up of regex patters. The regex should match the bucket filepath. Regexes use [RE2 syntax](https://pkg.go.dev/regexp/syntax). \nFiles that don’t match one of the regexes will not be processed. \nThis process happens locally on the host hence it is an expensive operation. It is recommended to use this attribute only if there is a specific need to filter out files locally.\n" + - name: timestamp_epoch + type: integer + title: Timestamp Epoch + multi: false + required: false + description: "This attribute can be used to filter out files/objects which have a timestamp older than the specified value. The value of this attribute should be in unix epoch (seconds) format. \nThis process happens locally on the host hence it is an expensive operation. It is recommended to use this attribute only if there is a specific need to filter out files locally.\n" + show_user: false + - name: expand_event_list_from_field + type: text + title: Expand Event List From Field + multi: false + required: false + show_user: false + description: "If the file-set using this input expects to receive multiple messages bundled under a specific field or an array of objects then the config option for 'expand_event_list_from_field' can be specified. This setting will be able to split the messages under the group value into separate events. \nThis can be specified at the global level or at the bucket level. For more info please refer to the [documentation](https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-gcs#attrib-expand_event_list_from_field-gcs).\n" + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original`. + type: bool + multi: false + default: false + - name: preserve_duplicate_custom_fields + required: true + show_user: false + title: Preserve duplicate custom fields + description: Preserve github.audit fields that were copied to Elastic Common Schema (ECS) fields. + type: bool + multi: false + default: false + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: | + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + - name: tags + type: text + title: Tags + multi: true + required: true + show_user: false + default: + - forwarded + - github.audit diff --git a/packages/github/manifest.yml b/packages/github/manifest.yml index 047b2441d6d..5ac74c604c3 100644 --- a/packages/github/manifest.yml +++ b/packages/github/manifest.yml @@ -7,7 +7,7 @@ format_version: "3.4.0" categories: [security, "productivity_security"] conditions: kibana: - version: "^8.16.0 || ^9.0.0" + version: "^8.17.1 || ^9.0.0" icons: - src: /img/github.svg title: GitHub @@ -70,7 +70,10 @@ policy_templates: description: "Collect GitHub logs from Azure Event Hub" - type: azure-blob-storage title: Collect GitHub logs from Azure Blob Storage - description: Collect GitHub logs from Azure Blob Storage + description: Collect GitHub logs from Azure Blob Storage. + - type: gcs + title: Collect GitHub logs from Google Cloud Storage + description: Collect GitHub logs from Google Cloud Storage. - type: cel title: Collect GitHub Security Advisories data via API description: Collect GitHub Security Advisories data via API. From 95d1c6cd55022358ff2a262356838c4d0901842e Mon Sep 17 00:00:00 2001 From: Shourie Ganguly Date: Fri, 12 Sep 2025 19:14:38 +0530 Subject: [PATCH 04/18] updated changelog && added new line --- packages/github/changelog.yml | 2 +- packages/github/data_stream/audit/agent/stream/gcs.yml.hbs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/github/changelog.yml b/packages/github/changelog.yml index 0878dda935f..0237f8ee422 100644 --- a/packages/github/changelog.yml +++ b/packages/github/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Added support for abs and gcs inputs in audit data stream. type: enhancement - link: https://github.com/elastic/integrations/pull/1111 + link: https://github.com/elastic/integrations/pull/15303 - version: "2.14.0" changes: - description: Add links panel widget in dashboards. diff --git a/packages/github/data_stream/audit/agent/stream/gcs.yml.hbs b/packages/github/data_stream/audit/agent/stream/gcs.yml.hbs index 08c4239e968..5a426e4a387 100644 --- a/packages/github/data_stream/audit/agent/stream/gcs.yml.hbs +++ b/packages/github/data_stream/audit/agent/stream/gcs.yml.hbs @@ -50,4 +50,4 @@ publisher_pipeline.disable_host: true {{#if processors}} processors: {{processors}} -{{/if}} \ No newline at end of file +{{/if}} From 954bb10f727ee2907147eab92b1ffccfc6cc04e4 Mon Sep 17 00:00:00 2001 From: Shourie Ganguly Date: Mon, 15 Sep 2025 19:46:16 +0530 Subject: [PATCH 05/18] created gcs mock-service that supports essential sdk api calls for system tests and added gcs system tests --- .../_dev/deploy/docker/docker-compose.yml | 34 ++- .../deploy/docker/gcs-mock-service/go.mod | 3 + .../deploy/docker/gcs-mock-service/main.go | 222 ++++++++++++++++++ .../_dev/test/system/test-gcs-config.yml | 28 +++ .../audit/agent/stream/gcs.yml.hbs | 6 +- .../github/data_stream/audit/fields/beats.yml | 12 + .../github/data_stream/audit/manifest.yml | 7 + 7 files changed, 307 insertions(+), 5 deletions(-) create mode 100644 packages/github/_dev/deploy/docker/gcs-mock-service/go.mod create mode 100644 packages/github/_dev/deploy/docker/gcs-mock-service/main.go create mode 100644 packages/github/data_stream/audit/_dev/test/system/test-gcs-config.yml diff --git a/packages/github/_dev/deploy/docker/docker-compose.yml b/packages/github/_dev/deploy/docker/docker-compose.yml index c546224c013..013022ddd13 100644 --- a/packages/github/_dev/deploy/docker/docker-compose.yml +++ b/packages/github/_dev/deploy/docker/docker-compose.yml @@ -12,17 +12,47 @@ services: - http-server - --addr=:8080 - --config=/files/config.yml + azure-blob-storage-emulator: image: mcr.microsoft.com/azure-storage/azurite:latest command: azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --skipApiVersionCheck --disableProductStyleUrl ports: - "10000/tcp" - uploader: + azure-cli: image: mcr.microsoft.com/azure-cli depends_on: - azure-blob-storage-emulator volumes: - ./sample_logs:/sample_logs entrypoint: > - sh -c " sleep 5 && export AZURE_STORAGE_CONNECTION_STRING='DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azure-blob-storage-emulator:10000/devstoreaccount1;' && az storage container create --name test-container && az storage blob upload --container-name test-container --file /sample_logs/cloud-storage-data.log --name cloud-storage-data.log" + sh -c " sleep 5 && + export AZURE_STORAGE_CONNECTION_STRING='DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azure-blob-storage-emulator:10000/devstoreaccount1;' && + az storage container create --name test-container && + az storage blob upload --container-name test-container --file /sample_logs/cloud-storage-data.log --name cloud-storage-data.log" + gcs-mock-service: + image: golang:1.24-alpine + working_dir: /app + volumes: + - ./gcs-mock-service:/app + ports: + - "4443:4443" + healthcheck: + test: "wget --no-verbose --tries=1 --spider http://localhost:4443/health || exit 1" + interval: 10s + timeout: 5s + command: ["go", "run", "main.go", "-host=0.0.0.0", "-port=4443"] + gcs-uploader: + image: curlimages/curl:latest + depends_on: + - gcs-mock-service + volumes: + - ./sample_logs/:/data + entrypoint: > + sh -c " + sleep 5 && + echo 'Creating bucket...' && + curl -X POST http://gcs-mock-service:4443/storage/v1/b -H 'Content-Type: application/json' -d '{\"name\": \"testbucket\"}' && + echo 'Uploading cloud-storage-data.log with correct content-type...' && + curl -X POST 'http://gcs-mock-service:4443/upload/storage/v1/b/testbucket/o?uploadType=media&name=cloud-storage-data.ndjson' -H 'Content-Type: application/x-ndjson' -T '/data/cloud-storage-data.log' && + echo 'Upload finished. Object available in emulator.'" diff --git a/packages/github/_dev/deploy/docker/gcs-mock-service/go.mod b/packages/github/_dev/deploy/docker/gcs-mock-service/go.mod new file mode 100644 index 00000000000..661288576d4 --- /dev/null +++ b/packages/github/_dev/deploy/docker/gcs-mock-service/go.mod @@ -0,0 +1,3 @@ +module gcs-mock-service + +go 1.24.7 diff --git a/packages/github/_dev/deploy/docker/gcs-mock-service/main.go b/packages/github/_dev/deploy/docker/gcs-mock-service/main.go new file mode 100644 index 00000000000..15d7287dd43 --- /dev/null +++ b/packages/github/_dev/deploy/docker/gcs-mock-service/main.go @@ -0,0 +1,222 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package main + +import ( + "encoding/json" + "flag" + "fmt" + "io" + "log" + "net/http" + "strconv" + "strings" +) + +// ObjectData stores the raw data and its content type. +type ObjectData struct { + Data []byte + ContentType string +} + +// The in-memory store to hold ObjectData structs. +var inMemoryStore = make(map[string]map[string]ObjectData) + +// GCSListResponse mimics the structure of a real GCS object list response. +type GCSListResponse struct { + Kind string `json:"kind"` + Items []GCSObject `json:"items"` +} + +// GCSObject mimics the structure of a GCS object resource with ContentType. +type GCSObject struct { + Kind string `json:"kind"` + Name string `json:"name"` + Bucket string `json:"bucket"` + Size string `json:"size"` + ContentType string `json:"contentType"` +} + +func handleRequests(w http.ResponseWriter, r *http.Request) { + path := r.URL.Path + log.Printf("Received request: %s %s", r.Method, path) + + switch r.Method { + case "GET": + if strings.HasPrefix(path, "/health") { + healthHandler(w, r) + return + } + if strings.HasPrefix(path, "/storage/v1/b/") && strings.HasSuffix(path, "/o") { + handleListObjects(w, r) + return + } + // route for getting an object (either path-style or API-style) + handleGetObject(w, r) + case "POST": + // route for creating a bucket: /storage/v1/b + if path == "/storage/v1/b" { + handleCreateBucket(w, r) + return + } + // route for uploading an object: /upload/storage/v1/b/{bucket}/o + if strings.HasPrefix(path, "/upload/storage/v1/b/") { + handleUploadObject(w, r) + return + } + default: + http.NotFound(w, r) + } +} + +// healthHandler responds with a simple "OK" message. +func healthHandler(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + fmt.Fprint(w, "OK") +} + +// handleCreateBucket creates a new, empty bucket. +func handleCreateBucket(w http.ResponseWriter, r *http.Request) { + var bucketInfo struct { + Name string `json:"name"` + } + if err := json.NewDecoder(r.Body).Decode(&bucketInfo); err != nil { + http.Error(w, "invalid JSON body", http.StatusBadRequest) + return + } + bucketName := bucketInfo.Name + if _, exists := inMemoryStore[bucketName]; exists { + http.Error(w, "bucket already exists", http.StatusConflict) + return + } + inMemoryStore[bucketName] = make(map[string]ObjectData) + log.Printf("created bucket: %s", bucketName) + w.WriteHeader(http.StatusOK) + json.NewEncoder(w).Encode(bucketInfo) +} + +// handleUploadObject uploads a new file to a bucket. +func handleUploadObject(w http.ResponseWriter, r *http.Request) { + pathParts := strings.Split(strings.Trim(r.URL.Path, "/"), "/") + if len(pathParts) < 5 { + http.Error(w, "invalid upload URL", http.StatusBadRequest) + return + } + bucketName := pathParts[4] + objectName := r.URL.Query().Get("name") + if objectName == "" { + http.Error(w, "missing 'name' query parameter", http.StatusBadRequest) + return + } + + if _, ok := inMemoryStore[bucketName]; !ok { + http.Error(w, "bucket not found", http.StatusNotFound) + return + } + + data, err := io.ReadAll(r.Body) + if err != nil { + http.Error(w, "failed to read request body", http.StatusInternalServerError) + return + } + defer r.Body.Close() + + contentType := r.Header.Get("Content-Type") + if contentType == "" { + contentType = "application/octet-stream" + } + + inMemoryStore[bucketName][objectName] = ObjectData{ + Data: data, + ContentType: contentType, + } + log.Printf("uploaded object '%s' to bucket '%s' with Content-Type '%s'", objectName, bucketName, contentType) + + response := GCSObject{ + Kind: "storage#object", + Name: objectName, + Bucket: bucketName, + Size: strconv.Itoa(len(data)), + ContentType: contentType, + } + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) +} + +// handleGetObject is for retrieving an object from a bucket. +func handleGetObject(w http.ResponseWriter, r *http.Request) { + var bucketName, objectName string + path := strings.Trim(r.URL.Path, "/") + parts := strings.Split(path, "/") + + if strings.HasPrefix(path, "storage/v1/b/") { + // api style: /storage/v1/b/{bucket}/o/{object} + if len(parts) >= 6 { + bucketName, objectName = parts[3], parts[5] + } + } else { + // path style: /{bucket}/{object} + if len(parts) >= 2 { + bucketName, objectName = parts[0], parts[1] + } + } + + if bucketName == "" || objectName == "" { + http.Error(w, "not found: invalid URL format", http.StatusNotFound) + return + } + + if bucket, ok := inMemoryStore[bucketName]; ok { + if object, ok := bucket[objectName]; ok { + w.Header().Set("Content-Type", object.ContentType) + w.Write(object.Data) + return + } + } + http.Error(w, "not found", http.StatusNotFound) +} + +// handleListObjects lists all objects in a bucket. +func handleListObjects(w http.ResponseWriter, r *http.Request) { + bucketName := strings.Split(strings.Trim(r.URL.Path, "/"), "/")[3] + + if bucket, ok := inMemoryStore[bucketName]; ok { + response := GCSListResponse{ + Kind: "storage#objects", + Items: []GCSObject{}, + } + for name, object := range bucket { + item := GCSObject{ + Kind: "storage#object", + Name: name, + Bucket: bucketName, + Size: strconv.Itoa(len(object.Data)), + ContentType: object.ContentType, + } + response.Items = append(response.Items, item) + } + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) + return + } + http.Error(w, "not found", http.StatusNotFound) +} + +func main() { + host := flag.String("host", "localhost", "host to listen on") + port := flag.String("port", "4443", "port to listen on") + flag.Parse() + + addr := fmt.Sprintf("%s:%s", *host, *port) + + fmt.Printf("Starting mock GCS server on http://%s\n", addr) + fmt.Println("Store is empty. Create buckets and objects via API calls.") + + http.HandleFunc("/", handleRequests) + + if err := http.ListenAndServe(addr, nil); err != nil { + log.Fatalf("failed to start server: %v", err) + } +} diff --git a/packages/github/data_stream/audit/_dev/test/system/test-gcs-config.yml b/packages/github/data_stream/audit/_dev/test/system/test-gcs-config.yml new file mode 100644 index 00000000000..1f34b36ea59 --- /dev/null +++ b/packages/github/data_stream/audit/_dev/test/system/test-gcs-config.yml @@ -0,0 +1,28 @@ +deployer: docker +service: gcs-mock-service +input: gcs +vars: +data_stream: + vars: + project_id: fake-gcs-project + alternative_host: "http://{{Hostname}}:{{Port}}" + number_of_workers: 1 + poll: true + poll_interval: 15s + service_account_key: | + { + "type": "service_account", + "project_id": "fake-gcs-project", + "private_key_id": "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2", + "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDLG1W45M5A25x/\n... (fake key data) ...\nB00//FFAohg=\n-----END PRIVATE KEY-----\n", + "client_email": "fake-service-account@fake-gcs-project.iam.gserviceaccount.com", + "client_id": "123456789012345678901", + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://oauth2.googleapis.com/token", + "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/fake-service-account%40fake-gcs-project.iam.gserviceaccount.com" + } + buckets: | + - name: testbucket +assert: + hit_count: 3 diff --git a/packages/github/data_stream/audit/agent/stream/gcs.yml.hbs b/packages/github/data_stream/audit/agent/stream/gcs.yml.hbs index 5a426e4a387..f72ce4a7acd 100644 --- a/packages/github/data_stream/audit/agent/stream/gcs.yml.hbs +++ b/packages/github/data_stream/audit/agent/stream/gcs.yml.hbs @@ -1,6 +1,9 @@ {{#if project_id}} project_id: {{project_id}} {{/if}} +{{#if alternative_host}} +alternative_host: {{alternative_host}} +{{/if}} {{#if service_account_key}} auth.credentials_json.account_key: {{service_account_key}} {{/if}} @@ -30,9 +33,6 @@ timestamp_epoch: {{timestamp_epoch}} {{#if expand_event_list_from_field}} expand_event_list_from_field: {{expand_event_list_from_field}} {{/if}} -{{#if alternative_host}} -alternative_host: {{alternative_host}} -{{/if}} tags: {{#if preserve_original_event}} diff --git a/packages/github/data_stream/audit/fields/beats.yml b/packages/github/data_stream/audit/fields/beats.yml index ee5180577d7..18b2ddf7138 100644 --- a/packages/github/data_stream/audit/fields/beats.yml +++ b/packages/github/data_stream/audit/fields/beats.yml @@ -31,3 +31,15 @@ - name: blob.content_type type: keyword description: The content type of the Azure Blob Storage blob object +- name: gcs.storage + type: group + fields: + - name: bucket.name + type: keyword + description: The name of the Google Cloud Storage Bucket. + - name: object.name + type: keyword + description: The content type of the Google Cloud Storage object. + - name: object.content_type + type: keyword + description: The content type of the Google Cloud Storage object. diff --git a/packages/github/data_stream/audit/manifest.yml b/packages/github/data_stream/audit/manifest.yml index 9d198042129..1dbfd8a1a2b 100644 --- a/packages/github/data_stream/audit/manifest.yml +++ b/packages/github/data_stream/audit/manifest.yml @@ -592,6 +592,13 @@ streams: required: true show_user: true default: my-project-id + - name: alternative_host + type: text + title: Alternative Host + description: Used to override the default host for the storage client (default is storage.googleapis.com) + required: false + multi: false + show_user: false - name: service_account_key type: password title: Credentials JSON Key From 1d06c8683a125999873bb513fff6f96e8642c40e Mon Sep 17 00:00:00 2001 From: Shourie Ganguly Date: Mon, 15 Sep 2025 19:49:33 +0530 Subject: [PATCH 06/18] implemented health condition dependency to help ci checks. --- .../_dev/deploy/docker/docker-compose.yml | 38 +++++++++++-------- .../deploy/docker/gcs-mock-service/main.go | 2 +- .../_dev/test/system/test-gcs-config.yml | 14 +------ packages/github/docs/README.md | 3 ++ 4 files changed, 27 insertions(+), 30 deletions(-) diff --git a/packages/github/_dev/deploy/docker/docker-compose.yml b/packages/github/_dev/deploy/docker/docker-compose.yml index 013022ddd13..6425fc0917d 100644 --- a/packages/github/_dev/deploy/docker/docker-compose.yml +++ b/packages/github/_dev/deploy/docker/docker-compose.yml @@ -12,47 +12,53 @@ services: - http-server - --addr=:8080 - --config=/files/config.yml - azure-blob-storage-emulator: image: mcr.microsoft.com/azure-storage/azurite:latest command: azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --skipApiVersionCheck --disableProductStyleUrl ports: - "10000/tcp" + healthcheck: + test: nc 127.0.0.1 10000 -z + interval: 10s + timeout: 5s azure-cli: image: mcr.microsoft.com/azure-cli depends_on: - - azure-blob-storage-emulator + azure-blob-storage-emulator: + condition: service_healthy volumes: - ./sample_logs:/sample_logs entrypoint: > - sh -c " sleep 5 && - export AZURE_STORAGE_CONNECTION_STRING='DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azure-blob-storage-emulator:10000/devstoreaccount1;' && + sh -c " + sleep 5 && + export AZURE_STORAGE_CONNECTION_STRING='DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azure-blob-storage-emulator:10000/devstoreaccount1;' && az storage container create --name test-container && az storage blob upload --container-name test-container --file /sample_logs/cloud-storage-data.log --name cloud-storage-data.log" - gcs-mock-service: - image: golang:1.24-alpine + gcs-mock-service: + image: golang:1.24.7-alpine working_dir: /app volumes: - ./gcs-mock-service:/app ports: - - "4443:4443" + - "4443/tcp" healthcheck: test: "wget --no-verbose --tries=1 --spider http://localhost:4443/health || exit 1" interval: 10s timeout: 5s - command: ["go", "run", "main.go", "-host=0.0.0.0", "-port=4443"] + command: ["go", "run", "main.go"] gcs-uploader: image: curlimages/curl:latest depends_on: - - gcs-mock-service + gcs-mock-service: + condition: service_healthy volumes: - ./sample_logs/:/data entrypoint: > - sh -c " - sleep 5 && - echo 'Creating bucket...' && - curl -X POST http://gcs-mock-service:4443/storage/v1/b -H 'Content-Type: application/json' -d '{\"name\": \"testbucket\"}' && - echo 'Uploading cloud-storage-data.log with correct content-type...' && - curl -X POST 'http://gcs-mock-service:4443/upload/storage/v1/b/testbucket/o?uploadType=media&name=cloud-storage-data.ndjson' -H 'Content-Type: application/x-ndjson' -T '/data/cloud-storage-data.log' && - echo 'Upload finished. Object available in emulator.'" + sh -c " + sleep 5 && + echo 'Creating bucket...' && + curl -X POST http://gcs-mock-service:4443/storage/v1/b -H 'Content-Type: application/json' -d '{\"name\": \"testbucket\"}' && + echo 'Uploading cloud-storage-data.log with correct content-type...' && + curl -X POST 'http://gcs-mock-service:4443/upload/storage/v1/b/testbucket/o?uploadType=media&name=cloud-storage-data.ndjson' -H 'Content-Type: application/x-ndjson' -T '/data/cloud-storage-data.log' && + echo 'Upload finished. Object available in emulator.'" diff --git a/packages/github/_dev/deploy/docker/gcs-mock-service/main.go b/packages/github/_dev/deploy/docker/gcs-mock-service/main.go index 15d7287dd43..77e29042e9c 100644 --- a/packages/github/_dev/deploy/docker/gcs-mock-service/main.go +++ b/packages/github/_dev/deploy/docker/gcs-mock-service/main.go @@ -205,7 +205,7 @@ func handleListObjects(w http.ResponseWriter, r *http.Request) { } func main() { - host := flag.String("host", "localhost", "host to listen on") + host := flag.String("host", "0.0.0.0", "host to listen on") port := flag.String("port", "4443", "port to listen on") flag.Parse() diff --git a/packages/github/data_stream/audit/_dev/test/system/test-gcs-config.yml b/packages/github/data_stream/audit/_dev/test/system/test-gcs-config.yml index 1f34b36ea59..d937cb37dc4 100644 --- a/packages/github/data_stream/audit/_dev/test/system/test-gcs-config.yml +++ b/packages/github/data_stream/audit/_dev/test/system/test-gcs-config.yml @@ -9,19 +9,7 @@ data_stream: number_of_workers: 1 poll: true poll_interval: 15s - service_account_key: | - { - "type": "service_account", - "project_id": "fake-gcs-project", - "private_key_id": "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2", - "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDLG1W45M5A25x/\n... (fake key data) ...\nB00//FFAohg=\n-----END PRIVATE KEY-----\n", - "client_email": "fake-service-account@fake-gcs-project.iam.gserviceaccount.com", - "client_id": "123456789012345678901", - "auth_uri": "https://accounts.google.com/o/oauth2/auth", - "token_uri": "https://oauth2.googleapis.com/token", - "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", - "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/fake-service-account%40fake-gcs-project.iam.gserviceaccount.com" - } + service_account_key: "{\"type\":\"service_account\",\"project_id\":\"fake-gcs-project\"}" buckets: | - name: testbucket assert: diff --git a/packages/github/docs/README.md b/packages/github/docs/README.md index a5ab4c6ae61..b09c176764f 100644 --- a/packages/github/docs/README.md +++ b/packages/github/docs/README.md @@ -52,6 +52,9 @@ To collect audit log events from AWS S3 or AWS SQS, follow the [guide](https://d | data_stream.type | Data stream type. | constant_keyword | | event.dataset | Event dataset | constant_keyword | | event.module | Event module | constant_keyword | +| gcs.storage.bucket.name | The name of the Google Cloud Storage Bucket. | keyword | +| gcs.storage.object.content_type | The content type of the Google Cloud Storage object. | keyword | +| gcs.storage.object.name | The content type of the Google Cloud Storage object. | keyword | | github.active | | boolean | | github.actor_id | The id of the actor who performed the action. | keyword | | github.actor_ip | The IP address of the entity performing the action. | ip | From 64c8af0dbc5b091c39c75a7b291649f7226b27da Mon Sep 17 00:00:00 2001 From: Shourie Ganguly Date: Tue, 16 Sep 2025 16:00:07 +0530 Subject: [PATCH 07/18] updated docs --- packages/github/_dev/build/docs/README.md | 10 +++++++++- packages/github/_dev/deploy/docker/docker-compose.yml | 2 +- packages/github/docs/README.md | 10 +++++++++- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/packages/github/_dev/build/docs/README.md b/packages/github/_dev/build/docs/README.md index 52d81135cc8..8761b4ea1cd 100644 --- a/packages/github/_dev/build/docs/README.md +++ b/packages/github/_dev/build/docs/README.md @@ -25,14 +25,22 @@ For Organizations: The GitHub audit log records all events related to the GitHub organization/enterprise. See [Organization audit log actions](https://docs.github.com/en/organizations/keeping-your-organization-secure/reviewing-the-audit-log-for-your-organization#audit-log-actions) and [Enterprise audit log actions](https://docs.github.com/en/enterprise-cloud@latest/admin/monitoring-activity-in-your-enterprise/reviewing-audit-logs-for-your-enterprise/about-the-audit-log-for-your-enterprise) for more details. -Github integration can collect audit logs from three sources: [Github API](https://docs.github.com/en/enterprise-cloud@latest/admin/monitoring-activity-in-your-enterprise/reviewing-audit-logs-for-your-enterprise/using-the-audit-log-api-for-your-enterprise), [Azure Event Hubs](https://docs.github.com/en/enterprise-cloud@latest/admin/monitoring-activity-in-your-enterprise/reviewing-audit-logs-for-your-enterprise/streaming-the-audit-log-for-your-enterprise#setting-up-streaming-to-azure-event-hubs), and [AWS S3 or AWS SQS](https://docs.github.com/en/enterprise-cloud@latest/admin/monitoring-activity-in-your-enterprise/reviewing-audit-logs-for-your-enterprise/streaming-the-audit-log-for-your-enterprise#setting-up-streaming-to-amazon-s3). +Github integration can collect audit logs from three sources: [Github API](https://docs.github.com/en/enterprise-cloud@latest/admin/monitoring-activity-in-your-enterprise/reviewing-audit-logs-for-your-enterprise/using-the-audit-log-api-for-your-enterprise), [Azure Event Hubs](https://docs.github.com/en/enterprise-cloud@latest/admin/monitoring-activity-in-your-enterprise/reviewing-audit-logs-for-your-enterprise/streaming-the-audit-log-for-your-enterprise#setting-up-streaming-to-azure-event-hubs), [Azure Blob Storage](https://docs.github.com/en/enterprise-cloud@latest/admin/monitoring-activity-in-your-enterprise/reviewing-audit-logs-for-your-enterprise/streaming-the-audit-log-for-your-enterprise#setting-up-streaming-to-azure-blob-storage), [AWS S3 or AWS SQS](https://docs.github.com/en/enterprise-cloud@latest/admin/monitoring-activity-in-your-enterprise/reviewing-audit-logs-for-your-enterprise/streaming-the-audit-log-for-your-enterprise#setting-up-streaming-to-amazon-s3) and [Google Cloud Storage](https://docs.github.com/en/enterprise-cloud@latest/admin/monitoring-activity-in-your-enterprise/reviewing-audit-logs-for-your-enterprise/streaming-the-audit-log-for-your-enterprise#setting-up-streaming-to-google-cloud-storage). When using Github API to collect audit log events, below requirements must be met for Personal Access Token (PAT): - You must use a Personal Access Token with `read:audit_log` scope. This applies to both organization and enterprise admins. - If you're an enterprise admin, ensure your token also includes `admin:enterprise` scope to access enterprise-wide logs. To collect audit log events from Azure Event Hubs, follow the [guide](https://docs.github.com/en/enterprise-cloud@latest/admin/monitoring-activity-in-your-enterprise/reviewing-audit-logs-for-your-enterprise/streaming-the-audit-log-for-your-enterprise#setting-up-streaming-to-azure-event-hubs) to setup audit log streaming. +To collect audit log events from Azure Blob Storage, follow the [guide](https://docs.github.com/en/enterprise-cloud@latest/admin/monitoring-activity-in-your-enterprise/reviewing-audit-logs-for-your-enterprise/streaming-the-audit-log-for-your-enterprise#setting-up-streaming-to-azure-blob-storage) to setup audit log streaming. To collect audit log events from AWS S3 or AWS SQS, follow the [guide](https://docs.github.com/en/enterprise-cloud@latest/admin/monitoring-activity-in-your-enterprise/reviewing-audit-logs-for-your-enterprise/streaming-the-audit-log-for-your-enterprise#setting-up-streaming-to-amazon-s3) to setup audit log streaming. For more details, refer to this [documentation](https://docs.github.com/en/enterprise-cloud@latest/admin/monitoring-activity-in-your-enterprise/reviewing-audit-logs-for-your-enterprise/streaming-the-audit-log-for-your-enterprise). +To collect audit log events from Google Cloud Storage, follow the [guide](https://docs.github.com/en/enterprise-cloud@latest/admin/monitoring-activity-in-your-enterprise/reviewing-audit-logs-for-your-enterprise/streaming-the-audit-log-for-your-enterprise#setting-up-streaming-to-google-cloud-storage) to setup audit log streaming. + +For Filebeat input documentation, please refer to the following: + - [Azure Event Hub](https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-azure-eventhub) + - [Azure Blob Storage](https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-azure-blob-storage) + - [AWS S3](https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-aws-s3) + - [Google Cloud Storage](https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-gcs) *This integration is not compatible with GitHub Enterprise server.* diff --git a/packages/github/_dev/deploy/docker/docker-compose.yml b/packages/github/_dev/deploy/docker/docker-compose.yml index 6425fc0917d..4304c86a4d6 100644 --- a/packages/github/_dev/deploy/docker/docker-compose.yml +++ b/packages/github/_dev/deploy/docker/docker-compose.yml @@ -61,4 +61,4 @@ services: curl -X POST http://gcs-mock-service:4443/storage/v1/b -H 'Content-Type: application/json' -d '{\"name\": \"testbucket\"}' && echo 'Uploading cloud-storage-data.log with correct content-type...' && curl -X POST 'http://gcs-mock-service:4443/upload/storage/v1/b/testbucket/o?uploadType=media&name=cloud-storage-data.ndjson' -H 'Content-Type: application/x-ndjson' -T '/data/cloud-storage-data.log' && - echo 'Upload finished. Object available in emulator.'" + echo 'Upload finished. Object available in gcs-mock-service.'" diff --git a/packages/github/docs/README.md b/packages/github/docs/README.md index b09c176764f..82f918ba582 100644 --- a/packages/github/docs/README.md +++ b/packages/github/docs/README.md @@ -25,14 +25,22 @@ For Organizations: The GitHub audit log records all events related to the GitHub organization/enterprise. See [Organization audit log actions](https://docs.github.com/en/organizations/keeping-your-organization-secure/reviewing-the-audit-log-for-your-organization#audit-log-actions) and [Enterprise audit log actions](https://docs.github.com/en/enterprise-cloud@latest/admin/monitoring-activity-in-your-enterprise/reviewing-audit-logs-for-your-enterprise/about-the-audit-log-for-your-enterprise) for more details. -Github integration can collect audit logs from three sources: [Github API](https://docs.github.com/en/enterprise-cloud@latest/admin/monitoring-activity-in-your-enterprise/reviewing-audit-logs-for-your-enterprise/using-the-audit-log-api-for-your-enterprise), [Azure Event Hubs](https://docs.github.com/en/enterprise-cloud@latest/admin/monitoring-activity-in-your-enterprise/reviewing-audit-logs-for-your-enterprise/streaming-the-audit-log-for-your-enterprise#setting-up-streaming-to-azure-event-hubs), and [AWS S3 or AWS SQS](https://docs.github.com/en/enterprise-cloud@latest/admin/monitoring-activity-in-your-enterprise/reviewing-audit-logs-for-your-enterprise/streaming-the-audit-log-for-your-enterprise#setting-up-streaming-to-amazon-s3). +Github integration can collect audit logs from three sources: [Github API](https://docs.github.com/en/enterprise-cloud@latest/admin/monitoring-activity-in-your-enterprise/reviewing-audit-logs-for-your-enterprise/using-the-audit-log-api-for-your-enterprise), [Azure Event Hubs](https://docs.github.com/en/enterprise-cloud@latest/admin/monitoring-activity-in-your-enterprise/reviewing-audit-logs-for-your-enterprise/streaming-the-audit-log-for-your-enterprise#setting-up-streaming-to-azure-event-hubs), [Azure Blob Storage](https://docs.github.com/en/enterprise-cloud@latest/admin/monitoring-activity-in-your-enterprise/reviewing-audit-logs-for-your-enterprise/streaming-the-audit-log-for-your-enterprise#setting-up-streaming-to-azure-blob-storage), [AWS S3 or AWS SQS](https://docs.github.com/en/enterprise-cloud@latest/admin/monitoring-activity-in-your-enterprise/reviewing-audit-logs-for-your-enterprise/streaming-the-audit-log-for-your-enterprise#setting-up-streaming-to-amazon-s3) and [Google Cloud Storage](https://docs.github.com/en/enterprise-cloud@latest/admin/monitoring-activity-in-your-enterprise/reviewing-audit-logs-for-your-enterprise/streaming-the-audit-log-for-your-enterprise#setting-up-streaming-to-google-cloud-storage). When using Github API to collect audit log events, below requirements must be met for Personal Access Token (PAT): - You must use a Personal Access Token with `read:audit_log` scope. This applies to both organization and enterprise admins. - If you're an enterprise admin, ensure your token also includes `admin:enterprise` scope to access enterprise-wide logs. To collect audit log events from Azure Event Hubs, follow the [guide](https://docs.github.com/en/enterprise-cloud@latest/admin/monitoring-activity-in-your-enterprise/reviewing-audit-logs-for-your-enterprise/streaming-the-audit-log-for-your-enterprise#setting-up-streaming-to-azure-event-hubs) to setup audit log streaming. +To collect audit log events from Azure Blob Storage, follow the [guide](https://docs.github.com/en/enterprise-cloud@latest/admin/monitoring-activity-in-your-enterprise/reviewing-audit-logs-for-your-enterprise/streaming-the-audit-log-for-your-enterprise#setting-up-streaming-to-azure-blob-storage) to setup audit log streaming. To collect audit log events from AWS S3 or AWS SQS, follow the [guide](https://docs.github.com/en/enterprise-cloud@latest/admin/monitoring-activity-in-your-enterprise/reviewing-audit-logs-for-your-enterprise/streaming-the-audit-log-for-your-enterprise#setting-up-streaming-to-amazon-s3) to setup audit log streaming. For more details, refer to this [documentation](https://docs.github.com/en/enterprise-cloud@latest/admin/monitoring-activity-in-your-enterprise/reviewing-audit-logs-for-your-enterprise/streaming-the-audit-log-for-your-enterprise). +To collect audit log events from Google Cloud Storage, follow the [guide](https://docs.github.com/en/enterprise-cloud@latest/admin/monitoring-activity-in-your-enterprise/reviewing-audit-logs-for-your-enterprise/streaming-the-audit-log-for-your-enterprise#setting-up-streaming-to-google-cloud-storage) to setup audit log streaming. + +For Filebeat input documentation, please refer to the following: + - [Azure Event Hub](https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-azure-eventhub) + - [Azure Blob Storage](https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-azure-blob-storage) + - [AWS S3](https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-aws-s3) + - [Google Cloud Storage](https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-gcs) *This integration is not compatible with GitHub Enterprise server.* From 36ee36356412b71b447e18ce94286688be36b008 Mon Sep 17 00:00:00 2001 From: Shourie Ganguly Date: Tue, 16 Sep 2025 16:23:56 +0530 Subject: [PATCH 08/18] added policy tests --- .../audit/_dev/test/policy/test-abs.expected | 43 +++++++++++++++++++ .../audit/_dev/test/policy/test-abs.yml | 13 ++++++ .../audit/_dev/test/policy/test-gcs.expected | 43 +++++++++++++++++++ .../audit/_dev/test/policy/test-gcs.yml | 13 ++++++ 4 files changed, 112 insertions(+) create mode 100644 packages/github/data_stream/audit/_dev/test/policy/test-abs.expected create mode 100644 packages/github/data_stream/audit/_dev/test/policy/test-abs.yml create mode 100644 packages/github/data_stream/audit/_dev/test/policy/test-gcs.expected create mode 100644 packages/github/data_stream/audit/_dev/test/policy/test-gcs.yml diff --git a/packages/github/data_stream/audit/_dev/test/policy/test-abs.expected b/packages/github/data_stream/audit/_dev/test/policy/test-abs.expected new file mode 100644 index 00000000000..dddc09c1aeb --- /dev/null +++ b/packages/github/data_stream/audit/_dev/test/policy/test-abs.expected @@ -0,0 +1,43 @@ +inputs: + - data_stream: + namespace: ep + meta: + package: + name: github + name: test-abs-github + streams: + - account_name: devstoreaccount1 + auth.shared_credentials.account_key: ${SECRET_0} + containers: + - name: test-container + data_stream: + dataset: github.audit + type: logs + file_selectors: null + max_workers: 3 + poll: true + poll_interval: 15s + publisher_pipeline.disable_host: true + tags: + - preserve_original_event + - preserve_duplicate_custom_fields + - forwarded + - github.audit + type: azure-blob-storage + use_output: default +output_permissions: + default: + _elastic_agent_checks: + cluster: + - monitor + _elastic_agent_monitoring: + indices: [] + uuid-for-permissions-on-related-indices: + indices: + - names: + - logs-github.audit-ep + privileges: + - auto_configure + - create_doc +secret_references: + - {} diff --git a/packages/github/data_stream/audit/_dev/test/policy/test-abs.yml b/packages/github/data_stream/audit/_dev/test/policy/test-abs.yml new file mode 100644 index 00000000000..b9564bd3a77 --- /dev/null +++ b/packages/github/data_stream/audit/_dev/test/policy/test-abs.yml @@ -0,0 +1,13 @@ +input: azure-blob-storage +vars: +data_stream: + vars: + account_name: devstoreaccount1 + service_account_key: "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" + number_of_workers: 3 + poll: true + poll_interval: 15s + containers: | + - name: test-container + preserve_original_event: true + preserve_duplicate_custom_fields: true diff --git a/packages/github/data_stream/audit/_dev/test/policy/test-gcs.expected b/packages/github/data_stream/audit/_dev/test/policy/test-gcs.expected new file mode 100644 index 00000000000..1933ba72070 --- /dev/null +++ b/packages/github/data_stream/audit/_dev/test/policy/test-gcs.expected @@ -0,0 +1,43 @@ +inputs: + - data_stream: + namespace: ep + meta: + package: + name: github + name: test-gcs-github + streams: + - auth.credentials_json.account_key: ${SECRET_0} + buckets: + - name: testbucket + data_stream: + dataset: github.audit + type: logs + file_selectors: null + max_workers: 3 + poll: true + poll_interval: 15s + project_id: gcs-project + publisher_pipeline.disable_host: true + tags: + - preserve_original_event + - preserve_duplicate_custom_fields + - forwarded + - github.audit + type: gcs + use_output: default +output_permissions: + default: + _elastic_agent_checks: + cluster: + - monitor + _elastic_agent_monitoring: + indices: [] + uuid-for-permissions-on-related-indices: + indices: + - names: + - logs-github.audit-ep + privileges: + - auto_configure + - create_doc +secret_references: + - {} diff --git a/packages/github/data_stream/audit/_dev/test/policy/test-gcs.yml b/packages/github/data_stream/audit/_dev/test/policy/test-gcs.yml new file mode 100644 index 00000000000..26dbd96261a --- /dev/null +++ b/packages/github/data_stream/audit/_dev/test/policy/test-gcs.yml @@ -0,0 +1,13 @@ +input: gcs +vars: +data_stream: + vars: + project_id: gcs-project + number_of_workers: 3 + poll: true + poll_interval: 15s + service_account_key: "{\"type\":\"service_account\",\"project_id\":\"fake-gcs-project\"}" + buckets: | + - name: testbucket + preserve_original_event: true + preserve_duplicate_custom_fields: true \ No newline at end of file From ccaa0343d9a100e28e20f8fd1ca2e7005b161991 Mon Sep 17 00:00:00 2001 From: Shourie Ganguly Date: Tue, 16 Sep 2025 18:01:36 +0530 Subject: [PATCH 09/18] added missing newline to gcs policy yml --- packages/github/data_stream/audit/_dev/test/policy/test-gcs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/github/data_stream/audit/_dev/test/policy/test-gcs.yml b/packages/github/data_stream/audit/_dev/test/policy/test-gcs.yml index 26dbd96261a..65c4885b493 100644 --- a/packages/github/data_stream/audit/_dev/test/policy/test-gcs.yml +++ b/packages/github/data_stream/audit/_dev/test/policy/test-gcs.yml @@ -10,4 +10,4 @@ data_stream: buckets: | - name: testbucket preserve_original_event: true - preserve_duplicate_custom_fields: true \ No newline at end of file + preserve_duplicate_custom_fields: true From 291402e8e639aa08865ddbc7d4d87c5ec91c5478 Mon Sep 17 00:00:00 2001 From: Shourie Ganguly Date: Wed, 17 Sep 2025 10:22:20 +0530 Subject: [PATCH 10/18] addressed PR comments --- packages/github/_dev/build/docs/README.md | 4 +--- .../github/_dev/deploy/docker/files/config.yml | 2 +- packages/github/changelog.yml | 10 +++++----- packages/github/data_stream/audit/manifest.yml | 13 ++++++++----- .../test-ghas-dependabot-json.log-expected.json | 16 ++++++++-------- .../elasticsearch/ingest_pipeline/default.yml | 4 ++-- .../data_stream/dependabot/sample_event.json | 2 +- .../test/pipeline/test-github-issues-json.log | 2 +- .../test-github-issues-json.log-expected.json | 4 ++-- packages/github/data_stream/issues/manifest.yml | 4 ++-- packages/github/docs/README.md | 6 ++---- .../transform/latest_code_scanning/transform.yml | 2 +- .../transform/latest_dependabot/transform.yml | 2 +- .../transform/latest_issues/transform.yml | 2 +- .../latest_secret_scanning/transform.yml | 2 +- ...hub-4da91aa0-12fc-11ed-af77-016e1a977d80.json | 2 +- ...hub-591d69e0-17b6-11ed-809a-7b4be950fe9c.json | 2 +- ...hub-6197be80-220c-11ed-88c4-e3caca48250a.json | 2 +- ...hub-6a6d7c40-17ab-11ed-809a-7b4be950fe9c.json | 2 +- ...hub-f0104680-ae18-11ed-83fa-df5d96a45724.json | 4 ++-- 20 files changed, 43 insertions(+), 44 deletions(-) diff --git a/packages/github/_dev/build/docs/README.md b/packages/github/_dev/build/docs/README.md index 8761b4ea1cd..9b024e6aab9 100644 --- a/packages/github/_dev/build/docs/README.md +++ b/packages/github/_dev/build/docs/README.md @@ -25,9 +25,7 @@ For Organizations: The GitHub audit log records all events related to the GitHub organization/enterprise. See [Organization audit log actions](https://docs.github.com/en/organizations/keeping-your-organization-secure/reviewing-the-audit-log-for-your-organization#audit-log-actions) and [Enterprise audit log actions](https://docs.github.com/en/enterprise-cloud@latest/admin/monitoring-activity-in-your-enterprise/reviewing-audit-logs-for-your-enterprise/about-the-audit-log-for-your-enterprise) for more details. -Github integration can collect audit logs from three sources: [Github API](https://docs.github.com/en/enterprise-cloud@latest/admin/monitoring-activity-in-your-enterprise/reviewing-audit-logs-for-your-enterprise/using-the-audit-log-api-for-your-enterprise), [Azure Event Hubs](https://docs.github.com/en/enterprise-cloud@latest/admin/monitoring-activity-in-your-enterprise/reviewing-audit-logs-for-your-enterprise/streaming-the-audit-log-for-your-enterprise#setting-up-streaming-to-azure-event-hubs), [Azure Blob Storage](https://docs.github.com/en/enterprise-cloud@latest/admin/monitoring-activity-in-your-enterprise/reviewing-audit-logs-for-your-enterprise/streaming-the-audit-log-for-your-enterprise#setting-up-streaming-to-azure-blob-storage), [AWS S3 or AWS SQS](https://docs.github.com/en/enterprise-cloud@latest/admin/monitoring-activity-in-your-enterprise/reviewing-audit-logs-for-your-enterprise/streaming-the-audit-log-for-your-enterprise#setting-up-streaming-to-amazon-s3) and [Google Cloud Storage](https://docs.github.com/en/enterprise-cloud@latest/admin/monitoring-activity-in-your-enterprise/reviewing-audit-logs-for-your-enterprise/streaming-the-audit-log-for-your-enterprise#setting-up-streaming-to-google-cloud-storage). - -When using Github API to collect audit log events, below requirements must be met for Personal Access Token (PAT): +When using GitHub API to collect audit log events, below requirements must be met for Personal Access Token (PAT): - You must use a Personal Access Token with `read:audit_log` scope. This applies to both organization and enterprise admins. - If you're an enterprise admin, ensure your token also includes `admin:enterprise` scope to access enterprise-wide logs. diff --git a/packages/github/_dev/deploy/docker/files/config.yml b/packages/github/_dev/deploy/docker/files/config.yml index 2e41dc7ae33..199abe1ef7c 100644 --- a/packages/github/_dev/deploy/docker/files/config.yml +++ b/packages/github/_dev/deploy/docker/files/config.yml @@ -766,7 +766,7 @@ rules: "closed_at": null, "author_association": "NONE", "active_lock_reason": null, - "body": "Structured Threat Information Expression (STIX) is a language for expressing cyber threat and observable information. While we have several Threat Intel integrations which map STIX formatted data to Elastic Common Schema, users will always have need to ingest IOC's from threat feeds that we don't support out of the box. 'How do I ingest STIX feeds' remains a very common questions across community Slack, Discuss, Github, etc. A custom package would solve for this. \r\n\r\nTo allow for the broad range of STIX formatted feeds, we should provide a way for users to ingest data from ANY STIX feed, via a 'Custom STIX' package. The package will leverage our httpjson input under the hood, but include an ingest pipeline which maps STIX fields to ECS (we expect there'll still be a need for custom fields, as not all STIX fields have a corresponding field in ECS). \r\n\r\nThere may be cases where some feeds/vendors don't strictly conform to STIX, and in those cases, users may have to modify our pipeline and that's ok. We'll focus on correctly formatted STIX data. ", + "body": "Structured Threat Information Expression (STIX) is a language for expressing cyber threat and observable information. While we have several Threat Intel integrations which map STIX formatted data to Elastic Common Schema, users will always have need to ingest IOC's from threat feeds that we don't support out of the box. 'How do I ingest STIX feeds' remains a very common questions across community Slack, Discuss, GitHub, etc. A custom package would solve for this. \r\n\r\nTo allow for the broad range of STIX formatted feeds, we should provide a way for users to ingest data from ANY STIX feed, via a 'Custom STIX' package. The package will leverage our httpjson input under the hood, but include an ingest pipeline which maps STIX fields to ECS (we expect there'll still be a need for custom fields, as not all STIX fields have a corresponding field in ECS). \r\n\r\nThere may be cases where some feeds/vendors don't strictly conform to STIX, and in those cases, users may have to modify our pipeline and that's ok. We'll focus on correctly formatted STIX data. ", "reactions": { "url": "https://api.github.com/repos/elastic/integrations/issues/4710/reactions", "total_count": 0, diff --git a/packages/github/changelog.yml b/packages/github/changelog.yml index 0237f8ee422..48f1ad0e2db 100644 --- a/packages/github/changelog.yml +++ b/packages/github/changelog.yml @@ -201,7 +201,7 @@ link: https://github.com/elastic/integrations/pull/7976 - version: 1.23.1 changes: - - description: Fix docs for Github Audit log permissions. + - description: Fix docs for GitHub Audit log permissions. type: bugfix link: https://github.com/elastic/integrations/pull/7954 - version: 1.23.0 @@ -301,7 +301,7 @@ link: https://github.com/elastic/integrations/pull/5765 - version: "1.9.0" changes: - - description: Release Github datastreams as GA. + - description: Release GitHub datastreams as GA. type: enhancement link: https://github.com/elastic/integrations/pull/5677 - version: "1.8.2" @@ -311,12 +311,12 @@ link: https://github.com/elastic/integrations/pull/5123 - version: "1.8.1" changes: - - description: Fix pagination in Github audit + - description: Fix pagination in GitHub audit type: bugfix link: https://github.com/elastic/integrations/issues/5210 - version: "1.8.0" changes: - - description: Add Github Issues datastream + - description: Add GitHub Issues datastream type: enhancement link: https://github.com/elastic/integrations/pull/5292 - version: "1.7.0" @@ -351,7 +351,7 @@ link: https://github.com/elastic/integrations/pull/3881 - version: "1.2.2" changes: - - description: Update Github Secret Scanning fingerprint with resolved_at + - description: Update GitHub Secret Scanning fingerprint with resolved_at type: bugfix link: https://github.com/elastic/integrations/pull/3802 - version: "1.2.1" diff --git a/packages/github/data_stream/audit/manifest.yml b/packages/github/data_stream/audit/manifest.yml index 1dbfd8a1a2b..5dc283b8d5e 100644 --- a/packages/github/data_stream/audit/manifest.yml +++ b/packages/github/data_stream/audit/manifest.yml @@ -603,7 +603,7 @@ streams: type: password title: Credentials JSON Key description: | - This attribute contains the json service account credentials string, which can be generated from the google cloud console, ref[Service Account Keys](https://cloud.google.com/iam/docs/creating-managing-service-account-keys). + This attribute contains the JSON service account credentials string, which can be generated from the google cloud console. Refer to [Service Account Keys](https://cloud.google.com/iam/docs/creating-managing-service-account-keys) for details. Required if a Service Account File is not provided. multi: false required: false @@ -613,7 +613,7 @@ streams: type: text title: Credentials File Path description: | - This attribute contains the service account credentials file, which can be generated from the google cloud console, ref [Service Account Keys](https://cloud.google.com/iam/docs/creating-managing-service-account-keys). + This attribute contains the service account credentials file, which can be generated from the google cloud console. Refer to [Service Account Keys](https://cloud.google.com/iam/docs/creating-managing-service-account-keys) for details. Required if a Service Account Key is not provided. multi: false required: false @@ -666,13 +666,15 @@ streams: show_user: false default: | # - regex: "event/" - description: "If the bucket will have events that correspond to files that this integration shouldn’t process, file_selectors can be used to limit the files that are processed. \nThis is a list of selectors which is made up of regex patters. The regex should match the bucket filepath. Regexes use [RE2 syntax](https://pkg.go.dev/regexp/syntax). \nFiles that don’t match one of the regexes will not be processed. \nThis process happens locally on the host hence it is an expensive operation. It is recommended to use this attribute only if there is a specific need to filter out files locally.\n" + description: > + "If the bucket will have events that correspond to files that this integration shouldn’t process, file_selectors can be used to limit the files that are processed. \nThis is a list of selectors which is made up of regex patters. The regex should match the bucket filepath. Regexes use [RE2 syntax](https://pkg.go.dev/regexp/syntax). \nFiles that don’t match one of the regexes will not be processed. \nThis process happens locally on the host hence it is an expensive operation. It is recommended to use this attribute only if there is a specific need to filter out files locally.\n" - name: timestamp_epoch type: integer title: Timestamp Epoch multi: false required: false - description: "This attribute can be used to filter out files/objects which have a timestamp older than the specified value. The value of this attribute should be in unix epoch (seconds) format. \nThis process happens locally on the host hence it is an expensive operation. It is recommended to use this attribute only if there is a specific need to filter out files locally.\n" + description: > + "This attribute can be used to filter out files/objects which have a timestamp older than the specified value. The value of this attribute should be in unix epoch (seconds) format. \nThis process happens locally on the host hence it is an expensive operation. It is recommended to use this attribute only if there is a specific need to filter out files locally.\n" show_user: false - name: expand_event_list_from_field type: text @@ -680,7 +682,8 @@ streams: multi: false required: false show_user: false - description: "If the file-set using this input expects to receive multiple messages bundled under a specific field or an array of objects then the config option for 'expand_event_list_from_field' can be specified. This setting will be able to split the messages under the group value into separate events. \nThis can be specified at the global level or at the bucket level. For more info please refer to the [documentation](https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-gcs#attrib-expand_event_list_from_field-gcs).\n" + description: > + "If the file-set using this input expects to receive multiple messages bundled under a specific field or an array of objects then the config option for 'expand_event_list_from_field' can be specified. This setting will be able to split the messages under the group value into separate events. \nThis can be specified at the global level or at the bucket level. For more info please refer to the [documentation](https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-gcs#attrib-expand_event_list_from_field-gcs).\n" - name: preserve_original_event required: true show_user: true diff --git a/packages/github/data_stream/dependabot/_dev/test/pipeline/test-ghas-dependabot-json.log-expected.json b/packages/github/data_stream/dependabot/_dev/test/pipeline/test-ghas-dependabot-json.log-expected.json index 42d42a45e89..11c4340d29a 100644 --- a/packages/github/data_stream/dependabot/_dev/test/pipeline/test-ghas-dependabot-json.log-expected.json +++ b/packages/github/data_stream/dependabot/_dev/test/pipeline/test-ghas-dependabot-json.log-expected.json @@ -98,7 +98,7 @@ "https://nodesecurity.io/advisories/17" ], "scanner": { - "vendor": "Github" + "vendor": "GitHub" }, "score": { "base": 0.0 @@ -205,7 +205,7 @@ "https://nodesecurity.io/advisories/154" ], "scanner": { - "vendor": "Github" + "vendor": "GitHub" }, "score": { "base": 0.0 @@ -315,7 +315,7 @@ "https://github.com/advisories/GHSA-3j7m-hmh3-9jmp" ], "scanner": { - "vendor": "Github" + "vendor": "GitHub" }, "score": { "base": 6.1, @@ -422,7 +422,7 @@ "https://github.com/advisories/GHSA-6g6m-m6h5-w9gf" ], "scanner": { - "vendor": "Github" + "vendor": "GitHub" }, "score": { "base": 7.7, @@ -511,7 +511,7 @@ "https://github.com/advisories/GHSA-5mrr-rgp6-x4gr" ], "scanner": { - "vendor": "Github" + "vendor": "GitHub" }, "score": { "base": 0.0 @@ -618,7 +618,7 @@ "https://github.com/advisories/GHSA-mjxr-4v3x-q3m4" ], "scanner": { - "vendor": "Github" + "vendor": "GitHub" }, "score": { "base": 5.3, @@ -734,7 +734,7 @@ "https://github.com/advisories/GHSA-rjqq-98f6-6j3r" ], "scanner": { - "vendor": "Github" + "vendor": "GitHub" }, "score": { "base": 5.3, @@ -846,7 +846,7 @@ "https://github.com/advisories/GHSA-9qrh-qjmc-5w2p" ], "scanner": { - "vendor": "Github" + "vendor": "GitHub" }, "score": { "base": 7.5, diff --git a/packages/github/data_stream/dependabot/elasticsearch/ingest_pipeline/default.yml b/packages/github/data_stream/dependabot/elasticsearch/ingest_pipeline/default.yml index 34e0371e621..1d33fb741ea 100644 --- a/packages/github/data_stream/dependabot/elasticsearch/ingest_pipeline/default.yml +++ b/packages/github/data_stream/dependabot/elasticsearch/ingest_pipeline/default.yml @@ -215,7 +215,7 @@ processors: ignore_missing: true - set: field: vulnerability.scanner.vendor - value: "Github" + value: "GitHub" - convert: field: github.dependabot.security_advisory.cvss.score type: float @@ -260,7 +260,7 @@ processors: source: >- ZonedDateTime start = ZonedDateTime.parse(ctx.event.start); ZonedDateTime end = ZonedDateTime.parse(ctx.event.end); ctx.event.duration = ChronoUnit.NANOS.between(start, end); ################################# - # For Github Overview Dashboard # + # For GitHub Overview Dashboard # ################################# - lowercase: field: github.dependabot.state diff --git a/packages/github/data_stream/dependabot/sample_event.json b/packages/github/data_stream/dependabot/sample_event.json index 812fac06446..a5a1c949b62 100644 --- a/packages/github/data_stream/dependabot/sample_event.json +++ b/packages/github/data_stream/dependabot/sample_event.json @@ -121,7 +121,7 @@ "https://nodesecurity.io/advisories/17" ], "scanner": { - "vendor": "Github" + "vendor": "GitHub" }, "score": { "base": 0 diff --git a/packages/github/data_stream/issues/_dev/test/pipeline/test-github-issues-json.log b/packages/github/data_stream/issues/_dev/test/pipeline/test-github-issues-json.log index 70bf6b982be..5cb5cb36a49 100644 --- a/packages/github/data_stream/issues/_dev/test/pipeline/test-github-issues-json.log +++ b/packages/github/data_stream/issues/_dev/test/pipeline/test-github-issues-json.log @@ -1,5 +1,5 @@ {"id":1,"node_id":"MDU6SXNzdWUx","url":"https://api.github.com/repos/octocat/Hello-World/issues/1347","repository_url":"https://api.github.com/repos/octocat/Hello-World","labels_url":"https://api.github.com/repos/octocat/Hello-World/issues/1347/labels{/name}","comments_url":"https://api.github.com/repos/octocat/Hello-World/issues/1347/comments","events_url":"https://api.github.com/repos/octocat/Hello-World/issues/1347/events","html_url":"https://github.com/octocat/Hello-World/issues/1347","number":1347,"state":"open","title":"Found a bug","body":"I'm having a problem with this.","user":{"login":"octocat","id":1,"node_id":"MDQ6VXNlcjE=","avatar_url":"https://github.com/images/error/octocat_happy.gif","gravatar_id":"","url":"https://api.github.com/users/octocat","html_url":"https://github.com/octocat","followers_url":"https://api.github.com/users/octocat/followers","following_url":"https://api.github.com/users/octocat/following{/other_user}","gists_url":"https://api.github.com/users/octocat/gists{/gist_id}","starred_url":"https://api.github.com/users/octocat/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/octocat/subscriptions","organizations_url":"https://api.github.com/users/octocat/orgs","repos_url":"https://api.github.com/users/octocat/repos","events_url":"https://api.github.com/users/octocat/events{/privacy}","received_events_url":"https://api.github.com/users/octocat/received_events","type":"User","site_admin":true},"labels":[{"id":208045946,"node_id":"MDU6TGFiZWwyMDgwNDU5NDY=","url":"https://api.github.com/repos/octocat/Hello-World/labels/bug","name":"bug","description":"Something isn't working","color":"f29513","default":true}],"assignee":{"login":"octocat","id":1,"node_id":"MDQ6VXNlcjE=","avatar_url":"https://github.com/images/error/octocat_happy.gif","gravatar_id":"","url":"https://api.github.com/users/octocat","html_url":"https://github.com/octocat","followers_url":"https://api.github.com/users/octocat/followers","following_url":"https://api.github.com/users/octocat/following{/other_user}","gists_url":"https://api.github.com/users/octocat/gists{/gist_id}","starred_url":"https://api.github.com/users/octocat/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/octocat/subscriptions","organizations_url":"https://api.github.com/users/octocat/orgs","repos_url":"https://api.github.com/users/octocat/repos","events_url":"https://api.github.com/users/octocat/events{/privacy}","received_events_url":"https://api.github.com/users/octocat/received_events","type":"User","site_admin":false},"assignees":[{"login":"octocat","id":1,"node_id":"MDQ6VXNlcjE=","avatar_url":"https://github.com/images/error/octocat_happy.gif","gravatar_id":"","url":"https://api.github.com/users/octocat","html_url":"https://github.com/octocat","followers_url":"https://api.github.com/users/octocat/followers","following_url":"https://api.github.com/users/octocat/following{/other_user}","gists_url":"https://api.github.com/users/octocat/gists{/gist_id}","starred_url":"https://api.github.com/users/octocat/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/octocat/subscriptions","organizations_url":"https://api.github.com/users/octocat/orgs","repos_url":"https://api.github.com/users/octocat/repos","events_url":"https://api.github.com/users/octocat/events{/privacy}","received_events_url":"https://api.github.com/users/octocat/received_events","type":"User","site_admin":false}],"milestone":{"url":"https://api.github.com/repos/octocat/Hello-World/milestones/1","html_url":"https://github.com/octocat/Hello-World/milestones/v1.0","labels_url":"https://api.github.com/repos/octocat/Hello-World/milestones/1/labels","id":1002604,"node_id":"MDk6TWlsZXN0b25lMTAwMjYwNA==","number":1,"state":"open","title":"v1.0","description":"Tracking milestone for version 1.0","creator":{"login":"octocat","id":1,"node_id":"MDQ6VXNlcjE=","avatar_url":"https://github.com/images/error/octocat_happy.gif","gravatar_id":"","url":"https://api.github.com/users/octocat","html_url":"https://github.com/octocat","followers_url":"https://api.github.com/users/octocat/followers","following_url":"https://api.github.com/users/octocat/following{/other_user}","gists_url":"https://api.github.com/users/octocat/gists{/gist_id}","starred_url":"https://api.github.com/users/octocat/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/octocat/subscriptions","organizations_url":"https://api.github.com/users/octocat/orgs","repos_url":"https://api.github.com/users/octocat/repos","events_url":"https://api.github.com/users/octocat/events{/privacy}","received_events_url":"https://api.github.com/users/octocat/received_events","type":"User","site_admin":false},"open_issues":4,"closed_issues":8,"created_at":"2011-04-10T20:09:31Z","updated_at":"2014-03-03T18:58:10Z","closed_at":"2013-02-12T13:22:01Z","due_on":"2012-10-09T23:39:01Z"},"locked":true,"active_lock_reason":"too heated","comments":0,"pull_request":{"url":"https://api.github.com/repos/octocat/Hello-World/pulls/1347","html_url":"https://github.com/octocat/Hello-World/pull/1347","diff_url":"https://github.com/octocat/Hello-World/pull/1347.diff","patch_url":"https://github.com/octocat/Hello-World/pull/1347.patch"},"closed_at":null,"created_at":"2011-04-22T13:33:48Z","updated_at":"2011-04-22T13:33:48Z","closed_by":{"login":"octocat","id":1,"node_id":"MDQ6VXNlcjE=","avatar_url":"https://github.com/images/error/octocat_happy.gif","gravatar_id":"","url":"https://api.github.com/users/octocat","html_url":"https://github.com/octocat","followers_url":"https://api.github.com/users/octocat/followers","following_url":"https://api.github.com/users/octocat/following{/other_user}","gists_url":"https://api.github.com/users/octocat/gists{/gist_id}","starred_url":"https://api.github.com/users/octocat/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/octocat/subscriptions","organizations_url":"https://api.github.com/users/octocat/orgs","repos_url":"https://api.github.com/users/octocat/repos","events_url":"https://api.github.com/users/octocat/events{/privacy}","received_events_url":"https://api.github.com/users/octocat/received_events","type":"User","site_admin":false},"repository": {"id":1296269,"node_id":"MDEwOlJlcG9zaXRvcnkxMjk2MjY5","name":"Hello-World","full_name":"octocat/Hello-World","owner":{"login":"octocat","id":1,"node_id":"MDQ6VXNlcjE=","avatar_url":"https://github.com/images/error/octocat_happy.gif","gravatar_id":"","url":"https://api.github.com/users/octocat","html_url":"https://github.com/octocat","followers_url":"https://api.github.com/users/octocat/followers","following_url":"https://api.github.com/users/octocat/following{/other_user}","gists_url":"https://api.github.com/users/octocat/gists{/gist_id}","starred_url":"https://api.github.com/users/octocat/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/octocat/subscriptions","organizations_url":"https://api.github.com/users/octocat/orgs","repos_url":"https://api.github.com/users/octocat/repos","events_url":"https://api.github.com/users/octocat/events{/privacy}","received_events_url":"https://api.github.com/users/octocat/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/octocat/Hello-World","description":"This your first repo!","fork":false,"url":"https://api.github.com/repos/octocat/Hello-World","archive_url":"https://api.github.com/repos/octocat/Hello-World/{archive_format}{/ref}","assignees_url":"https://api.github.com/repos/octocat/Hello-World/assignees{/user}","blobs_url":"https://api.github.com/repos/octocat/Hello-World/git/blobs{/sha}","branches_url":"https://api.github.com/repos/octocat/Hello-World/branches{/branch}","collaborators_url":"https://api.github.com/repos/octocat/Hello-World/collaborators{/collaborator}","comments_url":"https://api.github.com/repos/octocat/Hello-World/comments{/number}","commits_url":"https://api.github.com/repos/octocat/Hello-World/commits{/sha}","compare_url":"https://api.github.com/repos/octocat/Hello-World/compare/{base}...{head}","contents_url":"https://api.github.com/repos/octocat/Hello-World/contents/{+path}","contributors_url":"https://api.github.com/repos/octocat/Hello-World/contributors","deployments_url":"https://api.github.com/repos/octocat/Hello-World/deployments","downloads_url":"https://api.github.com/repos/octocat/Hello-World/downloads","events_url":"https://api.github.com/repos/octocat/Hello-World/events","forks_url":"https://api.github.com/repos/octocat/Hello-World/forks","git_commits_url":"https://api.github.com/repos/octocat/Hello-World/git/commits{/sha}","git_refs_url":"https://api.github.com/repos/octocat/Hello-World/git/refs{/sha}","git_tags_url":"https://api.github.com/repos/octocat/Hello-World/git/tags{/sha}","git_url":"git:github.com/octocat/Hello-World.git","issue_comment_url":"https://api.github.com/repos/octocat/Hello-World/issues/comments{/number}","issue_events_url":"https://api.github.com/repos/octocat/Hello-World/issues/events{/number}","issues_url":"https://api.github.com/repos/octocat/Hello-World/issues{/number}","keys_url":"https://api.github.com/repos/octocat/Hello-World/keys{/key_id}","labels_url":"https://api.github.com/repos/octocat/Hello-World/labels{/name}","languages_url":"https://api.github.com/repos/octocat/Hello-World/languages","merges_url":"https://api.github.com/repos/octocat/Hello-World/merges","milestones_url":"https://api.github.com/repos/octocat/Hello-World/milestones{/number}","notifications_url":"https://api.github.com/repos/octocat/Hello-World/notifications{?since,all,participating}","pulls_url":"https://api.github.com/repos/octocat/Hello-World/pulls{/number}","releases_url":"https://api.github.com/repos/octocat/Hello-World/releases{/id}","ssh_url":"git@github.com:octocat/Hello-World.git","stargazers_url":"https://api.github.com/repos/octocat/Hello-World/stargazers","statuses_url":"https://api.github.com/repos/octocat/Hello-World/statuses/{sha}","subscribers_url":"https://api.github.com/repos/octocat/Hello-World/subscribers","subscription_url":"https://api.github.com/repos/octocat/Hello-World/subscription","tags_url":"https://api.github.com/repos/octocat/Hello-World/tags","teams_url":"https://api.github.com/repos/octocat/Hello-World/teams","trees_url":"https://api.github.com/repos/octocat/Hello-World/git/trees{/sha}","clone_url":"https://github.com/octocat/Hello-World.git","mirror_url":"git:git.example.com/octocat/Hello-World","hooks_url":"https://api.github.com/repos/octocat/Hello-World/hooks","svn_url":"https://svn.github.com/octocat/Hello-World","homepage":"https://github.com","language":null,"forks_count":9,"stargazers_count":80,"watchers_count":80,"size":108,"default_branch":"master","open_issues_count":0,"is_template":true,"topics":["octocat","atom","electron","api"],"has_issues":true,"has_projects":true,"has_wiki":true,"has_pages":false,"has_downloads":true,"archived":false,"disabled":false,"visibility":"public","pushed_at":"2011-01-26T19:06:43Z","created_at":"2011-01-26T19:01:12Z","updated_at":"2011-01-26T19:14:43Z","permissions":{"admin":false,"push":false,"pull":true},"allow_rebase_merge":true,"template_repository":null,"temp_clone_token":"ABTLWHOULUVAXGTRYU7OC2876QJ2O","allow_squash_merge":true,"allow_auto_merge":false,"delete_branch_on_merge":true,"allow_merge_commit":true,"subscribers_count":42,"network_count":0,"license":{"key":"mit","name":"MIT License","url":"https://api.github.com/licenses/mit","spdx_id":"MIT","node_id":"MDc6TGljZW5zZW1pdA==","html_url":"https://github.com/licenses/mit"},"forks":1,"open_issues":1,"watchers":1},"author_association":"COLLABORATOR","state_reason":"completed"} -{"url":"https://api.github.com/repos/elastic/integrations/issues/4710","repository_url":"https://api.github.com/repos/elastic/integrations","labels_url":"https://api.github.com/repos/elastic/integrations/issues/4710/labels{/name}","comments_url":"https://api.github.com/repos/elastic/integrations/issues/4710/comments","events_url":"https://api.github.com/repos/elastic/integrations/issues/4710/events","html_url":"https://github.com/elastic/integrations/issues/4710","id":1461928292,"node_id":"I_kwDODAw23M5XI0Fk","number":4710,"title":"Custom STIX Package","user":{"login":"jamiehynds","id":62879768,"node_id":"MDQ6VXNlcjYyODc5NzY4","avatar_url":"https://avatars.githubusercontent.com/u/62879768?v=4","gravatar_id":"","url":"https://api.github.com/users/jamiehynds","html_url":"https://github.com/jamiehynds","followers_url":"https://api.github.com/users/jamiehynds/followers","following_url":"https://api.github.com/users/jamiehynds/following{/other_user}","gists_url":"https://api.github.com/users/jamiehynds/gists{/gist_id}","starred_url":"https://api.github.com/users/jamiehynds/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jamiehynds/subscriptions","organizations_url":"https://api.github.com/users/jamiehynds/orgs","repos_url":"https://api.github.com/users/jamiehynds/repos","events_url":"https://api.github.com/users/jamiehynds/events{/privacy}","received_events_url":"https://api.github.com/users/jamiehynds/received_events","type":"User","site_admin":false},"labels":[{"id":2404921703,"node_id":"MDU6TGFiZWwyNDA0OTIxNzAz","url":"https://api.github.com/repos/elastic/integrations/labels/Team:Security-Service%20Integrations","name":"Team:Security-Service Integrations","color":"1d76db","default":false,"description":"Label for the Security External Integrations team"},{"id":3104073484,"node_id":"MDU6TGFiZWwzMTA0MDczNDg0","url":"https://api.github.com/repos/elastic/integrations/labels/Integration:Threat%20Intel","name":"Integration:Threat Intel","color":"ffffff","default":false,"description":""}],"state":"open","locked":false,"assignee":{"login":"P1llus","id":8027539,"node_id":"MDQ6VXNlcjgwMjc1Mzk=","avatar_url":"https://avatars.githubusercontent.com/u/8027539?v=4","gravatar_id":"","url":"https://api.github.com/users/P1llus","html_url":"https://github.com/P1llus","followers_url":"https://api.github.com/users/P1llus/followers","following_url":"https://api.github.com/users/P1llus/following{/other_user}","gists_url":"https://api.github.com/users/P1llus/gists{/gist_id}","starred_url":"https://api.github.com/users/P1llus/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/P1llus/subscriptions","organizations_url":"https://api.github.com/users/P1llus/orgs","repos_url":"https://api.github.com/users/P1llus/repos","events_url":"https://api.github.com/users/P1llus/events{/privacy}","received_events_url":"https://api.github.com/users/P1llus/received_events","type":"User","site_admin":false},"assignees":[{"login":"P1llus","id":8027539,"node_id":"MDQ6VXNlcjgwMjc1Mzk=","avatar_url":"https://avatars.githubusercontent.com/u/8027539?v=4","gravatar_id":"","url":"https://api.github.com/users/P1llus","html_url":"https://github.com/P1llus","followers_url":"https://api.github.com/users/P1llus/followers","following_url":"https://api.github.com/users/P1llus/following{/other_user}","gists_url":"https://api.github.com/users/P1llus/gists{/gist_id}","starred_url":"https://api.github.com/users/P1llus/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/P1llus/subscriptions","organizations_url":"https://api.github.com/users/P1llus/orgs","repos_url":"https://api.github.com/users/P1llus/repos","events_url":"https://api.github.com/users/P1llus/events{/privacy}","received_events_url":"https://api.github.com/users/P1llus/received_events","type":"User","site_admin":false}],"milestone":null,"comments":2,"created_at":"2022-11-23T15:06:34Z","updated_at":"2022-11-23T15:07:18Z","closed_at":null,"author_association":"NONE","active_lock_reason":null,"body":"Structured Threat Information Expression (STIX) is a language for expressing cyber threat and observable information. While we have several Threat Intel integrations which map STIX formatted data to Elastic Common Schema, users will always have need to ingest IOC's from threat feeds that we don't support out of the box. 'How do I ingest STIX feeds' remains a very common questions across community Slack, Discuss, Github, etc. A custom package would solve for this. \r\n\r\nTo allow for the broad range of STIX formatted feeds, we should provide a way for users to ingest data from ANY STIX feed, via a 'Custom STIX' package. The package will leverage our httpjson input under the hood, but include an ingest pipeline which maps STIX fields to ECS (we expect there'll still be a need for custom fields, as not all STIX fields have a corresponding field in ECS). \r\n\r\nThere may be cases where some feeds/vendors don't strictly conform to STIX, and in those cases, users may have to modify our pipeline and that's ok. We'll focus on correctly formatted STIX data. ","reactions":{"url":"https://api.github.com/repos/elastic/integrations/issues/4710/reactions","total_count":0,"+1":0,"-1":0,"laugh":0,"hooray":0,"confused":0,"heart":0,"rocket":0,"eyes":0},"timeline_url":"https://api.github.com/repos/elastic/integrations/issues/4710/timeline","performed_via_github_app":null,"state_reason":null} +{"url":"https://api.github.com/repos/elastic/integrations/issues/4710","repository_url":"https://api.github.com/repos/elastic/integrations","labels_url":"https://api.github.com/repos/elastic/integrations/issues/4710/labels{/name}","comments_url":"https://api.github.com/repos/elastic/integrations/issues/4710/comments","events_url":"https://api.github.com/repos/elastic/integrations/issues/4710/events","html_url":"https://github.com/elastic/integrations/issues/4710","id":1461928292,"node_id":"I_kwDODAw23M5XI0Fk","number":4710,"title":"Custom STIX Package","user":{"login":"jamiehynds","id":62879768,"node_id":"MDQ6VXNlcjYyODc5NzY4","avatar_url":"https://avatars.githubusercontent.com/u/62879768?v=4","gravatar_id":"","url":"https://api.github.com/users/jamiehynds","html_url":"https://github.com/jamiehynds","followers_url":"https://api.github.com/users/jamiehynds/followers","following_url":"https://api.github.com/users/jamiehynds/following{/other_user}","gists_url":"https://api.github.com/users/jamiehynds/gists{/gist_id}","starred_url":"https://api.github.com/users/jamiehynds/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jamiehynds/subscriptions","organizations_url":"https://api.github.com/users/jamiehynds/orgs","repos_url":"https://api.github.com/users/jamiehynds/repos","events_url":"https://api.github.com/users/jamiehynds/events{/privacy}","received_events_url":"https://api.github.com/users/jamiehynds/received_events","type":"User","site_admin":false},"labels":[{"id":2404921703,"node_id":"MDU6TGFiZWwyNDA0OTIxNzAz","url":"https://api.github.com/repos/elastic/integrations/labels/Team:Security-Service%20Integrations","name":"Team:Security-Service Integrations","color":"1d76db","default":false,"description":"Label for the Security External Integrations team"},{"id":3104073484,"node_id":"MDU6TGFiZWwzMTA0MDczNDg0","url":"https://api.github.com/repos/elastic/integrations/labels/Integration:Threat%20Intel","name":"Integration:Threat Intel","color":"ffffff","default":false,"description":""}],"state":"open","locked":false,"assignee":{"login":"P1llus","id":8027539,"node_id":"MDQ6VXNlcjgwMjc1Mzk=","avatar_url":"https://avatars.githubusercontent.com/u/8027539?v=4","gravatar_id":"","url":"https://api.github.com/users/P1llus","html_url":"https://github.com/P1llus","followers_url":"https://api.github.com/users/P1llus/followers","following_url":"https://api.github.com/users/P1llus/following{/other_user}","gists_url":"https://api.github.com/users/P1llus/gists{/gist_id}","starred_url":"https://api.github.com/users/P1llus/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/P1llus/subscriptions","organizations_url":"https://api.github.com/users/P1llus/orgs","repos_url":"https://api.github.com/users/P1llus/repos","events_url":"https://api.github.com/users/P1llus/events{/privacy}","received_events_url":"https://api.github.com/users/P1llus/received_events","type":"User","site_admin":false},"assignees":[{"login":"P1llus","id":8027539,"node_id":"MDQ6VXNlcjgwMjc1Mzk=","avatar_url":"https://avatars.githubusercontent.com/u/8027539?v=4","gravatar_id":"","url":"https://api.github.com/users/P1llus","html_url":"https://github.com/P1llus","followers_url":"https://api.github.com/users/P1llus/followers","following_url":"https://api.github.com/users/P1llus/following{/other_user}","gists_url":"https://api.github.com/users/P1llus/gists{/gist_id}","starred_url":"https://api.github.com/users/P1llus/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/P1llus/subscriptions","organizations_url":"https://api.github.com/users/P1llus/orgs","repos_url":"https://api.github.com/users/P1llus/repos","events_url":"https://api.github.com/users/P1llus/events{/privacy}","received_events_url":"https://api.github.com/users/P1llus/received_events","type":"User","site_admin":false}],"milestone":null,"comments":2,"created_at":"2022-11-23T15:06:34Z","updated_at":"2022-11-23T15:07:18Z","closed_at":null,"author_association":"NONE","active_lock_reason":null,"body":"Structured Threat Information Expression (STIX) is a language for expressing cyber threat and observable information. While we have several Threat Intel integrations which map STIX formatted data to Elastic Common Schema, users will always have need to ingest IOC's from threat feeds that we don't support out of the box. 'How do I ingest STIX feeds' remains a very common questions across community Slack, Discuss, GitHub, etc. A custom package would solve for this. \r\n\r\nTo allow for the broad range of STIX formatted feeds, we should provide a way for users to ingest data from ANY STIX feed, via a 'Custom STIX' package. The package will leverage our httpjson input under the hood, but include an ingest pipeline which maps STIX fields to ECS (we expect there'll still be a need for custom fields, as not all STIX fields have a corresponding field in ECS). \r\n\r\nThere may be cases where some feeds/vendors don't strictly conform to STIX, and in those cases, users may have to modify our pipeline and that's ok. We'll focus on correctly formatted STIX data. ","reactions":{"url":"https://api.github.com/repos/elastic/integrations/issues/4710/reactions","total_count":0,"+1":0,"-1":0,"laugh":0,"hooray":0,"confused":0,"heart":0,"rocket":0,"eyes":0},"timeline_url":"https://api.github.com/repos/elastic/integrations/issues/4710/timeline","performed_via_github_app":null,"state_reason":null} {"url":"https://api.github.com/repos/elastic/integrations/issues/4707","repository_url":"https://api.github.com/repos/elastic/integrations","labels_url":"https://api.github.com/repos/elastic/integrations/issues/4707/labels{/name}","comments_url":"https://api.github.com/repos/elastic/integrations/issues/4707/comments","events_url":"https://api.github.com/repos/elastic/integrations/issues/4707/events","html_url":"https://github.com/elastic/integrations/issues/4707","id":1461726255,"node_id":"I_kwDODAw23M5XICwv","number":4707,"title":"[ Crowdstrike Falcon ] Parse of process field is wrong.","user":{"login":"leandrojmp","id":322886,"node_id":"MDQ6VXNlcjMyMjg4Ng==","avatar_url":"https://avatars.githubusercontent.com/u/322886?v=4","gravatar_id":"","url":"https://api.github.com/users/leandrojmp","html_url":"https://github.com/leandrojmp","followers_url":"https://api.github.com/users/leandrojmp/followers","following_url":"https://api.github.com/users/leandrojmp/following{/other_user}","gists_url":"https://api.github.com/users/leandrojmp/gists{/gist_id}","starred_url":"https://api.github.com/users/leandrojmp/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/leandrojmp/subscriptions","organizations_url":"https://api.github.com/users/leandrojmp/orgs","repos_url":"https://api.github.com/users/leandrojmp/repos","events_url":"https://api.github.com/users/leandrojmp/events{/privacy}","received_events_url":"https://api.github.com/users/leandrojmp/received_events","type":"User","site_admin":false},"labels":[{"id":2404921703,"node_id":"MDU6TGFiZWwyNDA0OTIxNzAz","url":"https://api.github.com/repos/elastic/integrations/labels/Team:Security-Service%20Integrations","name":"Team:Security-Service Integrations","color":"1d76db","default":false,"description":"Label for the Security External Integrations team"},{"id":2716642190,"node_id":"MDU6TGFiZWwyNzE2NjQyMTkw","url":"https://api.github.com/repos/elastic/integrations/labels/Integration:Crowdstrike","name":"Integration:Crowdstrike","color":"FFFFFF","default":false,"description":null}],"state":"open","locked":false,"assignee":null,"assignees":[],"milestone":null,"comments":1,"created_at":"2022-11-23T13:03:54Z","updated_at":"2022-11-23T14:58:03Z","closed_at":null,"author_association":"NONE","active_lock_reason":null,"body":"Hello,\r\n\r\nThe parse of the process fields in the Crowdstrike Falcon Pipeline is wrong, it is creating fields with dots in the name instead of nested json objects.\r\n\r\nIt creates this:\r\n```\r\n{\r\n \"process.executable\": \"value\"\r\n}\r\n```\r\n\r\nInstead of\r\n\r\n```\r\n{\r\n \"process\": {\r\n \"executable\": \"value\"\r\n }\r\n}\r\n```\r\n\r\nThis same issue was present in the [filebeat integration](https://github.com/elastic/beats/issues/27622) and I submitted a [fix](https://github.com/elastic/beats/pull/27623) more than an year ago.\r\n\r\nI will submit a PR later with the same fix, but I'm creating this issue to register the problem.","reactions":{"url":"https://api.github.com/repos/elastic/integrations/issues/4707/reactions","total_count":0,"+1":0,"-1":0,"laugh":0,"hooray":0,"confused":0,"heart":0,"rocket":0,"eyes":0},"timeline_url":"https://api.github.com/repos/elastic/integrations/issues/4707/timeline","performed_via_github_app":null,"state_reason":null } {"url":"https://api.github.com/repos/elastic/integrations/issues/4704","repository_url":"https://api.github.com/repos/elastic/integrations","labels_url":"https://api.github.com/repos/elastic/integrations/issues/4704/labels{/name}","comments_url":"https://api.github.com/repos/elastic/integrations/issues/4704/comments","events_url":"https://api.github.com/repos/elastic/integrations/issues/4704/events","html_url":"https://github.com/elastic/integrations/pull/4704","id":1461524465,"node_id":"PR_kwDODAw23M5DjMOh","number":4704,"title":"[Enhancement] [Infoblox Bloxone DDI] Update the Pagination Termination Condition and Added Filter instead of KQL in visualizations","user":{"login":"vinit-elastic","id":89848047,"node_id":"MDQ6VXNlcjg5ODQ4MDQ3","avatar_url":"https://avatars.githubusercontent.com/u/89848047?v=4","gravatar_id":"","url":"https://api.github.com/users/vinit-elastic","html_url":"https://github.com/vinit-elastic","followers_url":"https://api.github.com/users/vinit-elastic/followers","following_url":"https://api.github.com/users/vinit-elastic/following{/other_user}","gists_url":"https://api.github.com/users/vinit-elastic/gists{/gist_id}","starred_url":"https://api.github.com/users/vinit-elastic/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/vinit-elastic/subscriptions","organizations_url":"https://api.github.com/users/vinit-elastic/orgs","repos_url":"https://api.github.com/users/vinit-elastic/repos","events_url":"https://api.github.com/users/vinit-elastic/events{/privacy}","received_events_url":"https://api.github.com/users/vinit-elastic/received_events","type":"User","site_admin":false},"labels":[{"id":1498531540,"node_id":"MDU6TGFiZWwxNDk4NTMxNTQw","url":"https://api.github.com/repos/elastic/integrations/labels/enhancement","name":"enhancement","color":"a2eeef","default":true,"description":"New feature or request"},{"id":2404921703,"node_id":"MDU6TGFiZWwyNDA0OTIxNzAz","url":"https://api.github.com/repos/elastic/integrations/labels/Team:Security-Service%20Integrations","name":"Team:Security-Service Integrations","color":"1d76db","default":false,"description":"Label for the Security External Integrations team"},{"id":4528694847,"node_id":"LA_kwDODAw23M8AAAABDe5mPw","url":"https://api.github.com/repos/elastic/integrations/labels/Integration:infoblox_bloxone_ddi","name":"Integration:infoblox_bloxone_ddi","color":"FFFFFF","default":false,"description":"Infoblox BloxOne DDI (DNS, DHCP, IP management)"}],"state":"open","locked":false,"assignee":null,"assignees":[],"milestone":null,"comments":4,"created_at":"2022-11-23T10:57:54Z","updated_at":"2022-11-23T11:15:48Z","closed_at":null,"author_association":"COLLABORATOR","active_lock_reason":null,"draft":false,"pull_request":{"url":"https://api.github.com/repos/elastic/integrations/pulls/4704","html_url":"https://github.com/elastic/integrations/pull/4704","diff_url":"https://github.com/elastic/integrations/pull/4704.diff","patch_url":"https://github.com/elastic/integrations/pull/4704.patch","merged_at":null},"body":" Type of change\r\n- Enhancement\r\n\r\n\r\n## What does this PR do?\r\nUpdate the Pagination Termination Condition for Infoblox Bloxone DDI connector.\r\n\r\nCurrent condition for pagination termination contains `[[else]][[.last_response.terminate_pagination]][[end]]` which results in error logs when pagination is completed.\r\n\r\nRemoving this `else` condition will not result in error logs.\r\n\r\n\r\n\r\n## Checklist\r\n\r\n- [x] I have reviewed [tips for building integrations](https://github.com/elastic/integrations/blob/main/docs/tips_for_building_integrations.md) and this pull request is aligned with them.\r\n- [x] I have verified that all data streams collect metrics or logs.\r\n- [x] I have added an entry to my package's `changelog.yml` file.\r\n- [x] I have verified that Kibana version constraints are current according to [guidelines](https://github.com/elastic/elastic-package/blob/master/docs/howto/stack_version_support.md#when-to-update-the-condition).\r\n\r\n\r\n\r\n## How to test this PR locally\r\n- Clone integrations repo.\r\n- Install elastic package locally.\r\n- Start elastic stack using elastic-package.\r\n- Move to integrations/packages/infoblox_bloxone_ddi directory.\r\n- Run the following command to run tests. \r\n> elastic-package test \r\n\r\n\r\n## Related issues\r\n\r\n\r\n- Relates https://github.com/elastic/integrations/issues/4527\r\n\r\n## Screenshots\r\n![image](https://user-images.githubusercontent.com/89848047/203529720-18c110cd-5343-4d70-bbfb-eed0b81313af.png)\r\n![image](https://user-images.githubusercontent.com/89848047/203529765-4b183071-6ed9-4bc8-8a98-fd04955bb4d7.png)\r\n![image](https://user-images.githubusercontent.com/89848047/203529819-78128d98-630e-49e5-bf97-f86059c0cc26.png)\r\n![image](https://user-images.githubusercontent.com/89848047/203529936-df7e3d6f-031e-4b0c-992f-93707a507735.png)\r\n\r\n\r\n","reactions":{"url":"https://api.github.com/repos/elastic/integrations/issues/4704/reactions","total_count":0,"+1":0,"-1":0,"laugh":0,"hooray":0,"confused":0,"heart":0,"rocket":0,"eyes":0},"timeline_url":"https://api.github.com/repos/elastic/integrations/issues/4704/timeline","performed_via_github_app":null,"state_reason":null } {"url":"https://api.github.com/repos/elastic/integrations/issues/4703","repository_url":"https://api.github.com/repos/elastic/integrations","labels_url":"https://api.github.com/repos/elastic/integrations/issues/4703/labels{/name}","comments_url":"https://api.github.com/repos/elastic/integrations/issues/4703/comments","events_url":"https://api.github.com/repos/elastic/integrations/issues/4703/events","html_url":"https://github.com/elastic/integrations/pull/4703","id":1461503410,"node_id":"PR_kwDODAw23M5DjHjk","number":4703,"title":"[Enhancement] [AWS Security Hub] Update the Pagination Termination Condition","user":{"login":"vinit-elastic","id":89848047,"node_id":"MDQ6VXNlcjg5ODQ4MDQ3","avatar_url":"https://avatars.githubusercontent.com/u/89848047?v=4","gravatar_id":"","url":"https://api.github.com/users/vinit-elastic","html_url":"https://github.com/vinit-elastic","followers_url":"https://api.github.com/users/vinit-elastic/followers","following_url":"https://api.github.com/users/vinit-elastic/following{/other_user}","gists_url":"https://api.github.com/users/vinit-elastic/gists{/gist_id}","starred_url":"https://api.github.com/users/vinit-elastic/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/vinit-elastic/subscriptions","organizations_url":"https://api.github.com/users/vinit-elastic/orgs","repos_url":"https://api.github.com/users/vinit-elastic/repos","events_url":"https://api.github.com/users/vinit-elastic/events{/privacy}","received_events_url":"https://api.github.com/users/vinit-elastic/received_events","type":"User","site_admin":false},"labels":[{"id":1498531540,"node_id":"MDU6TGFiZWwxNDk4NTMxNTQw","url":"https://api.github.com/repos/elastic/integrations/labels/enhancement","name":"enhancement","color":"a2eeef","default":true,"description":"New feature or request"},{"id":2404921703,"node_id":"MDU6TGFiZWwyNDA0OTIxNzAz","url":"https://api.github.com/repos/elastic/integrations/labels/Team:Security-Service%20Integrations","name":"Team:Security-Service Integrations","color":"1d76db","default":false,"description":"Label for the Security External Integrations team"},{"id":2607750240,"node_id":"MDU6TGFiZWwyNjA3NzUwMjQw","url":"https://api.github.com/repos/elastic/integrations/labels/Integration:AWS","name":"Integration:AWS","color":"FFFFFF","default":false,"description":""}],"state":"open","locked":false,"assignee":null,"assignees":[],"milestone":null,"comments":4,"created_at":"2022-11-23T10:44:59Z","updated_at":"2022-11-23T11:23:56Z","closed_at":null,"author_association":"COLLABORATOR","active_lock_reason":null,"draft":false,"pull_request":{"url":"https://api.github.com/repos/elastic/integrations/pulls/4703","html_url":"https://github.com/elastic/integrations/pull/4703","diff_url":"https://github.com/elastic/integrations/pull/4703.diff","patch_url":"https://github.com/elastic/integrations/pull/4703.patch","merged_at":null},"body":" Type of change\r\n- Enhancement\r\n\r\n\r\n## What does this PR do?\r\nUpdate the Pagination Termination Condition for AWS Security Hub connector.\r\n\r\nCurrent condition for pagination termination contains `[[else]][[.last_response.terminate_pagination]][[end]]` which results in error logs when pagination is completed.\r\n\r\nRemoving this `else` condition will not result in error logs.\r\n\r\n\r\n\r\n## Checklist\r\n\r\n- [x] I have reviewed [tips for building integrations](https://github.com/elastic/integrations/blob/main/docs/tips_for_building_integrations.md) and this pull request is aligned with them.\r\n- [x] I have verified that all data streams collect metrics or logs.\r\n- [x] I have added an entry to my package's `changelog.yml` file.\r\n- [x] I have verified that Kibana version constraints are current according to [guidelines](https://github.com/elastic/elastic-package/blob/master/docs/howto/stack_version_support.md#when-to-update-the-condition).\r\n\r\n\r\n\r\n## How to test this PR locally\r\n- Clone integrations repo.\r\n- Install elastic package locally.\r\n- Start elastic stack using elastic-package.\r\n- Move to integrations/packages/aws directory.\r\n- Run the following command to run tests. \r\n> elastic-package test \r\n\r\n\r\n## Related issues\r\n\r\n\r\n- Relates https://github.com/elastic/integrations/issues/4527\r\n\r\n## Screenshots\r\n![image](https://user-images.githubusercontent.com/89848047/203526900-06519c95-3de5-4f88-a65d-555d03d7f9a1.png)\r\n![image](https://user-images.githubusercontent.com/89848047/203527000-b050f059-03c6-4ead-a88e-9f2e42efb537.png)\r\n![image](https://user-images.githubusercontent.com/89848047/203527123-ea1f8513-8371-4d06-8a17-b71c0dae7a5a.png)\r\n\r\n\r\n","reactions":{"url":"https://api.github.com/repos/elastic/integrations/issues/4703/reactions","total_count":0,"+1":0,"-1":0,"laugh":0,"hooray":0,"confused":0,"heart":0,"rocket":0,"eyes":0},"timeline_url":"https://api.github.com/repos/elastic/integrations/issues/4703/timeline","performed_via_github_app":null,"state_reason":null } \ No newline at end of file diff --git a/packages/github/data_stream/issues/_dev/test/pipeline/test-github-issues-json.log-expected.json b/packages/github/data_stream/issues/_dev/test/pipeline/test-github-issues-json.log-expected.json index 50b06599fc1..aeb5bd586e3 100644 --- a/packages/github/data_stream/issues/_dev/test/pipeline/test-github-issues-json.log-expected.json +++ b/packages/github/data_stream/issues/_dev/test/pipeline/test-github-issues-json.log-expected.json @@ -114,7 +114,7 @@ }, "event": { "created": "2022-11-23T15:07:18.000Z", - "original": "{\"url\":\"https://api.github.com/repos/elastic/integrations/issues/4710\",\"repository_url\":\"https://api.github.com/repos/elastic/integrations\",\"labels_url\":\"https://api.github.com/repos/elastic/integrations/issues/4710/labels{/name}\",\"comments_url\":\"https://api.github.com/repos/elastic/integrations/issues/4710/comments\",\"events_url\":\"https://api.github.com/repos/elastic/integrations/issues/4710/events\",\"html_url\":\"https://github.com/elastic/integrations/issues/4710\",\"id\":1461928292,\"node_id\":\"I_kwDODAw23M5XI0Fk\",\"number\":4710,\"title\":\"Custom STIX Package\",\"user\":{\"login\":\"jamiehynds\",\"id\":62879768,\"node_id\":\"MDQ6VXNlcjYyODc5NzY4\",\"avatar_url\":\"https://avatars.githubusercontent.com/u/62879768?v=4\",\"gravatar_id\":\"\",\"url\":\"https://api.github.com/users/jamiehynds\",\"html_url\":\"https://github.com/jamiehynds\",\"followers_url\":\"https://api.github.com/users/jamiehynds/followers\",\"following_url\":\"https://api.github.com/users/jamiehynds/following{/other_user}\",\"gists_url\":\"https://api.github.com/users/jamiehynds/gists{/gist_id}\",\"starred_url\":\"https://api.github.com/users/jamiehynds/starred{/owner}{/repo}\",\"subscriptions_url\":\"https://api.github.com/users/jamiehynds/subscriptions\",\"organizations_url\":\"https://api.github.com/users/jamiehynds/orgs\",\"repos_url\":\"https://api.github.com/users/jamiehynds/repos\",\"events_url\":\"https://api.github.com/users/jamiehynds/events{/privacy}\",\"received_events_url\":\"https://api.github.com/users/jamiehynds/received_events\",\"type\":\"User\",\"site_admin\":false},\"labels\":[{\"id\":2404921703,\"node_id\":\"MDU6TGFiZWwyNDA0OTIxNzAz\",\"url\":\"https://api.github.com/repos/elastic/integrations/labels/Team:Security-Service%20Integrations\",\"name\":\"Team:Security-Service Integrations\",\"color\":\"1d76db\",\"default\":false,\"description\":\"Label for the Security External Integrations team\"},{\"id\":3104073484,\"node_id\":\"MDU6TGFiZWwzMTA0MDczNDg0\",\"url\":\"https://api.github.com/repos/elastic/integrations/labels/Integration:Threat%20Intel\",\"name\":\"Integration:Threat Intel\",\"color\":\"ffffff\",\"default\":false,\"description\":\"\"}],\"state\":\"open\",\"locked\":false,\"assignee\":{\"login\":\"P1llus\",\"id\":8027539,\"node_id\":\"MDQ6VXNlcjgwMjc1Mzk=\",\"avatar_url\":\"https://avatars.githubusercontent.com/u/8027539?v=4\",\"gravatar_id\":\"\",\"url\":\"https://api.github.com/users/P1llus\",\"html_url\":\"https://github.com/P1llus\",\"followers_url\":\"https://api.github.com/users/P1llus/followers\",\"following_url\":\"https://api.github.com/users/P1llus/following{/other_user}\",\"gists_url\":\"https://api.github.com/users/P1llus/gists{/gist_id}\",\"starred_url\":\"https://api.github.com/users/P1llus/starred{/owner}{/repo}\",\"subscriptions_url\":\"https://api.github.com/users/P1llus/subscriptions\",\"organizations_url\":\"https://api.github.com/users/P1llus/orgs\",\"repos_url\":\"https://api.github.com/users/P1llus/repos\",\"events_url\":\"https://api.github.com/users/P1llus/events{/privacy}\",\"received_events_url\":\"https://api.github.com/users/P1llus/received_events\",\"type\":\"User\",\"site_admin\":false},\"assignees\":[{\"login\":\"P1llus\",\"id\":8027539,\"node_id\":\"MDQ6VXNlcjgwMjc1Mzk=\",\"avatar_url\":\"https://avatars.githubusercontent.com/u/8027539?v=4\",\"gravatar_id\":\"\",\"url\":\"https://api.github.com/users/P1llus\",\"html_url\":\"https://github.com/P1llus\",\"followers_url\":\"https://api.github.com/users/P1llus/followers\",\"following_url\":\"https://api.github.com/users/P1llus/following{/other_user}\",\"gists_url\":\"https://api.github.com/users/P1llus/gists{/gist_id}\",\"starred_url\":\"https://api.github.com/users/P1llus/starred{/owner}{/repo}\",\"subscriptions_url\":\"https://api.github.com/users/P1llus/subscriptions\",\"organizations_url\":\"https://api.github.com/users/P1llus/orgs\",\"repos_url\":\"https://api.github.com/users/P1llus/repos\",\"events_url\":\"https://api.github.com/users/P1llus/events{/privacy}\",\"received_events_url\":\"https://api.github.com/users/P1llus/received_events\",\"type\":\"User\",\"site_admin\":false}],\"milestone\":null,\"comments\":2,\"created_at\":\"2022-11-23T15:06:34Z\",\"updated_at\":\"2022-11-23T15:07:18Z\",\"closed_at\":null,\"author_association\":\"NONE\",\"active_lock_reason\":null,\"body\":\"Structured Threat Information Expression (STIX) is a language for expressing cyber threat and observable information. While we have several Threat Intel integrations which map STIX formatted data to Elastic Common Schema, users will always have need to ingest IOC's from threat feeds that we don't support out of the box. 'How do I ingest STIX feeds' remains a very common questions across community Slack, Discuss, Github, etc. A custom package would solve for this. \\r\\n\\r\\nTo allow for the broad range of STIX formatted feeds, we should provide a way for users to ingest data from ANY STIX feed, via a 'Custom STIX' package. The package will leverage our httpjson input under the hood, but include an ingest pipeline which maps STIX fields to ECS (we expect there'll still be a need for custom fields, as not all STIX fields have a corresponding field in ECS). \\r\\n\\r\\nThere may be cases where some feeds/vendors don't strictly conform to STIX, and in those cases, users may have to modify our pipeline and that's ok. We'll focus on correctly formatted STIX data. \",\"reactions\":{\"url\":\"https://api.github.com/repos/elastic/integrations/issues/4710/reactions\",\"total_count\":0,\"+1\":0,\"-1\":0,\"laugh\":0,\"hooray\":0,\"confused\":0,\"heart\":0,\"rocket\":0,\"eyes\":0},\"timeline_url\":\"https://api.github.com/repos/elastic/integrations/issues/4710/timeline\",\"performed_via_github_app\":null,\"state_reason\":null}", + "original": "{\"url\":\"https://api.github.com/repos/elastic/integrations/issues/4710\",\"repository_url\":\"https://api.github.com/repos/elastic/integrations\",\"labels_url\":\"https://api.github.com/repos/elastic/integrations/issues/4710/labels{/name}\",\"comments_url\":\"https://api.github.com/repos/elastic/integrations/issues/4710/comments\",\"events_url\":\"https://api.github.com/repos/elastic/integrations/issues/4710/events\",\"html_url\":\"https://github.com/elastic/integrations/issues/4710\",\"id\":1461928292,\"node_id\":\"I_kwDODAw23M5XI0Fk\",\"number\":4710,\"title\":\"Custom STIX Package\",\"user\":{\"login\":\"jamiehynds\",\"id\":62879768,\"node_id\":\"MDQ6VXNlcjYyODc5NzY4\",\"avatar_url\":\"https://avatars.githubusercontent.com/u/62879768?v=4\",\"gravatar_id\":\"\",\"url\":\"https://api.github.com/users/jamiehynds\",\"html_url\":\"https://github.com/jamiehynds\",\"followers_url\":\"https://api.github.com/users/jamiehynds/followers\",\"following_url\":\"https://api.github.com/users/jamiehynds/following{/other_user}\",\"gists_url\":\"https://api.github.com/users/jamiehynds/gists{/gist_id}\",\"starred_url\":\"https://api.github.com/users/jamiehynds/starred{/owner}{/repo}\",\"subscriptions_url\":\"https://api.github.com/users/jamiehynds/subscriptions\",\"organizations_url\":\"https://api.github.com/users/jamiehynds/orgs\",\"repos_url\":\"https://api.github.com/users/jamiehynds/repos\",\"events_url\":\"https://api.github.com/users/jamiehynds/events{/privacy}\",\"received_events_url\":\"https://api.github.com/users/jamiehynds/received_events\",\"type\":\"User\",\"site_admin\":false},\"labels\":[{\"id\":2404921703,\"node_id\":\"MDU6TGFiZWwyNDA0OTIxNzAz\",\"url\":\"https://api.github.com/repos/elastic/integrations/labels/Team:Security-Service%20Integrations\",\"name\":\"Team:Security-Service Integrations\",\"color\":\"1d76db\",\"default\":false,\"description\":\"Label for the Security External Integrations team\"},{\"id\":3104073484,\"node_id\":\"MDU6TGFiZWwzMTA0MDczNDg0\",\"url\":\"https://api.github.com/repos/elastic/integrations/labels/Integration:Threat%20Intel\",\"name\":\"Integration:Threat Intel\",\"color\":\"ffffff\",\"default\":false,\"description\":\"\"}],\"state\":\"open\",\"locked\":false,\"assignee\":{\"login\":\"P1llus\",\"id\":8027539,\"node_id\":\"MDQ6VXNlcjgwMjc1Mzk=\",\"avatar_url\":\"https://avatars.githubusercontent.com/u/8027539?v=4\",\"gravatar_id\":\"\",\"url\":\"https://api.github.com/users/P1llus\",\"html_url\":\"https://github.com/P1llus\",\"followers_url\":\"https://api.github.com/users/P1llus/followers\",\"following_url\":\"https://api.github.com/users/P1llus/following{/other_user}\",\"gists_url\":\"https://api.github.com/users/P1llus/gists{/gist_id}\",\"starred_url\":\"https://api.github.com/users/P1llus/starred{/owner}{/repo}\",\"subscriptions_url\":\"https://api.github.com/users/P1llus/subscriptions\",\"organizations_url\":\"https://api.github.com/users/P1llus/orgs\",\"repos_url\":\"https://api.github.com/users/P1llus/repos\",\"events_url\":\"https://api.github.com/users/P1llus/events{/privacy}\",\"received_events_url\":\"https://api.github.com/users/P1llus/received_events\",\"type\":\"User\",\"site_admin\":false},\"assignees\":[{\"login\":\"P1llus\",\"id\":8027539,\"node_id\":\"MDQ6VXNlcjgwMjc1Mzk=\",\"avatar_url\":\"https://avatars.githubusercontent.com/u/8027539?v=4\",\"gravatar_id\":\"\",\"url\":\"https://api.github.com/users/P1llus\",\"html_url\":\"https://github.com/P1llus\",\"followers_url\":\"https://api.github.com/users/P1llus/followers\",\"following_url\":\"https://api.github.com/users/P1llus/following{/other_user}\",\"gists_url\":\"https://api.github.com/users/P1llus/gists{/gist_id}\",\"starred_url\":\"https://api.github.com/users/P1llus/starred{/owner}{/repo}\",\"subscriptions_url\":\"https://api.github.com/users/P1llus/subscriptions\",\"organizations_url\":\"https://api.github.com/users/P1llus/orgs\",\"repos_url\":\"https://api.github.com/users/P1llus/repos\",\"events_url\":\"https://api.github.com/users/P1llus/events{/privacy}\",\"received_events_url\":\"https://api.github.com/users/P1llus/received_events\",\"type\":\"User\",\"site_admin\":false}],\"milestone\":null,\"comments\":2,\"created_at\":\"2022-11-23T15:06:34Z\",\"updated_at\":\"2022-11-23T15:07:18Z\",\"closed_at\":null,\"author_association\":\"NONE\",\"active_lock_reason\":null,\"body\":\"Structured Threat Information Expression (STIX) is a language for expressing cyber threat and observable information. While we have several Threat Intel integrations which map STIX formatted data to Elastic Common Schema, users will always have need to ingest IOC's from threat feeds that we don't support out of the box. 'How do I ingest STIX feeds' remains a very common questions across community Slack, Discuss, GitHub, etc. A custom package would solve for this. \\r\\n\\r\\nTo allow for the broad range of STIX formatted feeds, we should provide a way for users to ingest data from ANY STIX feed, via a 'Custom STIX' package. The package will leverage our httpjson input under the hood, but include an ingest pipeline which maps STIX fields to ECS (we expect there'll still be a need for custom fields, as not all STIX fields have a corresponding field in ECS). \\r\\n\\r\\nThere may be cases where some feeds/vendors don't strictly conform to STIX, and in those cases, users may have to modify our pipeline and that's ok. We'll focus on correctly formatted STIX data. \",\"reactions\":{\"url\":\"https://api.github.com/repos/elastic/integrations/issues/4710/reactions\",\"total_count\":0,\"+1\":0,\"-1\":0,\"laugh\":0,\"hooray\":0,\"confused\":0,\"heart\":0,\"rocket\":0,\"eyes\":0},\"timeline_url\":\"https://api.github.com/repos/elastic/integrations/issues/4710/timeline\",\"performed_via_github_app\":null,\"state_reason\":null}", "type": [ "change" ] @@ -140,7 +140,7 @@ } ], "author_association": "NONE", - "body": "Structured Threat Information Expression (STIX) is a language for expressing cyber threat and observable information. While we have several Threat Intel integrations which map STIX formatted data to Elastic Common Schema, users will always have need to ingest IOC's from threat feeds that we don't support out of the box. 'How do I ingest STIX feeds' remains a very common questions across community Slack, Discuss, Github, etc. A custom package would solve for this. \r\n\r\nTo allow for the broad range of STIX formatted feeds, we should provide a way for users to ingest data from ANY STIX feed, via a 'Custom STIX' package. The package will leverage our httpjson input under the hood, but include an ingest pipeline which maps STIX fields to ECS (we expect there'll still be a need for custom fields, as not all STIX fields have a corresponding field in ECS). \r\n\r\nThere may be cases where some feeds/vendors don't strictly conform to STIX, and in those cases, users may have to modify our pipeline and that's ok. We'll focus on correctly formatted STIX data. ", + "body": "Structured Threat Information Expression (STIX) is a language for expressing cyber threat and observable information. While we have several Threat Intel integrations which map STIX formatted data to Elastic Common Schema, users will always have need to ingest IOC's from threat feeds that we don't support out of the box. 'How do I ingest STIX feeds' remains a very common questions across community Slack, Discuss, GitHub, etc. A custom package would solve for this. \r\n\r\nTo allow for the broad range of STIX formatted feeds, we should provide a way for users to ingest data from ANY STIX feed, via a 'Custom STIX' package. The package will leverage our httpjson input under the hood, but include an ingest pipeline which maps STIX fields to ECS (we expect there'll still be a need for custom fields, as not all STIX fields have a corresponding field in ECS). \r\n\r\nThere may be cases where some feeds/vendors don't strictly conform to STIX, and in those cases, users may have to modify our pipeline and that's ok. We'll focus on correctly formatted STIX data. ", "comments": 2, "comments_url": "https://api.github.com/repos/elastic/integrations/issues/4710/comments", "created_at": "2022-11-23T15:06:34.000Z", diff --git a/packages/github/data_stream/issues/manifest.yml b/packages/github/data_stream/issues/manifest.yml index 41f4ce1e2be..a1335e109ff 100644 --- a/packages/github/data_stream/issues/manifest.yml +++ b/packages/github/data_stream/issues/manifest.yml @@ -1,5 +1,5 @@ type: logs -title: Github Issue +title: GitHub Issue release: beta streams: - input: httpjson @@ -119,5 +119,5 @@ streams: show_user: false description: "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. \nThis executes in the agent before the logs are parsed. \nSee [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.\n" template_path: httpjson.yml.hbs - title: Github Issues + title: GitHub Issues description: Collect GitHub issues as events via the API diff --git a/packages/github/docs/README.md b/packages/github/docs/README.md index 82f918ba582..370e2c9c9db 100644 --- a/packages/github/docs/README.md +++ b/packages/github/docs/README.md @@ -25,9 +25,7 @@ For Organizations: The GitHub audit log records all events related to the GitHub organization/enterprise. See [Organization audit log actions](https://docs.github.com/en/organizations/keeping-your-organization-secure/reviewing-the-audit-log-for-your-organization#audit-log-actions) and [Enterprise audit log actions](https://docs.github.com/en/enterprise-cloud@latest/admin/monitoring-activity-in-your-enterprise/reviewing-audit-logs-for-your-enterprise/about-the-audit-log-for-your-enterprise) for more details. -Github integration can collect audit logs from three sources: [Github API](https://docs.github.com/en/enterprise-cloud@latest/admin/monitoring-activity-in-your-enterprise/reviewing-audit-logs-for-your-enterprise/using-the-audit-log-api-for-your-enterprise), [Azure Event Hubs](https://docs.github.com/en/enterprise-cloud@latest/admin/monitoring-activity-in-your-enterprise/reviewing-audit-logs-for-your-enterprise/streaming-the-audit-log-for-your-enterprise#setting-up-streaming-to-azure-event-hubs), [Azure Blob Storage](https://docs.github.com/en/enterprise-cloud@latest/admin/monitoring-activity-in-your-enterprise/reviewing-audit-logs-for-your-enterprise/streaming-the-audit-log-for-your-enterprise#setting-up-streaming-to-azure-blob-storage), [AWS S3 or AWS SQS](https://docs.github.com/en/enterprise-cloud@latest/admin/monitoring-activity-in-your-enterprise/reviewing-audit-logs-for-your-enterprise/streaming-the-audit-log-for-your-enterprise#setting-up-streaming-to-amazon-s3) and [Google Cloud Storage](https://docs.github.com/en/enterprise-cloud@latest/admin/monitoring-activity-in-your-enterprise/reviewing-audit-logs-for-your-enterprise/streaming-the-audit-log-for-your-enterprise#setting-up-streaming-to-google-cloud-storage). - -When using Github API to collect audit log events, below requirements must be met for Personal Access Token (PAT): +When using GitHub API to collect audit log events, below requirements must be met for Personal Access Token (PAT): - You must use a Personal Access Token with `read:audit_log` scope. This applies to both organization and enterprise admins. - If you're an enterprise admin, ensure your token also includes `admin:enterprise` scope to access enterprise-wide logs. @@ -755,7 +753,7 @@ An example event for `dependabot` looks as following: "https://nodesecurity.io/advisories/17" ], "scanner": { - "vendor": "Github" + "vendor": "GitHub" }, "score": { "base": 0 diff --git a/packages/github/elasticsearch/transform/latest_code_scanning/transform.yml b/packages/github/elasticsearch/transform/latest_code_scanning/transform.yml index a46e300f258..25f3111fa94 100644 --- a/packages/github/elasticsearch/transform/latest_code_scanning/transform.yml +++ b/packages/github/elasticsearch/transform/latest_code_scanning/transform.yml @@ -22,7 +22,7 @@ latest: - github.code_scanning.created_at sort: "event.ingested" description: >- - Latest Code Scanning Alerts from Github's Code Scanning. As code scanning alerts get updated (dismissed/reopened), this transform stores only the latest state of each code scanning alert inside the destination index. Thus the transform's destination index contains only the latest state of the alerts. + Latest Code Scanning Alerts from GitHub's Code Scanning. As code scanning alerts get updated (dismissed/reopened), this transform stores only the latest state of each code scanning alert inside the destination index. Thus the transform's destination index contains only the latest state of the alerts. frequency: 30s settings: # This is required to prevent the transform from clobbering the Fleet-managed mappings. diff --git a/packages/github/elasticsearch/transform/latest_dependabot/transform.yml b/packages/github/elasticsearch/transform/latest_dependabot/transform.yml index 98874cb833f..7c3ff6acb4b 100644 --- a/packages/github/elasticsearch/transform/latest_dependabot/transform.yml +++ b/packages/github/elasticsearch/transform/latest_dependabot/transform.yml @@ -22,7 +22,7 @@ latest: - github.dependabot.created_at sort: "event.ingested" description: >- - Latest Alerts from Github's Dependabot. As Alerts get updated, this transform stores only the latest state of each alert inside the destination index. Thus the transform's destination index contains only the latest state of the alert. + Latest Alerts from GitHub's Dependabot. As Alerts get updated, this transform stores only the latest state of each alert inside the destination index. Thus the transform's destination index contains only the latest state of the alert. frequency: 30s settings: # This is required to prevent the transform from clobbering the Fleet-managed mappings. diff --git a/packages/github/elasticsearch/transform/latest_issues/transform.yml b/packages/github/elasticsearch/transform/latest_issues/transform.yml index 390de440073..0324d6d4d8a 100644 --- a/packages/github/elasticsearch/transform/latest_issues/transform.yml +++ b/packages/github/elasticsearch/transform/latest_issues/transform.yml @@ -22,7 +22,7 @@ latest: - github.issues.created_at sort: "event.ingested" description: >- - Latest Issues from Github. As issues get updated, this transform stores only the latest state of each issue inside the destination index. Thus the transform's destination index contains only the latest state of the issue. + Latest Issues from GitHub. As issues get updated, this transform stores only the latest state of each issue inside the destination index. Thus the transform's destination index contains only the latest state of the issue. frequency: 30s settings: # This is required to prevent the transform from clobbering the Fleet-managed mappings. diff --git a/packages/github/elasticsearch/transform/latest_secret_scanning/transform.yml b/packages/github/elasticsearch/transform/latest_secret_scanning/transform.yml index 10a2a1aabc2..29c5f5c694d 100644 --- a/packages/github/elasticsearch/transform/latest_secret_scanning/transform.yml +++ b/packages/github/elasticsearch/transform/latest_secret_scanning/transform.yml @@ -22,7 +22,7 @@ latest: - github.secret_scanning.created_at sort: "event.ingested" description: >- - Latest Secret Scanning Alerts from Github's Secret Scanning. As secret scanning alerts get updated, this transform stores only the latest state of each secret scanning alert inside the destination index. Thus the transform's destination index contains only the latest state of the alerts. + Latest Secret Scanning Alerts from GitHub's Secret Scanning. As secret scanning alerts get updated, this transform stores only the latest state of each secret scanning alert inside the destination index. Thus the transform's destination index contains only the latest state of the alerts. frequency: 30s settings: # This is required to prevent the transform from clobbering the Fleet-managed mappings. diff --git a/packages/github/kibana/dashboard/github-4da91aa0-12fc-11ed-af77-016e1a977d80.json b/packages/github/kibana/dashboard/github-4da91aa0-12fc-11ed-af77-016e1a977d80.json index 198711177d2..7267acdb683 100644 --- a/packages/github/kibana/dashboard/github-4da91aa0-12fc-11ed-af77-016e1a977d80.json +++ b/packages/github/kibana/dashboard/github-4da91aa0-12fc-11ed-af77-016e1a977d80.json @@ -1490,7 +1490,7 @@ "id": "", "params": { "fontSize": 12, - "markdown": "This dashboard provides an overview of the alerts ingested from Github Code Scanning.\n\nThe dashboard provides details on code scanning alerts that are open and resolved. It deep-dives into the top 10 repositories where code scanning alerts are found. It also calculates the mean-time to resolve (or dismiss) an open code scanning alert. The dashboard presents a view of alerts by severity and code scanning rules defining the alerts. Finally, it gives a layout of top users resolving the code scanning alerts.\n\n[**Integrations Page**](/app/integrations/detail/github/overview)", + "markdown": "This dashboard provides an overview of the alerts ingested from GitHub Code Scanning.\n\nThe dashboard provides details on code scanning alerts that are open and resolved. It deep-dives into the top 10 repositories where code scanning alerts are found. It also calculates the mean-time to resolve (or dismiss) an open code scanning alert. The dashboard presents a view of alerts by severity and code scanning rules defining the alerts. Finally, it gives a layout of top users resolving the code scanning alerts.\n\n[**Integrations Page**](/app/integrations/detail/github/overview)", "openLinksInNewTab": false }, "title": "", diff --git a/packages/github/kibana/dashboard/github-591d69e0-17b6-11ed-809a-7b4be950fe9c.json b/packages/github/kibana/dashboard/github-591d69e0-17b6-11ed-809a-7b4be950fe9c.json index ba125cd54d5..bcd8bea67fc 100644 --- a/packages/github/kibana/dashboard/github-591d69e0-17b6-11ed-809a-7b4be950fe9c.json +++ b/packages/github/kibana/dashboard/github-591d69e0-17b6-11ed-809a-7b4be950fe9c.json @@ -1219,7 +1219,7 @@ "id": "", "params": { "fontSize": 12, - "markdown": "This dashboard provides an overview of the events ingested from Github.\n\nThe dashboard provides details on secret scanning alerts that are open and resolved. It deep-dives into the top 10 repositories where secret scanning alerts are found. It also calculates the mean-time to resolve (or dismiss) an open secret scanning alert. The dashboard presents a view of the type of secrets that are currently open. Finally, it gives a layout of top users resolving the secret scanning alerts.\n\n[**Integrations Page**](/app/integrations/detail/github/overview)", + "markdown": "This dashboard provides an overview of the events ingested from GitHub.\n\nThe dashboard provides details on secret scanning alerts that are open and resolved. It deep-dives into the top 10 repositories where secret scanning alerts are found. It also calculates the mean-time to resolve (or dismiss) an open secret scanning alert. The dashboard presents a view of the type of secrets that are currently open. Finally, it gives a layout of top users resolving the secret scanning alerts.\n\n[**Integrations Page**](/app/integrations/detail/github/overview)", "openLinksInNewTab": false }, "title": "", diff --git a/packages/github/kibana/dashboard/github-6197be80-220c-11ed-88c4-e3caca48250a.json b/packages/github/kibana/dashboard/github-6197be80-220c-11ed-88c4-e3caca48250a.json index c32d789f02c..17fe4bcf867 100644 --- a/packages/github/kibana/dashboard/github-6197be80-220c-11ed-88c4-e3caca48250a.json +++ b/packages/github/kibana/dashboard/github-6197be80-220c-11ed-88c4-e3caca48250a.json @@ -1268,7 +1268,7 @@ "id": "", "params": { "fontSize": 12, - "markdown": "This dashboard provides an overview of the alerts ingested from Github Code Scanning.\n\nThe dashboard provides details on code scanning alerts that are open and resolved. It deep-dives into the top 10 repositories where code scanning alerts are found. It also calculates the mean-time to resolve (or dismiss) an open code scanning alert. The dashboard presents a view of alerts by severity and code scanning rules defining the alerts. Finally, it gives a layout of top users resolving the code scanning alerts.\n\n[**Integrations Page**](/app/integrations/detail/github/overview)", + "markdown": "This dashboard provides an overview of the alerts ingested from GitHub Code Scanning.\n\nThe dashboard provides details on code scanning alerts that are open and resolved. It deep-dives into the top 10 repositories where code scanning alerts are found. It also calculates the mean-time to resolve (or dismiss) an open code scanning alert. The dashboard presents a view of alerts by severity and code scanning rules defining the alerts. Finally, it gives a layout of top users resolving the code scanning alerts.\n\n[**Integrations Page**](/app/integrations/detail/github/overview)", "openLinksInNewTab": false }, "title": "", diff --git a/packages/github/kibana/dashboard/github-6a6d7c40-17ab-11ed-809a-7b4be950fe9c.json b/packages/github/kibana/dashboard/github-6a6d7c40-17ab-11ed-809a-7b4be950fe9c.json index 991d9334771..f7e43cab2dd 100644 --- a/packages/github/kibana/dashboard/github-6a6d7c40-17ab-11ed-809a-7b4be950fe9c.json +++ b/packages/github/kibana/dashboard/github-6a6d7c40-17ab-11ed-809a-7b4be950fe9c.json @@ -1475,7 +1475,7 @@ "id": "", "params": { "fontSize": 12, - "markdown": "This dashboard provides an overview of the alerts ingested from Github Code Scanning, Secret Scanning, and Dependabot.\n\nThe dashboard provides an overview of code scanning, secret scanning, and dependabot alerts that are open and resolved. It deep-dives into the top 10 repositories where alerts are found. The dashboard presents a view of alerts by severity. The dashboard gives a view alerts by type of GHAS Product.\n\n[**Integrations Page**](/app/integrations/detail/github/overview)", + "markdown": "This dashboard provides an overview of the alerts ingested from GitHub Code Scanning, Secret Scanning, and Dependabot.\n\nThe dashboard provides an overview of code scanning, secret scanning, and dependabot alerts that are open and resolved. It deep-dives into the top 10 repositories where alerts are found. The dashboard presents a view of alerts by severity. The dashboard gives a view alerts by type of GHAS Product.\n\n[**Integrations Page**](/app/integrations/detail/github/overview)", "openLinksInNewTab": false }, "title": "", diff --git a/packages/github/kibana/dashboard/github-f0104680-ae18-11ed-83fa-df5d96a45724.json b/packages/github/kibana/dashboard/github-f0104680-ae18-11ed-83fa-df5d96a45724.json index b2d7d847ea0..c704906c8b4 100644 --- a/packages/github/kibana/dashboard/github-f0104680-ae18-11ed-83fa-df5d96a45724.json +++ b/packages/github/kibana/dashboard/github-f0104680-ae18-11ed-83fa-df5d96a45724.json @@ -76,7 +76,7 @@ "store": "appState" }, "meta": { - "alias": "Github Issues", + "alias": "GitHub Issues", "disabled": false, "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", "negate": false, @@ -1301,7 +1301,7 @@ "id": "", "params": { "fontSize": 12, - "markdown": "This dashboard provides an overview of the issues ingested from Github.\n\nThe dashboard provides details on issues that are open and resolved. It provides a view of the top 10 repositories with issues. It also calculates the mean-time to fix (or close) an issue. The dashboard presents a view of top labels that are assigned to the issues. Finally, it gives a layout of top users creating and fixing the issues.\n\n[**Integrations Page**](/app/integrations/detail/github/overview)", + "markdown": "This dashboard provides an overview of the issues ingested from GitHub.\n\nThe dashboard provides details on issues that are open and resolved. It provides a view of the top 10 repositories with issues. It also calculates the mean-time to fix (or close) an issue. The dashboard presents a view of top labels that are assigned to the issues. Finally, it gives a layout of top users creating and fixing the issues.\n\n[**Integrations Page**](/app/integrations/detail/github/overview)", "openLinksInNewTab": false }, "title": "", From 409bec8e673eebf97051a2f29685cffd37815f8d Mon Sep 17 00:00:00 2001 From: Shourie Ganguly Date: Wed, 17 Sep 2025 10:24:51 +0530 Subject: [PATCH 11/18] elastic-package format --- packages/github/data_stream/audit/manifest.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/github/data_stream/audit/manifest.yml b/packages/github/data_stream/audit/manifest.yml index 5dc283b8d5e..bc714d0271b 100644 --- a/packages/github/data_stream/audit/manifest.yml +++ b/packages/github/data_stream/audit/manifest.yml @@ -668,6 +668,7 @@ streams: # - regex: "event/" description: > "If the bucket will have events that correspond to files that this integration shouldn’t process, file_selectors can be used to limit the files that are processed. \nThis is a list of selectors which is made up of regex patters. The regex should match the bucket filepath. Regexes use [RE2 syntax](https://pkg.go.dev/regexp/syntax). \nFiles that don’t match one of the regexes will not be processed. \nThis process happens locally on the host hence it is an expensive operation. It is recommended to use this attribute only if there is a specific need to filter out files locally.\n" + - name: timestamp_epoch type: integer title: Timestamp Epoch @@ -675,6 +676,7 @@ streams: required: false description: > "This attribute can be used to filter out files/objects which have a timestamp older than the specified value. The value of this attribute should be in unix epoch (seconds) format. \nThis process happens locally on the host hence it is an expensive operation. It is recommended to use this attribute only if there is a specific need to filter out files locally.\n" + show_user: false - name: expand_event_list_from_field type: text @@ -684,6 +686,7 @@ streams: show_user: false description: > "If the file-set using this input expects to receive multiple messages bundled under a specific field or an array of objects then the config option for 'expand_event_list_from_field' can be specified. This setting will be able to split the messages under the group value into separate events. \nThis can be specified at the global level or at the bucket level. For more info please refer to the [documentation](https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-gcs#attrib-expand_event_list_from_field-gcs).\n" + - name: preserve_original_event required: true show_user: true From 40fab57fb312c3952adeb47575a1026ab8a918a5 Mon Sep 17 00:00:00 2001 From: Shourie Ganguly Date: Wed, 17 Sep 2025 10:26:14 +0530 Subject: [PATCH 12/18] added missed change --- packages/github/_dev/build/docs/README.md | 2 +- packages/github/docs/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/github/_dev/build/docs/README.md b/packages/github/_dev/build/docs/README.md index 9b024e6aab9..99711086031 100644 --- a/packages/github/_dev/build/docs/README.md +++ b/packages/github/_dev/build/docs/README.md @@ -34,7 +34,7 @@ To collect audit log events from Azure Blob Storage, follow the [guide](https:// To collect audit log events from AWS S3 or AWS SQS, follow the [guide](https://docs.github.com/en/enterprise-cloud@latest/admin/monitoring-activity-in-your-enterprise/reviewing-audit-logs-for-your-enterprise/streaming-the-audit-log-for-your-enterprise#setting-up-streaming-to-amazon-s3) to setup audit log streaming. For more details, refer to this [documentation](https://docs.github.com/en/enterprise-cloud@latest/admin/monitoring-activity-in-your-enterprise/reviewing-audit-logs-for-your-enterprise/streaming-the-audit-log-for-your-enterprise). To collect audit log events from Google Cloud Storage, follow the [guide](https://docs.github.com/en/enterprise-cloud@latest/admin/monitoring-activity-in-your-enterprise/reviewing-audit-logs-for-your-enterprise/streaming-the-audit-log-for-your-enterprise#setting-up-streaming-to-google-cloud-storage) to setup audit log streaming. -For Filebeat input documentation, please refer to the following: +For Filebeat input documentation, refer to the following pages: - [Azure Event Hub](https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-azure-eventhub) - [Azure Blob Storage](https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-azure-blob-storage) - [AWS S3](https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-aws-s3) diff --git a/packages/github/docs/README.md b/packages/github/docs/README.md index 370e2c9c9db..93ed0756640 100644 --- a/packages/github/docs/README.md +++ b/packages/github/docs/README.md @@ -34,7 +34,7 @@ To collect audit log events from Azure Blob Storage, follow the [guide](https:// To collect audit log events from AWS S3 or AWS SQS, follow the [guide](https://docs.github.com/en/enterprise-cloud@latest/admin/monitoring-activity-in-your-enterprise/reviewing-audit-logs-for-your-enterprise/streaming-the-audit-log-for-your-enterprise#setting-up-streaming-to-amazon-s3) to setup audit log streaming. For more details, refer to this [documentation](https://docs.github.com/en/enterprise-cloud@latest/admin/monitoring-activity-in-your-enterprise/reviewing-audit-logs-for-your-enterprise/streaming-the-audit-log-for-your-enterprise). To collect audit log events from Google Cloud Storage, follow the [guide](https://docs.github.com/en/enterprise-cloud@latest/admin/monitoring-activity-in-your-enterprise/reviewing-audit-logs-for-your-enterprise/streaming-the-audit-log-for-your-enterprise#setting-up-streaming-to-google-cloud-storage) to setup audit log streaming. -For Filebeat input documentation, please refer to the following: +For Filebeat input documentation, refer to the following pages: - [Azure Event Hub](https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-azure-eventhub) - [Azure Blob Storage](https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-azure-blob-storage) - [AWS S3](https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-aws-s3) From 34b185a6803615377c5be0f360036fce2506fc55 Mon Sep 17 00:00:00 2001 From: Shourie Ganguly Date: Wed, 17 Sep 2025 13:52:04 +0530 Subject: [PATCH 13/18] updated mock-service to support manifests to preload data, removed uploader service in docker-compose --- .../_dev/deploy/docker/docker-compose.yml | 19 +-- .../_dev/deploy/docker/files/manifest.yml | 5 + .../deploy/docker/gcs-mock-service/go.mod | 2 + .../deploy/docker/gcs-mock-service/go.sum | 4 + .../deploy/docker/gcs-mock-service/main.go | 118 +++++++++++++++--- 5 files changed, 116 insertions(+), 32 deletions(-) create mode 100644 packages/github/_dev/deploy/docker/files/manifest.yml create mode 100644 packages/github/_dev/deploy/docker/gcs-mock-service/go.sum diff --git a/packages/github/_dev/deploy/docker/docker-compose.yml b/packages/github/_dev/deploy/docker/docker-compose.yml index 4304c86a4d6..27b279124d8 100644 --- a/packages/github/_dev/deploy/docker/docker-compose.yml +++ b/packages/github/_dev/deploy/docker/docker-compose.yml @@ -40,25 +40,12 @@ services: working_dir: /app volumes: - ./gcs-mock-service:/app + - ./files/manifest.yml:/files/manifest.yml:ro + - ./sample_logs/:/data ports: - "4443/tcp" healthcheck: test: "wget --no-verbose --tries=1 --spider http://localhost:4443/health || exit 1" interval: 10s timeout: 5s - command: ["go", "run", "main.go"] - gcs-uploader: - image: curlimages/curl:latest - depends_on: - gcs-mock-service: - condition: service_healthy - volumes: - - ./sample_logs/:/data - entrypoint: > - sh -c " - sleep 5 && - echo 'Creating bucket...' && - curl -X POST http://gcs-mock-service:4443/storage/v1/b -H 'Content-Type: application/json' -d '{\"name\": \"testbucket\"}' && - echo 'Uploading cloud-storage-data.log with correct content-type...' && - curl -X POST 'http://gcs-mock-service:4443/upload/storage/v1/b/testbucket/o?uploadType=media&name=cloud-storage-data.ndjson' -H 'Content-Type: application/x-ndjson' -T '/data/cloud-storage-data.log' && - echo 'Upload finished. Object available in gcs-mock-service.'" + command: go run main.go -manifest /files/manifest.yml diff --git a/packages/github/_dev/deploy/docker/files/manifest.yml b/packages/github/_dev/deploy/docker/files/manifest.yml new file mode 100644 index 00000000000..8adc1f3afc2 --- /dev/null +++ b/packages/github/_dev/deploy/docker/files/manifest.yml @@ -0,0 +1,5 @@ +buckets: + testbucket: + files: + - path: /data/cloud-storage-data.log + content-type: application/x-ndjson diff --git a/packages/github/_dev/deploy/docker/gcs-mock-service/go.mod b/packages/github/_dev/deploy/docker/gcs-mock-service/go.mod index 661288576d4..df08767f7e8 100644 --- a/packages/github/_dev/deploy/docker/gcs-mock-service/go.mod +++ b/packages/github/_dev/deploy/docker/gcs-mock-service/go.mod @@ -1,3 +1,5 @@ module gcs-mock-service go 1.24.7 + +require gopkg.in/yaml.v3 v3.0.1 diff --git a/packages/github/_dev/deploy/docker/gcs-mock-service/go.sum b/packages/github/_dev/deploy/docker/gcs-mock-service/go.sum new file mode 100644 index 00000000000..a62c313c5b0 --- /dev/null +++ b/packages/github/_dev/deploy/docker/gcs-mock-service/go.sum @@ -0,0 +1,4 @@ +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/packages/github/_dev/deploy/docker/gcs-mock-service/main.go b/packages/github/_dev/deploy/docker/gcs-mock-service/main.go index 77e29042e9c..baecae43313 100644 --- a/packages/github/_dev/deploy/docker/gcs-mock-service/main.go +++ b/packages/github/_dev/deploy/docker/gcs-mock-service/main.go @@ -11,8 +11,11 @@ import ( "io" "log" "net/http" + "os" "strconv" "strings" + + "gopkg.in/yaml.v3" ) // ObjectData stores the raw data and its content type. @@ -39,6 +42,22 @@ type GCSObject struct { ContentType string `json:"contentType"` } +// Manifest represents the top-level structure of the YAML file +type Manifest struct { + Buckets map[string]Bucket `yaml:"buckets"` +} + +// Bucket represents each bucket and its files +type Bucket struct { + Files []File `yaml:"files"` +} + +// File represents each file entry inside a bucket +type File struct { + Path string `yaml:"path"` + ContentType string `yaml:"content-type"` +} + func handleRequests(w http.ResponseWriter, r *http.Request) { path := r.URL.Path log.Printf("Received request: %s %s", r.Method, path) @@ -86,17 +105,27 @@ func handleCreateBucket(w http.ResponseWriter, r *http.Request) { http.Error(w, "invalid JSON body", http.StatusBadRequest) return } - bucketName := bucketInfo.Name - if _, exists := inMemoryStore[bucketName]; exists { - http.Error(w, "bucket already exists", http.StatusConflict) + if bucketInfo.Name == "" { + http.Error(w, "bucket name is required", http.StatusBadRequest) + return + } + if err := createBucket(bucketInfo.Name); err != nil { + http.Error(w, err.Error(), http.StatusConflict) return } - inMemoryStore[bucketName] = make(map[string]ObjectData) - log.Printf("created bucket: %s", bucketName) w.WriteHeader(http.StatusOK) json.NewEncoder(w).Encode(bucketInfo) } +func createBucket(bucketName string) error { + if _, exists := inMemoryStore[bucketName]; exists { + return fmt.Errorf("bucket already exists") + } + inMemoryStore[bucketName] = make(map[string]ObjectData) + log.Printf("created bucket: %s", bucketName) + return nil +} + // handleUploadObject uploads a new file to a bucket. func handleUploadObject(w http.ResponseWriter, r *http.Request) { pathParts := strings.Split(strings.Trim(r.URL.Path, "/"), "/") @@ -111,11 +140,6 @@ func handleUploadObject(w http.ResponseWriter, r *http.Request) { return } - if _, ok := inMemoryStore[bucketName]; !ok { - http.Error(w, "bucket not found", http.StatusNotFound) - return - } - data, err := io.ReadAll(r.Body) if err != nil { http.Error(w, "failed to read request body", http.StatusInternalServerError) @@ -128,21 +152,34 @@ func handleUploadObject(w http.ResponseWriter, r *http.Request) { contentType = "application/octet-stream" } + response, err := uploadObject(bucketName, objectName, data, contentType) + if err != nil { + http.Error(w, err.Error(), http.StatusNotFound) + return + } + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) +} + +func uploadObject(bucketName, objectName string, data []byte, contentType string) (*GCSObject, error) { + if _, ok := inMemoryStore[bucketName]; !ok { + return nil, fmt.Errorf("bucket not found") + } + inMemoryStore[bucketName][objectName] = ObjectData{ Data: data, ContentType: contentType, } - log.Printf("uploaded object '%s' to bucket '%s' with Content-Type '%s'", objectName, bucketName, contentType) + log.Printf("created object '%s' in bucket '%s' with Content-Type '%s'", objectName, bucketName, contentType) - response := GCSObject{ + return &GCSObject{ Kind: "storage#object", Name: objectName, Bucket: bucketName, Size: strconv.Itoa(len(data)), ContentType: contentType, - } - w.Header().Set("Content-Type", "application/json") - json.NewEncoder(w).Encode(response) + }, nil } // handleGetObject is for retrieving an object from a bucket. @@ -204,15 +241,64 @@ func handleListObjects(w http.ResponseWriter, r *http.Request) { http.Error(w, "not found", http.StatusNotFound) } +// readManifest reads and parses the YAML manifest file. +func readManifest(path string) (*Manifest, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("failed to read manifest: %w", err) + } + + var manifest Manifest + if err := yaml.Unmarshal(data, &manifest); err != nil { + return nil, fmt.Errorf("failed to parse manifest: %w", err) + } + + return &manifest, nil +} + +// processManifest processes the manifest to create buckets and upload files. +func processManifest(manifest *Manifest) error { + for bucketName, bucket := range manifest.Buckets { + for _, file := range bucket.Files { + fmt.Printf("preloading data for bucket: %s | path: %s | content-type: %s...\n", bucketName, file.Path, file.ContentType) + if err := createBucket(bucketName); err != nil { + return fmt.Errorf("failed to create bucket '%s': %w", bucketName, err) + } + data, err := os.ReadFile(file.Path) + if err != nil { + return fmt.Errorf("failed to read bucket data file '%s': %w", file.Path, err) + } + // atm we use only the file name as the object name. + pathParts := strings.Split(file.Path, "/") + if _, err := uploadObject(bucketName, pathParts[len(pathParts)-1], data, file.ContentType); err != nil { + return fmt.Errorf("failed to create object '%s' in bucket '%s': %w", file.Path, bucketName, err) + } + } + } + return nil +} + func main() { host := flag.String("host", "0.0.0.0", "host to listen on") port := flag.String("port", "4443", "port to listen on") + manifest := flag.String("manifest", "", "path to YAML manifest file for preloading buckets and objects") flag.Parse() addr := fmt.Sprintf("%s:%s", *host, *port) fmt.Printf("Starting mock GCS server on http://%s\n", addr) - fmt.Println("Store is empty. Create buckets and objects via API calls.") + // if a manifest file is provided, read it and preload the buckets and objects. + if *manifest != "" { + m, err := readManifest(*manifest) + if err != nil { + log.Fatalf("error reading manifest: %v", err) + } + if err := processManifest(m); err != nil { + log.Fatalf("error processing manifest: %v", err) + } + } else { + fmt.Println("Store is empty. Create buckets and objects via API calls.") + } http.HandleFunc("/", handleRequests) From ab46137b4bda967b1a2d1ef6dc1052627bab5761 Mon Sep 17 00:00:00 2001 From: Shourie Ganguly Date: Thu, 18 Sep 2025 09:06:44 +0530 Subject: [PATCH 14/18] reverted the code changes and updated the docs as suggested --- packages/github/_dev/build/docs/README.md | 2 ++ .../test-ghas-dependabot-json.log-expected.json | 16 ++++++++-------- .../elasticsearch/ingest_pipeline/default.yml | 2 +- .../data_stream/dependabot/sample_event.json | 2 +- .../test/pipeline/test-github-issues-json.log | 2 +- .../test-github-issues-json.log-expected.json | 4 ++-- packages/github/docs/README.md | 4 +++- 7 files changed, 18 insertions(+), 14 deletions(-) diff --git a/packages/github/_dev/build/docs/README.md b/packages/github/_dev/build/docs/README.md index 99711086031..61f72a66d2e 100644 --- a/packages/github/_dev/build/docs/README.md +++ b/packages/github/_dev/build/docs/README.md @@ -25,6 +25,8 @@ For Organizations: The GitHub audit log records all events related to the GitHub organization/enterprise. See [Organization audit log actions](https://docs.github.com/en/organizations/keeping-your-organization-secure/reviewing-the-audit-log-for-your-organization#audit-log-actions) and [Enterprise audit log actions](https://docs.github.com/en/enterprise-cloud@latest/admin/monitoring-activity-in-your-enterprise/reviewing-audit-logs-for-your-enterprise/about-the-audit-log-for-your-enterprise) for more details. +The GitHub integration can collect audit logs from the following sources: [GitHub API](https://docs.github.com/en/enterprise-cloud@latest/admin/monitoring-activity-in-your-enterprise/reviewing-audit-logs-for-your-enterprise/using-the-audit-log-api-for-your-enterprise), [Azure Event Hubs](https://docs.github.com/en/enterprise-cloud@latest/admin/monitoring-activity-in-your-enterprise/reviewing-audit-logs-for-your-enterprise/streaming-the-audit-log-for-your-enterprise#setting-up-streaming-to-azure-event-hubs), [Azure Blob Storage](https://docs.github.com/en/enterprise-cloud@latest/admin/monitoring-activity-in-your-enterprise/reviewing-audit-logs-for-your-enterprise/streaming-the-audit-log-for-your-enterprise#setting-up-streaming-to-azure-blob-storage), [AWS S3 or AWS SQS](https://docs.github.com/en/enterprise-cloud@latest/admin/monitoring-activity-in-your-enterprise/reviewing-audit-logs-for-your-enterprise/streaming-the-audit-log-for-your-enterprise#setting-up-streaming-to-amazon-s3) and [Google Cloud Storage](https://docs.github.com/en/enterprise-cloud@latest/admin/monitoring-activity-in-your-enterprise/reviewing-audit-logs-for-your-enterprise/streaming-the-audit-log-for-your-enterprise#setting-up-streaming-to-google-cloud-storage). + When using GitHub API to collect audit log events, below requirements must be met for Personal Access Token (PAT): - You must use a Personal Access Token with `read:audit_log` scope. This applies to both organization and enterprise admins. - If you're an enterprise admin, ensure your token also includes `admin:enterprise` scope to access enterprise-wide logs. diff --git a/packages/github/data_stream/dependabot/_dev/test/pipeline/test-ghas-dependabot-json.log-expected.json b/packages/github/data_stream/dependabot/_dev/test/pipeline/test-ghas-dependabot-json.log-expected.json index 11c4340d29a..42d42a45e89 100644 --- a/packages/github/data_stream/dependabot/_dev/test/pipeline/test-ghas-dependabot-json.log-expected.json +++ b/packages/github/data_stream/dependabot/_dev/test/pipeline/test-ghas-dependabot-json.log-expected.json @@ -98,7 +98,7 @@ "https://nodesecurity.io/advisories/17" ], "scanner": { - "vendor": "GitHub" + "vendor": "Github" }, "score": { "base": 0.0 @@ -205,7 +205,7 @@ "https://nodesecurity.io/advisories/154" ], "scanner": { - "vendor": "GitHub" + "vendor": "Github" }, "score": { "base": 0.0 @@ -315,7 +315,7 @@ "https://github.com/advisories/GHSA-3j7m-hmh3-9jmp" ], "scanner": { - "vendor": "GitHub" + "vendor": "Github" }, "score": { "base": 6.1, @@ -422,7 +422,7 @@ "https://github.com/advisories/GHSA-6g6m-m6h5-w9gf" ], "scanner": { - "vendor": "GitHub" + "vendor": "Github" }, "score": { "base": 7.7, @@ -511,7 +511,7 @@ "https://github.com/advisories/GHSA-5mrr-rgp6-x4gr" ], "scanner": { - "vendor": "GitHub" + "vendor": "Github" }, "score": { "base": 0.0 @@ -618,7 +618,7 @@ "https://github.com/advisories/GHSA-mjxr-4v3x-q3m4" ], "scanner": { - "vendor": "GitHub" + "vendor": "Github" }, "score": { "base": 5.3, @@ -734,7 +734,7 @@ "https://github.com/advisories/GHSA-rjqq-98f6-6j3r" ], "scanner": { - "vendor": "GitHub" + "vendor": "Github" }, "score": { "base": 5.3, @@ -846,7 +846,7 @@ "https://github.com/advisories/GHSA-9qrh-qjmc-5w2p" ], "scanner": { - "vendor": "GitHub" + "vendor": "Github" }, "score": { "base": 7.5, diff --git a/packages/github/data_stream/dependabot/elasticsearch/ingest_pipeline/default.yml b/packages/github/data_stream/dependabot/elasticsearch/ingest_pipeline/default.yml index 1d33fb741ea..32da2270fc8 100644 --- a/packages/github/data_stream/dependabot/elasticsearch/ingest_pipeline/default.yml +++ b/packages/github/data_stream/dependabot/elasticsearch/ingest_pipeline/default.yml @@ -215,7 +215,7 @@ processors: ignore_missing: true - set: field: vulnerability.scanner.vendor - value: "GitHub" + value: "Github" - convert: field: github.dependabot.security_advisory.cvss.score type: float diff --git a/packages/github/data_stream/dependabot/sample_event.json b/packages/github/data_stream/dependabot/sample_event.json index a5a1c949b62..812fac06446 100644 --- a/packages/github/data_stream/dependabot/sample_event.json +++ b/packages/github/data_stream/dependabot/sample_event.json @@ -121,7 +121,7 @@ "https://nodesecurity.io/advisories/17" ], "scanner": { - "vendor": "GitHub" + "vendor": "Github" }, "score": { "base": 0 diff --git a/packages/github/data_stream/issues/_dev/test/pipeline/test-github-issues-json.log b/packages/github/data_stream/issues/_dev/test/pipeline/test-github-issues-json.log index 5cb5cb36a49..70bf6b982be 100644 --- a/packages/github/data_stream/issues/_dev/test/pipeline/test-github-issues-json.log +++ b/packages/github/data_stream/issues/_dev/test/pipeline/test-github-issues-json.log @@ -1,5 +1,5 @@ {"id":1,"node_id":"MDU6SXNzdWUx","url":"https://api.github.com/repos/octocat/Hello-World/issues/1347","repository_url":"https://api.github.com/repos/octocat/Hello-World","labels_url":"https://api.github.com/repos/octocat/Hello-World/issues/1347/labels{/name}","comments_url":"https://api.github.com/repos/octocat/Hello-World/issues/1347/comments","events_url":"https://api.github.com/repos/octocat/Hello-World/issues/1347/events","html_url":"https://github.com/octocat/Hello-World/issues/1347","number":1347,"state":"open","title":"Found a bug","body":"I'm having a problem with this.","user":{"login":"octocat","id":1,"node_id":"MDQ6VXNlcjE=","avatar_url":"https://github.com/images/error/octocat_happy.gif","gravatar_id":"","url":"https://api.github.com/users/octocat","html_url":"https://github.com/octocat","followers_url":"https://api.github.com/users/octocat/followers","following_url":"https://api.github.com/users/octocat/following{/other_user}","gists_url":"https://api.github.com/users/octocat/gists{/gist_id}","starred_url":"https://api.github.com/users/octocat/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/octocat/subscriptions","organizations_url":"https://api.github.com/users/octocat/orgs","repos_url":"https://api.github.com/users/octocat/repos","events_url":"https://api.github.com/users/octocat/events{/privacy}","received_events_url":"https://api.github.com/users/octocat/received_events","type":"User","site_admin":true},"labels":[{"id":208045946,"node_id":"MDU6TGFiZWwyMDgwNDU5NDY=","url":"https://api.github.com/repos/octocat/Hello-World/labels/bug","name":"bug","description":"Something isn't working","color":"f29513","default":true}],"assignee":{"login":"octocat","id":1,"node_id":"MDQ6VXNlcjE=","avatar_url":"https://github.com/images/error/octocat_happy.gif","gravatar_id":"","url":"https://api.github.com/users/octocat","html_url":"https://github.com/octocat","followers_url":"https://api.github.com/users/octocat/followers","following_url":"https://api.github.com/users/octocat/following{/other_user}","gists_url":"https://api.github.com/users/octocat/gists{/gist_id}","starred_url":"https://api.github.com/users/octocat/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/octocat/subscriptions","organizations_url":"https://api.github.com/users/octocat/orgs","repos_url":"https://api.github.com/users/octocat/repos","events_url":"https://api.github.com/users/octocat/events{/privacy}","received_events_url":"https://api.github.com/users/octocat/received_events","type":"User","site_admin":false},"assignees":[{"login":"octocat","id":1,"node_id":"MDQ6VXNlcjE=","avatar_url":"https://github.com/images/error/octocat_happy.gif","gravatar_id":"","url":"https://api.github.com/users/octocat","html_url":"https://github.com/octocat","followers_url":"https://api.github.com/users/octocat/followers","following_url":"https://api.github.com/users/octocat/following{/other_user}","gists_url":"https://api.github.com/users/octocat/gists{/gist_id}","starred_url":"https://api.github.com/users/octocat/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/octocat/subscriptions","organizations_url":"https://api.github.com/users/octocat/orgs","repos_url":"https://api.github.com/users/octocat/repos","events_url":"https://api.github.com/users/octocat/events{/privacy}","received_events_url":"https://api.github.com/users/octocat/received_events","type":"User","site_admin":false}],"milestone":{"url":"https://api.github.com/repos/octocat/Hello-World/milestones/1","html_url":"https://github.com/octocat/Hello-World/milestones/v1.0","labels_url":"https://api.github.com/repos/octocat/Hello-World/milestones/1/labels","id":1002604,"node_id":"MDk6TWlsZXN0b25lMTAwMjYwNA==","number":1,"state":"open","title":"v1.0","description":"Tracking milestone for version 1.0","creator":{"login":"octocat","id":1,"node_id":"MDQ6VXNlcjE=","avatar_url":"https://github.com/images/error/octocat_happy.gif","gravatar_id":"","url":"https://api.github.com/users/octocat","html_url":"https://github.com/octocat","followers_url":"https://api.github.com/users/octocat/followers","following_url":"https://api.github.com/users/octocat/following{/other_user}","gists_url":"https://api.github.com/users/octocat/gists{/gist_id}","starred_url":"https://api.github.com/users/octocat/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/octocat/subscriptions","organizations_url":"https://api.github.com/users/octocat/orgs","repos_url":"https://api.github.com/users/octocat/repos","events_url":"https://api.github.com/users/octocat/events{/privacy}","received_events_url":"https://api.github.com/users/octocat/received_events","type":"User","site_admin":false},"open_issues":4,"closed_issues":8,"created_at":"2011-04-10T20:09:31Z","updated_at":"2014-03-03T18:58:10Z","closed_at":"2013-02-12T13:22:01Z","due_on":"2012-10-09T23:39:01Z"},"locked":true,"active_lock_reason":"too heated","comments":0,"pull_request":{"url":"https://api.github.com/repos/octocat/Hello-World/pulls/1347","html_url":"https://github.com/octocat/Hello-World/pull/1347","diff_url":"https://github.com/octocat/Hello-World/pull/1347.diff","patch_url":"https://github.com/octocat/Hello-World/pull/1347.patch"},"closed_at":null,"created_at":"2011-04-22T13:33:48Z","updated_at":"2011-04-22T13:33:48Z","closed_by":{"login":"octocat","id":1,"node_id":"MDQ6VXNlcjE=","avatar_url":"https://github.com/images/error/octocat_happy.gif","gravatar_id":"","url":"https://api.github.com/users/octocat","html_url":"https://github.com/octocat","followers_url":"https://api.github.com/users/octocat/followers","following_url":"https://api.github.com/users/octocat/following{/other_user}","gists_url":"https://api.github.com/users/octocat/gists{/gist_id}","starred_url":"https://api.github.com/users/octocat/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/octocat/subscriptions","organizations_url":"https://api.github.com/users/octocat/orgs","repos_url":"https://api.github.com/users/octocat/repos","events_url":"https://api.github.com/users/octocat/events{/privacy}","received_events_url":"https://api.github.com/users/octocat/received_events","type":"User","site_admin":false},"repository": {"id":1296269,"node_id":"MDEwOlJlcG9zaXRvcnkxMjk2MjY5","name":"Hello-World","full_name":"octocat/Hello-World","owner":{"login":"octocat","id":1,"node_id":"MDQ6VXNlcjE=","avatar_url":"https://github.com/images/error/octocat_happy.gif","gravatar_id":"","url":"https://api.github.com/users/octocat","html_url":"https://github.com/octocat","followers_url":"https://api.github.com/users/octocat/followers","following_url":"https://api.github.com/users/octocat/following{/other_user}","gists_url":"https://api.github.com/users/octocat/gists{/gist_id}","starred_url":"https://api.github.com/users/octocat/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/octocat/subscriptions","organizations_url":"https://api.github.com/users/octocat/orgs","repos_url":"https://api.github.com/users/octocat/repos","events_url":"https://api.github.com/users/octocat/events{/privacy}","received_events_url":"https://api.github.com/users/octocat/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/octocat/Hello-World","description":"This your first repo!","fork":false,"url":"https://api.github.com/repos/octocat/Hello-World","archive_url":"https://api.github.com/repos/octocat/Hello-World/{archive_format}{/ref}","assignees_url":"https://api.github.com/repos/octocat/Hello-World/assignees{/user}","blobs_url":"https://api.github.com/repos/octocat/Hello-World/git/blobs{/sha}","branches_url":"https://api.github.com/repos/octocat/Hello-World/branches{/branch}","collaborators_url":"https://api.github.com/repos/octocat/Hello-World/collaborators{/collaborator}","comments_url":"https://api.github.com/repos/octocat/Hello-World/comments{/number}","commits_url":"https://api.github.com/repos/octocat/Hello-World/commits{/sha}","compare_url":"https://api.github.com/repos/octocat/Hello-World/compare/{base}...{head}","contents_url":"https://api.github.com/repos/octocat/Hello-World/contents/{+path}","contributors_url":"https://api.github.com/repos/octocat/Hello-World/contributors","deployments_url":"https://api.github.com/repos/octocat/Hello-World/deployments","downloads_url":"https://api.github.com/repos/octocat/Hello-World/downloads","events_url":"https://api.github.com/repos/octocat/Hello-World/events","forks_url":"https://api.github.com/repos/octocat/Hello-World/forks","git_commits_url":"https://api.github.com/repos/octocat/Hello-World/git/commits{/sha}","git_refs_url":"https://api.github.com/repos/octocat/Hello-World/git/refs{/sha}","git_tags_url":"https://api.github.com/repos/octocat/Hello-World/git/tags{/sha}","git_url":"git:github.com/octocat/Hello-World.git","issue_comment_url":"https://api.github.com/repos/octocat/Hello-World/issues/comments{/number}","issue_events_url":"https://api.github.com/repos/octocat/Hello-World/issues/events{/number}","issues_url":"https://api.github.com/repos/octocat/Hello-World/issues{/number}","keys_url":"https://api.github.com/repos/octocat/Hello-World/keys{/key_id}","labels_url":"https://api.github.com/repos/octocat/Hello-World/labels{/name}","languages_url":"https://api.github.com/repos/octocat/Hello-World/languages","merges_url":"https://api.github.com/repos/octocat/Hello-World/merges","milestones_url":"https://api.github.com/repos/octocat/Hello-World/milestones{/number}","notifications_url":"https://api.github.com/repos/octocat/Hello-World/notifications{?since,all,participating}","pulls_url":"https://api.github.com/repos/octocat/Hello-World/pulls{/number}","releases_url":"https://api.github.com/repos/octocat/Hello-World/releases{/id}","ssh_url":"git@github.com:octocat/Hello-World.git","stargazers_url":"https://api.github.com/repos/octocat/Hello-World/stargazers","statuses_url":"https://api.github.com/repos/octocat/Hello-World/statuses/{sha}","subscribers_url":"https://api.github.com/repos/octocat/Hello-World/subscribers","subscription_url":"https://api.github.com/repos/octocat/Hello-World/subscription","tags_url":"https://api.github.com/repos/octocat/Hello-World/tags","teams_url":"https://api.github.com/repos/octocat/Hello-World/teams","trees_url":"https://api.github.com/repos/octocat/Hello-World/git/trees{/sha}","clone_url":"https://github.com/octocat/Hello-World.git","mirror_url":"git:git.example.com/octocat/Hello-World","hooks_url":"https://api.github.com/repos/octocat/Hello-World/hooks","svn_url":"https://svn.github.com/octocat/Hello-World","homepage":"https://github.com","language":null,"forks_count":9,"stargazers_count":80,"watchers_count":80,"size":108,"default_branch":"master","open_issues_count":0,"is_template":true,"topics":["octocat","atom","electron","api"],"has_issues":true,"has_projects":true,"has_wiki":true,"has_pages":false,"has_downloads":true,"archived":false,"disabled":false,"visibility":"public","pushed_at":"2011-01-26T19:06:43Z","created_at":"2011-01-26T19:01:12Z","updated_at":"2011-01-26T19:14:43Z","permissions":{"admin":false,"push":false,"pull":true},"allow_rebase_merge":true,"template_repository":null,"temp_clone_token":"ABTLWHOULUVAXGTRYU7OC2876QJ2O","allow_squash_merge":true,"allow_auto_merge":false,"delete_branch_on_merge":true,"allow_merge_commit":true,"subscribers_count":42,"network_count":0,"license":{"key":"mit","name":"MIT License","url":"https://api.github.com/licenses/mit","spdx_id":"MIT","node_id":"MDc6TGljZW5zZW1pdA==","html_url":"https://github.com/licenses/mit"},"forks":1,"open_issues":1,"watchers":1},"author_association":"COLLABORATOR","state_reason":"completed"} -{"url":"https://api.github.com/repos/elastic/integrations/issues/4710","repository_url":"https://api.github.com/repos/elastic/integrations","labels_url":"https://api.github.com/repos/elastic/integrations/issues/4710/labels{/name}","comments_url":"https://api.github.com/repos/elastic/integrations/issues/4710/comments","events_url":"https://api.github.com/repos/elastic/integrations/issues/4710/events","html_url":"https://github.com/elastic/integrations/issues/4710","id":1461928292,"node_id":"I_kwDODAw23M5XI0Fk","number":4710,"title":"Custom STIX Package","user":{"login":"jamiehynds","id":62879768,"node_id":"MDQ6VXNlcjYyODc5NzY4","avatar_url":"https://avatars.githubusercontent.com/u/62879768?v=4","gravatar_id":"","url":"https://api.github.com/users/jamiehynds","html_url":"https://github.com/jamiehynds","followers_url":"https://api.github.com/users/jamiehynds/followers","following_url":"https://api.github.com/users/jamiehynds/following{/other_user}","gists_url":"https://api.github.com/users/jamiehynds/gists{/gist_id}","starred_url":"https://api.github.com/users/jamiehynds/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jamiehynds/subscriptions","organizations_url":"https://api.github.com/users/jamiehynds/orgs","repos_url":"https://api.github.com/users/jamiehynds/repos","events_url":"https://api.github.com/users/jamiehynds/events{/privacy}","received_events_url":"https://api.github.com/users/jamiehynds/received_events","type":"User","site_admin":false},"labels":[{"id":2404921703,"node_id":"MDU6TGFiZWwyNDA0OTIxNzAz","url":"https://api.github.com/repos/elastic/integrations/labels/Team:Security-Service%20Integrations","name":"Team:Security-Service Integrations","color":"1d76db","default":false,"description":"Label for the Security External Integrations team"},{"id":3104073484,"node_id":"MDU6TGFiZWwzMTA0MDczNDg0","url":"https://api.github.com/repos/elastic/integrations/labels/Integration:Threat%20Intel","name":"Integration:Threat Intel","color":"ffffff","default":false,"description":""}],"state":"open","locked":false,"assignee":{"login":"P1llus","id":8027539,"node_id":"MDQ6VXNlcjgwMjc1Mzk=","avatar_url":"https://avatars.githubusercontent.com/u/8027539?v=4","gravatar_id":"","url":"https://api.github.com/users/P1llus","html_url":"https://github.com/P1llus","followers_url":"https://api.github.com/users/P1llus/followers","following_url":"https://api.github.com/users/P1llus/following{/other_user}","gists_url":"https://api.github.com/users/P1llus/gists{/gist_id}","starred_url":"https://api.github.com/users/P1llus/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/P1llus/subscriptions","organizations_url":"https://api.github.com/users/P1llus/orgs","repos_url":"https://api.github.com/users/P1llus/repos","events_url":"https://api.github.com/users/P1llus/events{/privacy}","received_events_url":"https://api.github.com/users/P1llus/received_events","type":"User","site_admin":false},"assignees":[{"login":"P1llus","id":8027539,"node_id":"MDQ6VXNlcjgwMjc1Mzk=","avatar_url":"https://avatars.githubusercontent.com/u/8027539?v=4","gravatar_id":"","url":"https://api.github.com/users/P1llus","html_url":"https://github.com/P1llus","followers_url":"https://api.github.com/users/P1llus/followers","following_url":"https://api.github.com/users/P1llus/following{/other_user}","gists_url":"https://api.github.com/users/P1llus/gists{/gist_id}","starred_url":"https://api.github.com/users/P1llus/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/P1llus/subscriptions","organizations_url":"https://api.github.com/users/P1llus/orgs","repos_url":"https://api.github.com/users/P1llus/repos","events_url":"https://api.github.com/users/P1llus/events{/privacy}","received_events_url":"https://api.github.com/users/P1llus/received_events","type":"User","site_admin":false}],"milestone":null,"comments":2,"created_at":"2022-11-23T15:06:34Z","updated_at":"2022-11-23T15:07:18Z","closed_at":null,"author_association":"NONE","active_lock_reason":null,"body":"Structured Threat Information Expression (STIX) is a language for expressing cyber threat and observable information. While we have several Threat Intel integrations which map STIX formatted data to Elastic Common Schema, users will always have need to ingest IOC's from threat feeds that we don't support out of the box. 'How do I ingest STIX feeds' remains a very common questions across community Slack, Discuss, GitHub, etc. A custom package would solve for this. \r\n\r\nTo allow for the broad range of STIX formatted feeds, we should provide a way for users to ingest data from ANY STIX feed, via a 'Custom STIX' package. The package will leverage our httpjson input under the hood, but include an ingest pipeline which maps STIX fields to ECS (we expect there'll still be a need for custom fields, as not all STIX fields have a corresponding field in ECS). \r\n\r\nThere may be cases where some feeds/vendors don't strictly conform to STIX, and in those cases, users may have to modify our pipeline and that's ok. We'll focus on correctly formatted STIX data. ","reactions":{"url":"https://api.github.com/repos/elastic/integrations/issues/4710/reactions","total_count":0,"+1":0,"-1":0,"laugh":0,"hooray":0,"confused":0,"heart":0,"rocket":0,"eyes":0},"timeline_url":"https://api.github.com/repos/elastic/integrations/issues/4710/timeline","performed_via_github_app":null,"state_reason":null} +{"url":"https://api.github.com/repos/elastic/integrations/issues/4710","repository_url":"https://api.github.com/repos/elastic/integrations","labels_url":"https://api.github.com/repos/elastic/integrations/issues/4710/labels{/name}","comments_url":"https://api.github.com/repos/elastic/integrations/issues/4710/comments","events_url":"https://api.github.com/repos/elastic/integrations/issues/4710/events","html_url":"https://github.com/elastic/integrations/issues/4710","id":1461928292,"node_id":"I_kwDODAw23M5XI0Fk","number":4710,"title":"Custom STIX Package","user":{"login":"jamiehynds","id":62879768,"node_id":"MDQ6VXNlcjYyODc5NzY4","avatar_url":"https://avatars.githubusercontent.com/u/62879768?v=4","gravatar_id":"","url":"https://api.github.com/users/jamiehynds","html_url":"https://github.com/jamiehynds","followers_url":"https://api.github.com/users/jamiehynds/followers","following_url":"https://api.github.com/users/jamiehynds/following{/other_user}","gists_url":"https://api.github.com/users/jamiehynds/gists{/gist_id}","starred_url":"https://api.github.com/users/jamiehynds/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jamiehynds/subscriptions","organizations_url":"https://api.github.com/users/jamiehynds/orgs","repos_url":"https://api.github.com/users/jamiehynds/repos","events_url":"https://api.github.com/users/jamiehynds/events{/privacy}","received_events_url":"https://api.github.com/users/jamiehynds/received_events","type":"User","site_admin":false},"labels":[{"id":2404921703,"node_id":"MDU6TGFiZWwyNDA0OTIxNzAz","url":"https://api.github.com/repos/elastic/integrations/labels/Team:Security-Service%20Integrations","name":"Team:Security-Service Integrations","color":"1d76db","default":false,"description":"Label for the Security External Integrations team"},{"id":3104073484,"node_id":"MDU6TGFiZWwzMTA0MDczNDg0","url":"https://api.github.com/repos/elastic/integrations/labels/Integration:Threat%20Intel","name":"Integration:Threat Intel","color":"ffffff","default":false,"description":""}],"state":"open","locked":false,"assignee":{"login":"P1llus","id":8027539,"node_id":"MDQ6VXNlcjgwMjc1Mzk=","avatar_url":"https://avatars.githubusercontent.com/u/8027539?v=4","gravatar_id":"","url":"https://api.github.com/users/P1llus","html_url":"https://github.com/P1llus","followers_url":"https://api.github.com/users/P1llus/followers","following_url":"https://api.github.com/users/P1llus/following{/other_user}","gists_url":"https://api.github.com/users/P1llus/gists{/gist_id}","starred_url":"https://api.github.com/users/P1llus/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/P1llus/subscriptions","organizations_url":"https://api.github.com/users/P1llus/orgs","repos_url":"https://api.github.com/users/P1llus/repos","events_url":"https://api.github.com/users/P1llus/events{/privacy}","received_events_url":"https://api.github.com/users/P1llus/received_events","type":"User","site_admin":false},"assignees":[{"login":"P1llus","id":8027539,"node_id":"MDQ6VXNlcjgwMjc1Mzk=","avatar_url":"https://avatars.githubusercontent.com/u/8027539?v=4","gravatar_id":"","url":"https://api.github.com/users/P1llus","html_url":"https://github.com/P1llus","followers_url":"https://api.github.com/users/P1llus/followers","following_url":"https://api.github.com/users/P1llus/following{/other_user}","gists_url":"https://api.github.com/users/P1llus/gists{/gist_id}","starred_url":"https://api.github.com/users/P1llus/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/P1llus/subscriptions","organizations_url":"https://api.github.com/users/P1llus/orgs","repos_url":"https://api.github.com/users/P1llus/repos","events_url":"https://api.github.com/users/P1llus/events{/privacy}","received_events_url":"https://api.github.com/users/P1llus/received_events","type":"User","site_admin":false}],"milestone":null,"comments":2,"created_at":"2022-11-23T15:06:34Z","updated_at":"2022-11-23T15:07:18Z","closed_at":null,"author_association":"NONE","active_lock_reason":null,"body":"Structured Threat Information Expression (STIX) is a language for expressing cyber threat and observable information. While we have several Threat Intel integrations which map STIX formatted data to Elastic Common Schema, users will always have need to ingest IOC's from threat feeds that we don't support out of the box. 'How do I ingest STIX feeds' remains a very common questions across community Slack, Discuss, Github, etc. A custom package would solve for this. \r\n\r\nTo allow for the broad range of STIX formatted feeds, we should provide a way for users to ingest data from ANY STIX feed, via a 'Custom STIX' package. The package will leverage our httpjson input under the hood, but include an ingest pipeline which maps STIX fields to ECS (we expect there'll still be a need for custom fields, as not all STIX fields have a corresponding field in ECS). \r\n\r\nThere may be cases where some feeds/vendors don't strictly conform to STIX, and in those cases, users may have to modify our pipeline and that's ok. We'll focus on correctly formatted STIX data. ","reactions":{"url":"https://api.github.com/repos/elastic/integrations/issues/4710/reactions","total_count":0,"+1":0,"-1":0,"laugh":0,"hooray":0,"confused":0,"heart":0,"rocket":0,"eyes":0},"timeline_url":"https://api.github.com/repos/elastic/integrations/issues/4710/timeline","performed_via_github_app":null,"state_reason":null} {"url":"https://api.github.com/repos/elastic/integrations/issues/4707","repository_url":"https://api.github.com/repos/elastic/integrations","labels_url":"https://api.github.com/repos/elastic/integrations/issues/4707/labels{/name}","comments_url":"https://api.github.com/repos/elastic/integrations/issues/4707/comments","events_url":"https://api.github.com/repos/elastic/integrations/issues/4707/events","html_url":"https://github.com/elastic/integrations/issues/4707","id":1461726255,"node_id":"I_kwDODAw23M5XICwv","number":4707,"title":"[ Crowdstrike Falcon ] Parse of process field is wrong.","user":{"login":"leandrojmp","id":322886,"node_id":"MDQ6VXNlcjMyMjg4Ng==","avatar_url":"https://avatars.githubusercontent.com/u/322886?v=4","gravatar_id":"","url":"https://api.github.com/users/leandrojmp","html_url":"https://github.com/leandrojmp","followers_url":"https://api.github.com/users/leandrojmp/followers","following_url":"https://api.github.com/users/leandrojmp/following{/other_user}","gists_url":"https://api.github.com/users/leandrojmp/gists{/gist_id}","starred_url":"https://api.github.com/users/leandrojmp/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/leandrojmp/subscriptions","organizations_url":"https://api.github.com/users/leandrojmp/orgs","repos_url":"https://api.github.com/users/leandrojmp/repos","events_url":"https://api.github.com/users/leandrojmp/events{/privacy}","received_events_url":"https://api.github.com/users/leandrojmp/received_events","type":"User","site_admin":false},"labels":[{"id":2404921703,"node_id":"MDU6TGFiZWwyNDA0OTIxNzAz","url":"https://api.github.com/repos/elastic/integrations/labels/Team:Security-Service%20Integrations","name":"Team:Security-Service Integrations","color":"1d76db","default":false,"description":"Label for the Security External Integrations team"},{"id":2716642190,"node_id":"MDU6TGFiZWwyNzE2NjQyMTkw","url":"https://api.github.com/repos/elastic/integrations/labels/Integration:Crowdstrike","name":"Integration:Crowdstrike","color":"FFFFFF","default":false,"description":null}],"state":"open","locked":false,"assignee":null,"assignees":[],"milestone":null,"comments":1,"created_at":"2022-11-23T13:03:54Z","updated_at":"2022-11-23T14:58:03Z","closed_at":null,"author_association":"NONE","active_lock_reason":null,"body":"Hello,\r\n\r\nThe parse of the process fields in the Crowdstrike Falcon Pipeline is wrong, it is creating fields with dots in the name instead of nested json objects.\r\n\r\nIt creates this:\r\n```\r\n{\r\n \"process.executable\": \"value\"\r\n}\r\n```\r\n\r\nInstead of\r\n\r\n```\r\n{\r\n \"process\": {\r\n \"executable\": \"value\"\r\n }\r\n}\r\n```\r\n\r\nThis same issue was present in the [filebeat integration](https://github.com/elastic/beats/issues/27622) and I submitted a [fix](https://github.com/elastic/beats/pull/27623) more than an year ago.\r\n\r\nI will submit a PR later with the same fix, but I'm creating this issue to register the problem.","reactions":{"url":"https://api.github.com/repos/elastic/integrations/issues/4707/reactions","total_count":0,"+1":0,"-1":0,"laugh":0,"hooray":0,"confused":0,"heart":0,"rocket":0,"eyes":0},"timeline_url":"https://api.github.com/repos/elastic/integrations/issues/4707/timeline","performed_via_github_app":null,"state_reason":null } {"url":"https://api.github.com/repos/elastic/integrations/issues/4704","repository_url":"https://api.github.com/repos/elastic/integrations","labels_url":"https://api.github.com/repos/elastic/integrations/issues/4704/labels{/name}","comments_url":"https://api.github.com/repos/elastic/integrations/issues/4704/comments","events_url":"https://api.github.com/repos/elastic/integrations/issues/4704/events","html_url":"https://github.com/elastic/integrations/pull/4704","id":1461524465,"node_id":"PR_kwDODAw23M5DjMOh","number":4704,"title":"[Enhancement] [Infoblox Bloxone DDI] Update the Pagination Termination Condition and Added Filter instead of KQL in visualizations","user":{"login":"vinit-elastic","id":89848047,"node_id":"MDQ6VXNlcjg5ODQ4MDQ3","avatar_url":"https://avatars.githubusercontent.com/u/89848047?v=4","gravatar_id":"","url":"https://api.github.com/users/vinit-elastic","html_url":"https://github.com/vinit-elastic","followers_url":"https://api.github.com/users/vinit-elastic/followers","following_url":"https://api.github.com/users/vinit-elastic/following{/other_user}","gists_url":"https://api.github.com/users/vinit-elastic/gists{/gist_id}","starred_url":"https://api.github.com/users/vinit-elastic/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/vinit-elastic/subscriptions","organizations_url":"https://api.github.com/users/vinit-elastic/orgs","repos_url":"https://api.github.com/users/vinit-elastic/repos","events_url":"https://api.github.com/users/vinit-elastic/events{/privacy}","received_events_url":"https://api.github.com/users/vinit-elastic/received_events","type":"User","site_admin":false},"labels":[{"id":1498531540,"node_id":"MDU6TGFiZWwxNDk4NTMxNTQw","url":"https://api.github.com/repos/elastic/integrations/labels/enhancement","name":"enhancement","color":"a2eeef","default":true,"description":"New feature or request"},{"id":2404921703,"node_id":"MDU6TGFiZWwyNDA0OTIxNzAz","url":"https://api.github.com/repos/elastic/integrations/labels/Team:Security-Service%20Integrations","name":"Team:Security-Service Integrations","color":"1d76db","default":false,"description":"Label for the Security External Integrations team"},{"id":4528694847,"node_id":"LA_kwDODAw23M8AAAABDe5mPw","url":"https://api.github.com/repos/elastic/integrations/labels/Integration:infoblox_bloxone_ddi","name":"Integration:infoblox_bloxone_ddi","color":"FFFFFF","default":false,"description":"Infoblox BloxOne DDI (DNS, DHCP, IP management)"}],"state":"open","locked":false,"assignee":null,"assignees":[],"milestone":null,"comments":4,"created_at":"2022-11-23T10:57:54Z","updated_at":"2022-11-23T11:15:48Z","closed_at":null,"author_association":"COLLABORATOR","active_lock_reason":null,"draft":false,"pull_request":{"url":"https://api.github.com/repos/elastic/integrations/pulls/4704","html_url":"https://github.com/elastic/integrations/pull/4704","diff_url":"https://github.com/elastic/integrations/pull/4704.diff","patch_url":"https://github.com/elastic/integrations/pull/4704.patch","merged_at":null},"body":" Type of change\r\n- Enhancement\r\n\r\n\r\n## What does this PR do?\r\nUpdate the Pagination Termination Condition for Infoblox Bloxone DDI connector.\r\n\r\nCurrent condition for pagination termination contains `[[else]][[.last_response.terminate_pagination]][[end]]` which results in error logs when pagination is completed.\r\n\r\nRemoving this `else` condition will not result in error logs.\r\n\r\n\r\n\r\n## Checklist\r\n\r\n- [x] I have reviewed [tips for building integrations](https://github.com/elastic/integrations/blob/main/docs/tips_for_building_integrations.md) and this pull request is aligned with them.\r\n- [x] I have verified that all data streams collect metrics or logs.\r\n- [x] I have added an entry to my package's `changelog.yml` file.\r\n- [x] I have verified that Kibana version constraints are current according to [guidelines](https://github.com/elastic/elastic-package/blob/master/docs/howto/stack_version_support.md#when-to-update-the-condition).\r\n\r\n\r\n\r\n## How to test this PR locally\r\n- Clone integrations repo.\r\n- Install elastic package locally.\r\n- Start elastic stack using elastic-package.\r\n- Move to integrations/packages/infoblox_bloxone_ddi directory.\r\n- Run the following command to run tests. \r\n> elastic-package test \r\n\r\n\r\n## Related issues\r\n\r\n\r\n- Relates https://github.com/elastic/integrations/issues/4527\r\n\r\n## Screenshots\r\n![image](https://user-images.githubusercontent.com/89848047/203529720-18c110cd-5343-4d70-bbfb-eed0b81313af.png)\r\n![image](https://user-images.githubusercontent.com/89848047/203529765-4b183071-6ed9-4bc8-8a98-fd04955bb4d7.png)\r\n![image](https://user-images.githubusercontent.com/89848047/203529819-78128d98-630e-49e5-bf97-f86059c0cc26.png)\r\n![image](https://user-images.githubusercontent.com/89848047/203529936-df7e3d6f-031e-4b0c-992f-93707a507735.png)\r\n\r\n\r\n","reactions":{"url":"https://api.github.com/repos/elastic/integrations/issues/4704/reactions","total_count":0,"+1":0,"-1":0,"laugh":0,"hooray":0,"confused":0,"heart":0,"rocket":0,"eyes":0},"timeline_url":"https://api.github.com/repos/elastic/integrations/issues/4704/timeline","performed_via_github_app":null,"state_reason":null } {"url":"https://api.github.com/repos/elastic/integrations/issues/4703","repository_url":"https://api.github.com/repos/elastic/integrations","labels_url":"https://api.github.com/repos/elastic/integrations/issues/4703/labels{/name}","comments_url":"https://api.github.com/repos/elastic/integrations/issues/4703/comments","events_url":"https://api.github.com/repos/elastic/integrations/issues/4703/events","html_url":"https://github.com/elastic/integrations/pull/4703","id":1461503410,"node_id":"PR_kwDODAw23M5DjHjk","number":4703,"title":"[Enhancement] [AWS Security Hub] Update the Pagination Termination Condition","user":{"login":"vinit-elastic","id":89848047,"node_id":"MDQ6VXNlcjg5ODQ4MDQ3","avatar_url":"https://avatars.githubusercontent.com/u/89848047?v=4","gravatar_id":"","url":"https://api.github.com/users/vinit-elastic","html_url":"https://github.com/vinit-elastic","followers_url":"https://api.github.com/users/vinit-elastic/followers","following_url":"https://api.github.com/users/vinit-elastic/following{/other_user}","gists_url":"https://api.github.com/users/vinit-elastic/gists{/gist_id}","starred_url":"https://api.github.com/users/vinit-elastic/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/vinit-elastic/subscriptions","organizations_url":"https://api.github.com/users/vinit-elastic/orgs","repos_url":"https://api.github.com/users/vinit-elastic/repos","events_url":"https://api.github.com/users/vinit-elastic/events{/privacy}","received_events_url":"https://api.github.com/users/vinit-elastic/received_events","type":"User","site_admin":false},"labels":[{"id":1498531540,"node_id":"MDU6TGFiZWwxNDk4NTMxNTQw","url":"https://api.github.com/repos/elastic/integrations/labels/enhancement","name":"enhancement","color":"a2eeef","default":true,"description":"New feature or request"},{"id":2404921703,"node_id":"MDU6TGFiZWwyNDA0OTIxNzAz","url":"https://api.github.com/repos/elastic/integrations/labels/Team:Security-Service%20Integrations","name":"Team:Security-Service Integrations","color":"1d76db","default":false,"description":"Label for the Security External Integrations team"},{"id":2607750240,"node_id":"MDU6TGFiZWwyNjA3NzUwMjQw","url":"https://api.github.com/repos/elastic/integrations/labels/Integration:AWS","name":"Integration:AWS","color":"FFFFFF","default":false,"description":""}],"state":"open","locked":false,"assignee":null,"assignees":[],"milestone":null,"comments":4,"created_at":"2022-11-23T10:44:59Z","updated_at":"2022-11-23T11:23:56Z","closed_at":null,"author_association":"COLLABORATOR","active_lock_reason":null,"draft":false,"pull_request":{"url":"https://api.github.com/repos/elastic/integrations/pulls/4703","html_url":"https://github.com/elastic/integrations/pull/4703","diff_url":"https://github.com/elastic/integrations/pull/4703.diff","patch_url":"https://github.com/elastic/integrations/pull/4703.patch","merged_at":null},"body":" Type of change\r\n- Enhancement\r\n\r\n\r\n## What does this PR do?\r\nUpdate the Pagination Termination Condition for AWS Security Hub connector.\r\n\r\nCurrent condition for pagination termination contains `[[else]][[.last_response.terminate_pagination]][[end]]` which results in error logs when pagination is completed.\r\n\r\nRemoving this `else` condition will not result in error logs.\r\n\r\n\r\n\r\n## Checklist\r\n\r\n- [x] I have reviewed [tips for building integrations](https://github.com/elastic/integrations/blob/main/docs/tips_for_building_integrations.md) and this pull request is aligned with them.\r\n- [x] I have verified that all data streams collect metrics or logs.\r\n- [x] I have added an entry to my package's `changelog.yml` file.\r\n- [x] I have verified that Kibana version constraints are current according to [guidelines](https://github.com/elastic/elastic-package/blob/master/docs/howto/stack_version_support.md#when-to-update-the-condition).\r\n\r\n\r\n\r\n## How to test this PR locally\r\n- Clone integrations repo.\r\n- Install elastic package locally.\r\n- Start elastic stack using elastic-package.\r\n- Move to integrations/packages/aws directory.\r\n- Run the following command to run tests. \r\n> elastic-package test \r\n\r\n\r\n## Related issues\r\n\r\n\r\n- Relates https://github.com/elastic/integrations/issues/4527\r\n\r\n## Screenshots\r\n![image](https://user-images.githubusercontent.com/89848047/203526900-06519c95-3de5-4f88-a65d-555d03d7f9a1.png)\r\n![image](https://user-images.githubusercontent.com/89848047/203527000-b050f059-03c6-4ead-a88e-9f2e42efb537.png)\r\n![image](https://user-images.githubusercontent.com/89848047/203527123-ea1f8513-8371-4d06-8a17-b71c0dae7a5a.png)\r\n\r\n\r\n","reactions":{"url":"https://api.github.com/repos/elastic/integrations/issues/4703/reactions","total_count":0,"+1":0,"-1":0,"laugh":0,"hooray":0,"confused":0,"heart":0,"rocket":0,"eyes":0},"timeline_url":"https://api.github.com/repos/elastic/integrations/issues/4703/timeline","performed_via_github_app":null,"state_reason":null } \ No newline at end of file diff --git a/packages/github/data_stream/issues/_dev/test/pipeline/test-github-issues-json.log-expected.json b/packages/github/data_stream/issues/_dev/test/pipeline/test-github-issues-json.log-expected.json index aeb5bd586e3..50b06599fc1 100644 --- a/packages/github/data_stream/issues/_dev/test/pipeline/test-github-issues-json.log-expected.json +++ b/packages/github/data_stream/issues/_dev/test/pipeline/test-github-issues-json.log-expected.json @@ -114,7 +114,7 @@ }, "event": { "created": "2022-11-23T15:07:18.000Z", - "original": "{\"url\":\"https://api.github.com/repos/elastic/integrations/issues/4710\",\"repository_url\":\"https://api.github.com/repos/elastic/integrations\",\"labels_url\":\"https://api.github.com/repos/elastic/integrations/issues/4710/labels{/name}\",\"comments_url\":\"https://api.github.com/repos/elastic/integrations/issues/4710/comments\",\"events_url\":\"https://api.github.com/repos/elastic/integrations/issues/4710/events\",\"html_url\":\"https://github.com/elastic/integrations/issues/4710\",\"id\":1461928292,\"node_id\":\"I_kwDODAw23M5XI0Fk\",\"number\":4710,\"title\":\"Custom STIX Package\",\"user\":{\"login\":\"jamiehynds\",\"id\":62879768,\"node_id\":\"MDQ6VXNlcjYyODc5NzY4\",\"avatar_url\":\"https://avatars.githubusercontent.com/u/62879768?v=4\",\"gravatar_id\":\"\",\"url\":\"https://api.github.com/users/jamiehynds\",\"html_url\":\"https://github.com/jamiehynds\",\"followers_url\":\"https://api.github.com/users/jamiehynds/followers\",\"following_url\":\"https://api.github.com/users/jamiehynds/following{/other_user}\",\"gists_url\":\"https://api.github.com/users/jamiehynds/gists{/gist_id}\",\"starred_url\":\"https://api.github.com/users/jamiehynds/starred{/owner}{/repo}\",\"subscriptions_url\":\"https://api.github.com/users/jamiehynds/subscriptions\",\"organizations_url\":\"https://api.github.com/users/jamiehynds/orgs\",\"repos_url\":\"https://api.github.com/users/jamiehynds/repos\",\"events_url\":\"https://api.github.com/users/jamiehynds/events{/privacy}\",\"received_events_url\":\"https://api.github.com/users/jamiehynds/received_events\",\"type\":\"User\",\"site_admin\":false},\"labels\":[{\"id\":2404921703,\"node_id\":\"MDU6TGFiZWwyNDA0OTIxNzAz\",\"url\":\"https://api.github.com/repos/elastic/integrations/labels/Team:Security-Service%20Integrations\",\"name\":\"Team:Security-Service Integrations\",\"color\":\"1d76db\",\"default\":false,\"description\":\"Label for the Security External Integrations team\"},{\"id\":3104073484,\"node_id\":\"MDU6TGFiZWwzMTA0MDczNDg0\",\"url\":\"https://api.github.com/repos/elastic/integrations/labels/Integration:Threat%20Intel\",\"name\":\"Integration:Threat Intel\",\"color\":\"ffffff\",\"default\":false,\"description\":\"\"}],\"state\":\"open\",\"locked\":false,\"assignee\":{\"login\":\"P1llus\",\"id\":8027539,\"node_id\":\"MDQ6VXNlcjgwMjc1Mzk=\",\"avatar_url\":\"https://avatars.githubusercontent.com/u/8027539?v=4\",\"gravatar_id\":\"\",\"url\":\"https://api.github.com/users/P1llus\",\"html_url\":\"https://github.com/P1llus\",\"followers_url\":\"https://api.github.com/users/P1llus/followers\",\"following_url\":\"https://api.github.com/users/P1llus/following{/other_user}\",\"gists_url\":\"https://api.github.com/users/P1llus/gists{/gist_id}\",\"starred_url\":\"https://api.github.com/users/P1llus/starred{/owner}{/repo}\",\"subscriptions_url\":\"https://api.github.com/users/P1llus/subscriptions\",\"organizations_url\":\"https://api.github.com/users/P1llus/orgs\",\"repos_url\":\"https://api.github.com/users/P1llus/repos\",\"events_url\":\"https://api.github.com/users/P1llus/events{/privacy}\",\"received_events_url\":\"https://api.github.com/users/P1llus/received_events\",\"type\":\"User\",\"site_admin\":false},\"assignees\":[{\"login\":\"P1llus\",\"id\":8027539,\"node_id\":\"MDQ6VXNlcjgwMjc1Mzk=\",\"avatar_url\":\"https://avatars.githubusercontent.com/u/8027539?v=4\",\"gravatar_id\":\"\",\"url\":\"https://api.github.com/users/P1llus\",\"html_url\":\"https://github.com/P1llus\",\"followers_url\":\"https://api.github.com/users/P1llus/followers\",\"following_url\":\"https://api.github.com/users/P1llus/following{/other_user}\",\"gists_url\":\"https://api.github.com/users/P1llus/gists{/gist_id}\",\"starred_url\":\"https://api.github.com/users/P1llus/starred{/owner}{/repo}\",\"subscriptions_url\":\"https://api.github.com/users/P1llus/subscriptions\",\"organizations_url\":\"https://api.github.com/users/P1llus/orgs\",\"repos_url\":\"https://api.github.com/users/P1llus/repos\",\"events_url\":\"https://api.github.com/users/P1llus/events{/privacy}\",\"received_events_url\":\"https://api.github.com/users/P1llus/received_events\",\"type\":\"User\",\"site_admin\":false}],\"milestone\":null,\"comments\":2,\"created_at\":\"2022-11-23T15:06:34Z\",\"updated_at\":\"2022-11-23T15:07:18Z\",\"closed_at\":null,\"author_association\":\"NONE\",\"active_lock_reason\":null,\"body\":\"Structured Threat Information Expression (STIX) is a language for expressing cyber threat and observable information. While we have several Threat Intel integrations which map STIX formatted data to Elastic Common Schema, users will always have need to ingest IOC's from threat feeds that we don't support out of the box. 'How do I ingest STIX feeds' remains a very common questions across community Slack, Discuss, GitHub, etc. A custom package would solve for this. \\r\\n\\r\\nTo allow for the broad range of STIX formatted feeds, we should provide a way for users to ingest data from ANY STIX feed, via a 'Custom STIX' package. The package will leverage our httpjson input under the hood, but include an ingest pipeline which maps STIX fields to ECS (we expect there'll still be a need for custom fields, as not all STIX fields have a corresponding field in ECS). \\r\\n\\r\\nThere may be cases where some feeds/vendors don't strictly conform to STIX, and in those cases, users may have to modify our pipeline and that's ok. We'll focus on correctly formatted STIX data. \",\"reactions\":{\"url\":\"https://api.github.com/repos/elastic/integrations/issues/4710/reactions\",\"total_count\":0,\"+1\":0,\"-1\":0,\"laugh\":0,\"hooray\":0,\"confused\":0,\"heart\":0,\"rocket\":0,\"eyes\":0},\"timeline_url\":\"https://api.github.com/repos/elastic/integrations/issues/4710/timeline\",\"performed_via_github_app\":null,\"state_reason\":null}", + "original": "{\"url\":\"https://api.github.com/repos/elastic/integrations/issues/4710\",\"repository_url\":\"https://api.github.com/repos/elastic/integrations\",\"labels_url\":\"https://api.github.com/repos/elastic/integrations/issues/4710/labels{/name}\",\"comments_url\":\"https://api.github.com/repos/elastic/integrations/issues/4710/comments\",\"events_url\":\"https://api.github.com/repos/elastic/integrations/issues/4710/events\",\"html_url\":\"https://github.com/elastic/integrations/issues/4710\",\"id\":1461928292,\"node_id\":\"I_kwDODAw23M5XI0Fk\",\"number\":4710,\"title\":\"Custom STIX Package\",\"user\":{\"login\":\"jamiehynds\",\"id\":62879768,\"node_id\":\"MDQ6VXNlcjYyODc5NzY4\",\"avatar_url\":\"https://avatars.githubusercontent.com/u/62879768?v=4\",\"gravatar_id\":\"\",\"url\":\"https://api.github.com/users/jamiehynds\",\"html_url\":\"https://github.com/jamiehynds\",\"followers_url\":\"https://api.github.com/users/jamiehynds/followers\",\"following_url\":\"https://api.github.com/users/jamiehynds/following{/other_user}\",\"gists_url\":\"https://api.github.com/users/jamiehynds/gists{/gist_id}\",\"starred_url\":\"https://api.github.com/users/jamiehynds/starred{/owner}{/repo}\",\"subscriptions_url\":\"https://api.github.com/users/jamiehynds/subscriptions\",\"organizations_url\":\"https://api.github.com/users/jamiehynds/orgs\",\"repos_url\":\"https://api.github.com/users/jamiehynds/repos\",\"events_url\":\"https://api.github.com/users/jamiehynds/events{/privacy}\",\"received_events_url\":\"https://api.github.com/users/jamiehynds/received_events\",\"type\":\"User\",\"site_admin\":false},\"labels\":[{\"id\":2404921703,\"node_id\":\"MDU6TGFiZWwyNDA0OTIxNzAz\",\"url\":\"https://api.github.com/repos/elastic/integrations/labels/Team:Security-Service%20Integrations\",\"name\":\"Team:Security-Service Integrations\",\"color\":\"1d76db\",\"default\":false,\"description\":\"Label for the Security External Integrations team\"},{\"id\":3104073484,\"node_id\":\"MDU6TGFiZWwzMTA0MDczNDg0\",\"url\":\"https://api.github.com/repos/elastic/integrations/labels/Integration:Threat%20Intel\",\"name\":\"Integration:Threat Intel\",\"color\":\"ffffff\",\"default\":false,\"description\":\"\"}],\"state\":\"open\",\"locked\":false,\"assignee\":{\"login\":\"P1llus\",\"id\":8027539,\"node_id\":\"MDQ6VXNlcjgwMjc1Mzk=\",\"avatar_url\":\"https://avatars.githubusercontent.com/u/8027539?v=4\",\"gravatar_id\":\"\",\"url\":\"https://api.github.com/users/P1llus\",\"html_url\":\"https://github.com/P1llus\",\"followers_url\":\"https://api.github.com/users/P1llus/followers\",\"following_url\":\"https://api.github.com/users/P1llus/following{/other_user}\",\"gists_url\":\"https://api.github.com/users/P1llus/gists{/gist_id}\",\"starred_url\":\"https://api.github.com/users/P1llus/starred{/owner}{/repo}\",\"subscriptions_url\":\"https://api.github.com/users/P1llus/subscriptions\",\"organizations_url\":\"https://api.github.com/users/P1llus/orgs\",\"repos_url\":\"https://api.github.com/users/P1llus/repos\",\"events_url\":\"https://api.github.com/users/P1llus/events{/privacy}\",\"received_events_url\":\"https://api.github.com/users/P1llus/received_events\",\"type\":\"User\",\"site_admin\":false},\"assignees\":[{\"login\":\"P1llus\",\"id\":8027539,\"node_id\":\"MDQ6VXNlcjgwMjc1Mzk=\",\"avatar_url\":\"https://avatars.githubusercontent.com/u/8027539?v=4\",\"gravatar_id\":\"\",\"url\":\"https://api.github.com/users/P1llus\",\"html_url\":\"https://github.com/P1llus\",\"followers_url\":\"https://api.github.com/users/P1llus/followers\",\"following_url\":\"https://api.github.com/users/P1llus/following{/other_user}\",\"gists_url\":\"https://api.github.com/users/P1llus/gists{/gist_id}\",\"starred_url\":\"https://api.github.com/users/P1llus/starred{/owner}{/repo}\",\"subscriptions_url\":\"https://api.github.com/users/P1llus/subscriptions\",\"organizations_url\":\"https://api.github.com/users/P1llus/orgs\",\"repos_url\":\"https://api.github.com/users/P1llus/repos\",\"events_url\":\"https://api.github.com/users/P1llus/events{/privacy}\",\"received_events_url\":\"https://api.github.com/users/P1llus/received_events\",\"type\":\"User\",\"site_admin\":false}],\"milestone\":null,\"comments\":2,\"created_at\":\"2022-11-23T15:06:34Z\",\"updated_at\":\"2022-11-23T15:07:18Z\",\"closed_at\":null,\"author_association\":\"NONE\",\"active_lock_reason\":null,\"body\":\"Structured Threat Information Expression (STIX) is a language for expressing cyber threat and observable information. While we have several Threat Intel integrations which map STIX formatted data to Elastic Common Schema, users will always have need to ingest IOC's from threat feeds that we don't support out of the box. 'How do I ingest STIX feeds' remains a very common questions across community Slack, Discuss, Github, etc. A custom package would solve for this. \\r\\n\\r\\nTo allow for the broad range of STIX formatted feeds, we should provide a way for users to ingest data from ANY STIX feed, via a 'Custom STIX' package. The package will leverage our httpjson input under the hood, but include an ingest pipeline which maps STIX fields to ECS (we expect there'll still be a need for custom fields, as not all STIX fields have a corresponding field in ECS). \\r\\n\\r\\nThere may be cases where some feeds/vendors don't strictly conform to STIX, and in those cases, users may have to modify our pipeline and that's ok. We'll focus on correctly formatted STIX data. \",\"reactions\":{\"url\":\"https://api.github.com/repos/elastic/integrations/issues/4710/reactions\",\"total_count\":0,\"+1\":0,\"-1\":0,\"laugh\":0,\"hooray\":0,\"confused\":0,\"heart\":0,\"rocket\":0,\"eyes\":0},\"timeline_url\":\"https://api.github.com/repos/elastic/integrations/issues/4710/timeline\",\"performed_via_github_app\":null,\"state_reason\":null}", "type": [ "change" ] @@ -140,7 +140,7 @@ } ], "author_association": "NONE", - "body": "Structured Threat Information Expression (STIX) is a language for expressing cyber threat and observable information. While we have several Threat Intel integrations which map STIX formatted data to Elastic Common Schema, users will always have need to ingest IOC's from threat feeds that we don't support out of the box. 'How do I ingest STIX feeds' remains a very common questions across community Slack, Discuss, GitHub, etc. A custom package would solve for this. \r\n\r\nTo allow for the broad range of STIX formatted feeds, we should provide a way for users to ingest data from ANY STIX feed, via a 'Custom STIX' package. The package will leverage our httpjson input under the hood, but include an ingest pipeline which maps STIX fields to ECS (we expect there'll still be a need for custom fields, as not all STIX fields have a corresponding field in ECS). \r\n\r\nThere may be cases where some feeds/vendors don't strictly conform to STIX, and in those cases, users may have to modify our pipeline and that's ok. We'll focus on correctly formatted STIX data. ", + "body": "Structured Threat Information Expression (STIX) is a language for expressing cyber threat and observable information. While we have several Threat Intel integrations which map STIX formatted data to Elastic Common Schema, users will always have need to ingest IOC's from threat feeds that we don't support out of the box. 'How do I ingest STIX feeds' remains a very common questions across community Slack, Discuss, Github, etc. A custom package would solve for this. \r\n\r\nTo allow for the broad range of STIX formatted feeds, we should provide a way for users to ingest data from ANY STIX feed, via a 'Custom STIX' package. The package will leverage our httpjson input under the hood, but include an ingest pipeline which maps STIX fields to ECS (we expect there'll still be a need for custom fields, as not all STIX fields have a corresponding field in ECS). \r\n\r\nThere may be cases where some feeds/vendors don't strictly conform to STIX, and in those cases, users may have to modify our pipeline and that's ok. We'll focus on correctly formatted STIX data. ", "comments": 2, "comments_url": "https://api.github.com/repos/elastic/integrations/issues/4710/comments", "created_at": "2022-11-23T15:06:34.000Z", diff --git a/packages/github/docs/README.md b/packages/github/docs/README.md index 93ed0756640..023d9d15a1d 100644 --- a/packages/github/docs/README.md +++ b/packages/github/docs/README.md @@ -25,6 +25,8 @@ For Organizations: The GitHub audit log records all events related to the GitHub organization/enterprise. See [Organization audit log actions](https://docs.github.com/en/organizations/keeping-your-organization-secure/reviewing-the-audit-log-for-your-organization#audit-log-actions) and [Enterprise audit log actions](https://docs.github.com/en/enterprise-cloud@latest/admin/monitoring-activity-in-your-enterprise/reviewing-audit-logs-for-your-enterprise/about-the-audit-log-for-your-enterprise) for more details. +The GitHub integration can collect audit logs from the following sources: [GitHub API](https://docs.github.com/en/enterprise-cloud@latest/admin/monitoring-activity-in-your-enterprise/reviewing-audit-logs-for-your-enterprise/using-the-audit-log-api-for-your-enterprise), [Azure Event Hubs](https://docs.github.com/en/enterprise-cloud@latest/admin/monitoring-activity-in-your-enterprise/reviewing-audit-logs-for-your-enterprise/streaming-the-audit-log-for-your-enterprise#setting-up-streaming-to-azure-event-hubs), [Azure Blob Storage](https://docs.github.com/en/enterprise-cloud@latest/admin/monitoring-activity-in-your-enterprise/reviewing-audit-logs-for-your-enterprise/streaming-the-audit-log-for-your-enterprise#setting-up-streaming-to-azure-blob-storage), [AWS S3 or AWS SQS](https://docs.github.com/en/enterprise-cloud@latest/admin/monitoring-activity-in-your-enterprise/reviewing-audit-logs-for-your-enterprise/streaming-the-audit-log-for-your-enterprise#setting-up-streaming-to-amazon-s3) and [Google Cloud Storage](https://docs.github.com/en/enterprise-cloud@latest/admin/monitoring-activity-in-your-enterprise/reviewing-audit-logs-for-your-enterprise/streaming-the-audit-log-for-your-enterprise#setting-up-streaming-to-google-cloud-storage). + When using GitHub API to collect audit log events, below requirements must be met for Personal Access Token (PAT): - You must use a Personal Access Token with `read:audit_log` scope. This applies to both organization and enterprise admins. - If you're an enterprise admin, ensure your token also includes `admin:enterprise` scope to access enterprise-wide logs. @@ -753,7 +755,7 @@ An example event for `dependabot` looks as following: "https://nodesecurity.io/advisories/17" ], "scanner": { - "vendor": "GitHub" + "vendor": "Github" }, "score": { "base": 0 From 73f4922cf056737bb50974f107cbc92a7cc2ec7b Mon Sep 17 00:00:00 2001 From: Shourie Ganguly Date: Thu, 18 Sep 2025 09:16:14 +0530 Subject: [PATCH 15/18] made consistent calle before caller tructure --- .../deploy/docker/gcs-mock-service/main.go | 145 +++++++++--------- 1 file changed, 70 insertions(+), 75 deletions(-) diff --git a/packages/github/_dev/deploy/docker/gcs-mock-service/main.go b/packages/github/_dev/deploy/docker/gcs-mock-service/main.go index baecae43313..5fbaac1ad49 100644 --- a/packages/github/_dev/deploy/docker/gcs-mock-service/main.go +++ b/packages/github/_dev/deploy/docker/gcs-mock-service/main.go @@ -58,36 +58,47 @@ type File struct { ContentType string `yaml:"content-type"` } -func handleRequests(w http.ResponseWriter, r *http.Request) { - path := r.URL.Path - log.Printf("Received request: %s %s", r.Method, path) +func createBucket(bucketName string) error { + if _, exists := inMemoryStore[bucketName]; exists { + return fmt.Errorf("bucket already exists") + } + inMemoryStore[bucketName] = make(map[string]ObjectData) + log.Printf("created bucket: %s", bucketName) + return nil +} - switch r.Method { - case "GET": - if strings.HasPrefix(path, "/health") { - healthHandler(w, r) - return - } - if strings.HasPrefix(path, "/storage/v1/b/") && strings.HasSuffix(path, "/o") { - handleListObjects(w, r) - return - } - // route for getting an object (either path-style or API-style) - handleGetObject(w, r) - case "POST": - // route for creating a bucket: /storage/v1/b - if path == "/storage/v1/b" { - handleCreateBucket(w, r) - return - } - // route for uploading an object: /upload/storage/v1/b/{bucket}/o - if strings.HasPrefix(path, "/upload/storage/v1/b/") { - handleUploadObject(w, r) - return - } - default: - http.NotFound(w, r) +func uploadObject(bucketName, objectName string, data []byte, contentType string) (*GCSObject, error) { + if _, ok := inMemoryStore[bucketName]; !ok { + return nil, fmt.Errorf("bucket not found") + } + + inMemoryStore[bucketName][objectName] = ObjectData{ + Data: data, + ContentType: contentType, + } + log.Printf("created object '%s' in bucket '%s' with Content-Type '%s'", objectName, bucketName, contentType) + + return &GCSObject{ + Kind: "storage#object", + Name: objectName, + Bucket: bucketName, + Size: strconv.Itoa(len(data)), + ContentType: contentType, + }, nil +} + +func readManifest(path string) (*Manifest, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("failed to read manifest: %w", err) } + + var manifest Manifest + if err := yaml.Unmarshal(data, &manifest); err != nil { + return nil, fmt.Errorf("failed to parse manifest: %w", err) + } + + return &manifest, nil } // healthHandler responds with a simple "OK" message. @@ -117,15 +128,6 @@ func handleCreateBucket(w http.ResponseWriter, r *http.Request) { json.NewEncoder(w).Encode(bucketInfo) } -func createBucket(bucketName string) error { - if _, exists := inMemoryStore[bucketName]; exists { - return fmt.Errorf("bucket already exists") - } - inMemoryStore[bucketName] = make(map[string]ObjectData) - log.Printf("created bucket: %s", bucketName) - return nil -} - // handleUploadObject uploads a new file to a bucket. func handleUploadObject(w http.ResponseWriter, r *http.Request) { pathParts := strings.Split(strings.Trim(r.URL.Path, "/"), "/") @@ -162,27 +164,7 @@ func handleUploadObject(w http.ResponseWriter, r *http.Request) { json.NewEncoder(w).Encode(response) } -func uploadObject(bucketName, objectName string, data []byte, contentType string) (*GCSObject, error) { - if _, ok := inMemoryStore[bucketName]; !ok { - return nil, fmt.Errorf("bucket not found") - } - - inMemoryStore[bucketName][objectName] = ObjectData{ - Data: data, - ContentType: contentType, - } - log.Printf("created object '%s' in bucket '%s' with Content-Type '%s'", objectName, bucketName, contentType) - - return &GCSObject{ - Kind: "storage#object", - Name: objectName, - Bucket: bucketName, - Size: strconv.Itoa(len(data)), - ContentType: contentType, - }, nil -} - -// handleGetObject is for retrieving an object from a bucket. +// handleGetObject retrieves an object from a bucket. func handleGetObject(w http.ResponseWriter, r *http.Request) { var bucketName, objectName string path := strings.Trim(r.URL.Path, "/") @@ -241,21 +223,6 @@ func handleListObjects(w http.ResponseWriter, r *http.Request) { http.Error(w, "not found", http.StatusNotFound) } -// readManifest reads and parses the YAML manifest file. -func readManifest(path string) (*Manifest, error) { - data, err := os.ReadFile(path) - if err != nil { - return nil, fmt.Errorf("failed to read manifest: %w", err) - } - - var manifest Manifest - if err := yaml.Unmarshal(data, &manifest); err != nil { - return nil, fmt.Errorf("failed to parse manifest: %w", err) - } - - return &manifest, nil -} - // processManifest processes the manifest to create buckets and upload files. func processManifest(manifest *Manifest) error { for bucketName, bucket := range manifest.Buckets { @@ -268,7 +235,6 @@ func processManifest(manifest *Manifest) error { if err != nil { return fmt.Errorf("failed to read bucket data file '%s': %w", file.Path, err) } - // atm we use only the file name as the object name. pathParts := strings.Split(file.Path, "/") if _, err := uploadObject(bucketName, pathParts[len(pathParts)-1], data, file.ContentType); err != nil { return fmt.Errorf("failed to create object '%s' in bucket '%s': %w", file.Path, bucketName, err) @@ -278,6 +244,36 @@ func processManifest(manifest *Manifest) error { return nil } +// handleRequests is the top-level router. +func handleRequests(w http.ResponseWriter, r *http.Request) { + path := r.URL.Path + log.Printf("Received request: %s %s", r.Method, path) + + switch r.Method { + case "GET": + if strings.HasPrefix(path, "/health") { + healthHandler(w, r) + return + } + if strings.HasPrefix(path, "/storage/v1/b/") && strings.HasSuffix(path, "/o") { + handleListObjects(w, r) + return + } + handleGetObject(w, r) + case "POST": + if path == "/storage/v1/b" { + handleCreateBucket(w, r) + return + } + if strings.HasPrefix(path, "/upload/storage/v1/b/") { + handleUploadObject(w, r) + return + } + default: + http.NotFound(w, r) + } +} + func main() { host := flag.String("host", "0.0.0.0", "host to listen on") port := flag.String("port", "4443", "port to listen on") @@ -287,7 +283,6 @@ func main() { addr := fmt.Sprintf("%s:%s", *host, *port) fmt.Printf("Starting mock GCS server on http://%s\n", addr) - // if a manifest file is provided, read it and preload the buckets and objects. if *manifest != "" { m, err := readManifest(*manifest) if err != nil { From b129277999c4ed3c00b9479a3f5290a948466bf6 Mon Sep 17 00:00:00 2001 From: Shourie Ganguly Date: Thu, 18 Sep 2025 10:36:40 +0530 Subject: [PATCH 16/18] updated with Dans suggestions --- .../deploy/docker/gcs-mock-service/main.go | 300 +++++++++--------- .../github/data_stream/audit/manifest.yml | 20 +- 2 files changed, 162 insertions(+), 158 deletions(-) diff --git a/packages/github/_dev/deploy/docker/gcs-mock-service/main.go b/packages/github/_dev/deploy/docker/gcs-mock-service/main.go index 5fbaac1ad49..b853ce44359 100644 --- a/packages/github/_dev/deploy/docker/gcs-mock-service/main.go +++ b/packages/github/_dev/deploy/docker/gcs-mock-service/main.go @@ -58,33 +58,54 @@ type File struct { ContentType string `yaml:"content-type"` } -func createBucket(bucketName string) error { - if _, exists := inMemoryStore[bucketName]; exists { - return fmt.Errorf("bucket already exists") - } - inMemoryStore[bucketName] = make(map[string]ObjectData) - log.Printf("created bucket: %s", bucketName) - return nil -} +func main() { + host := flag.String("host", "0.0.0.0", "host to listen on") + port := flag.String("port", "4443", "port to listen on") + manifest := flag.String("manifest", "", "path to YAML manifest file for preloading buckets and objects") + flag.Parse() -func uploadObject(bucketName, objectName string, data []byte, contentType string) (*GCSObject, error) { - if _, ok := inMemoryStore[bucketName]; !ok { - return nil, fmt.Errorf("bucket not found") + addr := fmt.Sprintf("%s:%s", *host, *port) + + fmt.Printf("Starting mock GCS server on http://%s\n", addr) + if *manifest != "" { + m, err := readManifest(*manifest) + if err != nil { + log.Fatalf("error reading manifest: %v", err) + } + if err := processManifest(m); err != nil { + log.Fatalf("error processing manifest: %v", err) + } + } else { + fmt.Println("Store is empty. Create buckets and objects via API calls.") } - inMemoryStore[bucketName][objectName] = ObjectData{ - Data: data, - ContentType: contentType, + http.HandleFunc("/", handleRequests) + + if err := http.ListenAndServe(addr, nil); err != nil { + log.Fatalf("failed to start server: %v", err) } - log.Printf("created object '%s' in bucket '%s' with Content-Type '%s'", objectName, bucketName, contentType) +} - return &GCSObject{ - Kind: "storage#object", - Name: objectName, - Bucket: bucketName, - Size: strconv.Itoa(len(data)), - ContentType: contentType, - }, nil +func processManifest(manifest *Manifest) error { + for bucketName, bucket := range manifest.Buckets { + for _, file := range bucket.Files { + fmt.Printf("preloading data for bucket: %s | path: %s | content-type: %s...\n", + bucketName, file.Path, file.ContentType) + + if err := createBucket(bucketName); err != nil { + return fmt.Errorf("failed to create bucket '%s': %w", bucketName, err) + } + data, err := os.ReadFile(file.Path) + if err != nil { + return fmt.Errorf("failed to read bucket data file '%s': %w", file.Path, err) + } + pathParts := strings.Split(file.Path, "/") + if _, err := uploadObject(bucketName, pathParts[len(pathParts)-1], data, file.ContentType); err != nil { + return fmt.Errorf("failed to create object '%s' in bucket '%s': %w", file.Path, bucketName, err) + } + } + } + return nil } func readManifest(path string) (*Manifest, error) { @@ -101,13 +122,95 @@ func readManifest(path string) (*Manifest, error) { return &manifest, nil } -// healthHandler responds with a simple "OK" message. +func handleRequests(w http.ResponseWriter, r *http.Request) { + path := r.URL.Path + log.Printf("Received request: %s %s", r.Method, path) + + switch r.Method { + case "GET": + if strings.HasPrefix(path, "/health") { + healthHandler(w, r) + return + } + if strings.HasPrefix(path, "/storage/v1/b/") && strings.HasSuffix(path, "/o") { + handleListObjects(w, r) + return + } + handleGetObject(w, r) + case "POST": + if path == "/storage/v1/b" { + handleCreateBucket(w, r) + return + } + if strings.HasPrefix(path, "/upload/storage/v1/b/") { + handleUploadObject(w, r) + return + } + default: + http.NotFound(w, r) + } +} + func healthHandler(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) fmt.Fprint(w, "OK") } -// handleCreateBucket creates a new, empty bucket. +func handleListObjects(w http.ResponseWriter, r *http.Request) { + bucketName := strings.Split(strings.Trim(r.URL.Path, "/"), "/")[3] + + if bucket, ok := inMemoryStore[bucketName]; ok { + response := GCSListResponse{ + Kind: "storage#objects", + Items: []GCSObject{}, + } + for name, object := range bucket { + item := GCSObject{ + Kind: "storage#object", + Name: name, + Bucket: bucketName, + Size: strconv.Itoa(len(object.Data)), + ContentType: object.ContentType, + } + response.Items = append(response.Items, item) + } + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) + return + } + http.Error(w, "not found", http.StatusNotFound) +} + +func handleGetObject(w http.ResponseWriter, r *http.Request) { + var bucketName, objectName string + path := strings.Trim(r.URL.Path, "/") + parts := strings.Split(path, "/") + + if strings.HasPrefix(path, "storage/v1/b/") { + if len(parts) >= 6 { + bucketName, objectName = parts[3], parts[5] + } + } else { + if len(parts) >= 2 { + bucketName, objectName = parts[0], parts[1] + } + } + + if bucketName == "" || objectName == "" { + http.Error(w, "not found: invalid URL format", http.StatusNotFound) + return + } + + if bucket, ok := inMemoryStore[bucketName]; ok { + if object, ok := bucket[objectName]; ok { + w.Header().Set("Content-Type", object.ContentType) + w.Write(object.Data) + return + } + } + http.Error(w, "not found", http.StatusNotFound) +} + func handleCreateBucket(w http.ResponseWriter, r *http.Request) { var bucketInfo struct { Name string `json:"name"` @@ -128,7 +231,6 @@ func handleCreateBucket(w http.ResponseWriter, r *http.Request) { json.NewEncoder(w).Encode(bucketInfo) } -// handleUploadObject uploads a new file to a bucket. func handleUploadObject(w http.ResponseWriter, r *http.Request) { pathParts := strings.Split(strings.Trim(r.URL.Path, "/"), "/") if len(pathParts) < 5 { @@ -164,140 +266,32 @@ func handleUploadObject(w http.ResponseWriter, r *http.Request) { json.NewEncoder(w).Encode(response) } -// handleGetObject retrieves an object from a bucket. -func handleGetObject(w http.ResponseWriter, r *http.Request) { - var bucketName, objectName string - path := strings.Trim(r.URL.Path, "/") - parts := strings.Split(path, "/") - - if strings.HasPrefix(path, "storage/v1/b/") { - // api style: /storage/v1/b/{bucket}/o/{object} - if len(parts) >= 6 { - bucketName, objectName = parts[3], parts[5] - } - } else { - // path style: /{bucket}/{object} - if len(parts) >= 2 { - bucketName, objectName = parts[0], parts[1] - } - } - - if bucketName == "" || objectName == "" { - http.Error(w, "not found: invalid URL format", http.StatusNotFound) - return - } - - if bucket, ok := inMemoryStore[bucketName]; ok { - if object, ok := bucket[objectName]; ok { - w.Header().Set("Content-Type", object.ContentType) - w.Write(object.Data) - return - } - } - http.Error(w, "not found", http.StatusNotFound) -} - -// handleListObjects lists all objects in a bucket. -func handleListObjects(w http.ResponseWriter, r *http.Request) { - bucketName := strings.Split(strings.Trim(r.URL.Path, "/"), "/")[3] - - if bucket, ok := inMemoryStore[bucketName]; ok { - response := GCSListResponse{ - Kind: "storage#objects", - Items: []GCSObject{}, - } - for name, object := range bucket { - item := GCSObject{ - Kind: "storage#object", - Name: name, - Bucket: bucketName, - Size: strconv.Itoa(len(object.Data)), - ContentType: object.ContentType, - } - response.Items = append(response.Items, item) - } - w.Header().Set("Content-Type", "application/json") - json.NewEncoder(w).Encode(response) - return - } - http.Error(w, "not found", http.StatusNotFound) -} - -// processManifest processes the manifest to create buckets and upload files. -func processManifest(manifest *Manifest) error { - for bucketName, bucket := range manifest.Buckets { - for _, file := range bucket.Files { - fmt.Printf("preloading data for bucket: %s | path: %s | content-type: %s...\n", bucketName, file.Path, file.ContentType) - if err := createBucket(bucketName); err != nil { - return fmt.Errorf("failed to create bucket '%s': %w", bucketName, err) - } - data, err := os.ReadFile(file.Path) - if err != nil { - return fmt.Errorf("failed to read bucket data file '%s': %w", file.Path, err) - } - pathParts := strings.Split(file.Path, "/") - if _, err := uploadObject(bucketName, pathParts[len(pathParts)-1], data, file.ContentType); err != nil { - return fmt.Errorf("failed to create object '%s' in bucket '%s': %w", file.Path, bucketName, err) - } - } +func createBucket(bucketName string) error { + if _, exists := inMemoryStore[bucketName]; exists { + return fmt.Errorf("bucket already exists") } + inMemoryStore[bucketName] = make(map[string]ObjectData) + log.Printf("created bucket: %s", bucketName) return nil } -// handleRequests is the top-level router. -func handleRequests(w http.ResponseWriter, r *http.Request) { - path := r.URL.Path - log.Printf("Received request: %s %s", r.Method, path) - - switch r.Method { - case "GET": - if strings.HasPrefix(path, "/health") { - healthHandler(w, r) - return - } - if strings.HasPrefix(path, "/storage/v1/b/") && strings.HasSuffix(path, "/o") { - handleListObjects(w, r) - return - } - handleGetObject(w, r) - case "POST": - if path == "/storage/v1/b" { - handleCreateBucket(w, r) - return - } - if strings.HasPrefix(path, "/upload/storage/v1/b/") { - handleUploadObject(w, r) - return - } - default: - http.NotFound(w, r) +func uploadObject(bucketName, objectName string, data []byte, contentType string) (*GCSObject, error) { + if _, ok := inMemoryStore[bucketName]; !ok { + return nil, fmt.Errorf("bucket not found") } -} - -func main() { - host := flag.String("host", "0.0.0.0", "host to listen on") - port := flag.String("port", "4443", "port to listen on") - manifest := flag.String("manifest", "", "path to YAML manifest file for preloading buckets and objects") - flag.Parse() - - addr := fmt.Sprintf("%s:%s", *host, *port) - fmt.Printf("Starting mock GCS server on http://%s\n", addr) - if *manifest != "" { - m, err := readManifest(*manifest) - if err != nil { - log.Fatalf("error reading manifest: %v", err) - } - if err := processManifest(m); err != nil { - log.Fatalf("error processing manifest: %v", err) - } - } else { - fmt.Println("Store is empty. Create buckets and objects via API calls.") + inMemoryStore[bucketName][objectName] = ObjectData{ + Data: data, + ContentType: contentType, } + log.Printf("created object '%s' in bucket '%s' with Content-Type '%s'", + objectName, bucketName, contentType) - http.HandleFunc("/", handleRequests) - - if err := http.ListenAndServe(addr, nil); err != nil { - log.Fatalf("failed to start server: %v", err) - } + return &GCSObject{ + Kind: "storage#object", + Name: objectName, + Bucket: bucketName, + Size: strconv.Itoa(len(data)), + ContentType: contentType, + }, nil } diff --git a/packages/github/data_stream/audit/manifest.yml b/packages/github/data_stream/audit/manifest.yml index bc714d0271b..5eca98c27b6 100644 --- a/packages/github/data_stream/audit/manifest.yml +++ b/packages/github/data_stream/audit/manifest.yml @@ -98,7 +98,9 @@ streams: multi: false required: false show_user: false - description: "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. \nThis executes in the agent before the logs are parsed. \nSee [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.\n" + description: > + "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. \nThis executes in the agent before the logs are parsed. \nSee [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.\n" + - input: azure-eventhub description: Collect GitHub audit logs from Azure Event Hub title: GitHub Audit Logs from Azure Event Hub @@ -508,7 +510,9 @@ streams: - name: containers type: yaml title: Containers - description: "This attribute contains the details about a specific container like, name, number_of_workers, poll, poll_interval etc. The attribute 'name' is specific to a container as it describes the container name, while the fields number_of_workers, poll, poll_interval can exist both at the container level and at the global level. \nIf you have already defined the attributes globally, then you can only specify the container name in this yaml config. If you want to override any specific attribute for a container, then, you can define it here. Any attribute defined in the yaml will override the global definitions. \nPlease see the relevant [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-azure-blob-storage.html#attrib-containers) for further information.\n" + description: > + "This attribute contains the details about a specific container like, name, number_of_workers, poll, poll_interval etc. The attribute 'name' is specific to a container as it describes the container name, while the fields number_of_workers, poll, poll_interval can exist both at the container level and at the global level. \nIf you have already defined the attributes globally, then you can only specify the container name in this yaml config. If you want to override any specific attribute for a container, then, you can define it here. Any attribute defined in the yaml will override the global definitions. \nPlease see the relevant [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-azure-blob-storage.html#attrib-containers) for further information.\n" + required: true show_user: true default: | @@ -528,13 +532,17 @@ streams: show_user: false default: | # - regex: "event/" - description: "If the container will have events that correspond to files that this integration shouldn’t process, file_selectors can be used to limit the files that are downloaded. \nThis is a list of selectors which is made up of regex patters. The regex should match the container filepath. Regexes use [RE2 syntax](https://pkg.go.dev/regexp/syntax). \nFiles that don’t match one of the regexes will not be processed. \nThis process happens locally on the host hence it is an expensive operation. It is recommended to use this attribute only if there is a specific need to filter out files locally.\n" + description: > + "If the container will have events that correspond to files that this integration shouldn’t process, file_selectors can be used to limit the files that are downloaded. \nThis is a list of selectors which is made up of regex patters. The regex should match the container filepath. Regexes use [RE2 syntax](https://pkg.go.dev/regexp/syntax). \nFiles that don’t match one of the regexes will not be processed. \nThis process happens locally on the host hence it is an expensive operation. It is recommended to use this attribute only if there is a specific need to filter out files locally.\n" + - name: timestamp_epoch type: integer title: Timestamp Epoch multi: false required: false - description: "This attribute can be used to filter out files/blobs which have a timestamp older than the specified value. The value of this attribute should be in unix epoch (seconds) format. \nThis process happens locally on the host hence it is an expensive operation. It is recommended to use this attribute only if there is a specific need to filter out files locally.\n" + description: > + "This attribute can be used to filter out files/blobs which have a timestamp older than the specified value. The value of this attribute should be in unix epoch (seconds) format. \nThis process happens locally on the host hence it is an expensive operation. It is recommended to use this attribute only if there is a specific need to filter out files locally.\n" + show_user: false - name: expand_event_list_from_field type: text @@ -542,7 +550,9 @@ streams: multi: false required: false show_user: false - description: "If the file-set using this input expects to receive multiple messages bundled under a specific field or an array of objects then the config option for 'expand_event_list_from_field' can be specified. This setting will be able to split the messages under the group value into separate events. \nThis can be specified at the global level or at the container level. For more info please refer to the [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-azure-blob-storage.html#attrib-expand_event_list_from_field).\n" + description: > + "If the file-set using this input expects to receive multiple messages bundled under a specific field or an array of objects then the config option for 'expand_event_list_from_field' can be specified. This setting will be able to split the messages under the group value into separate events. \nThis can be specified at the global level or at the container level. For more info please refer to the [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-azure-blob-storage.html#attrib-expand_event_list_from_field).\n" + - name: preserve_original_event required: true show_user: true From 45acc86c3882e2fb0e05e61193e51f170a1e21c1 Mon Sep 17 00:00:00 2001 From: Shourie Ganguly Date: Thu, 18 Sep 2025 11:30:54 +0530 Subject: [PATCH 17/18] updated the go program as per suggestions --- .../_dev/deploy/docker/docker-compose.yml | 2 +- .../deploy/docker/gcs-mock-service/main.go | 188 ++++++++++-------- 2 files changed, 104 insertions(+), 86 deletions(-) diff --git a/packages/github/_dev/deploy/docker/docker-compose.yml b/packages/github/_dev/deploy/docker/docker-compose.yml index 27b279124d8..0fc10891651 100644 --- a/packages/github/_dev/deploy/docker/docker-compose.yml +++ b/packages/github/_dev/deploy/docker/docker-compose.yml @@ -41,7 +41,7 @@ services: volumes: - ./gcs-mock-service:/app - ./files/manifest.yml:/files/manifest.yml:ro - - ./sample_logs/:/data + - ./sample_logs/:/data:ro ports: - "4443/tcp" healthcheck: diff --git a/packages/github/_dev/deploy/docker/gcs-mock-service/main.go b/packages/github/_dev/deploy/docker/gcs-mock-service/main.go index b853ce44359..735d585fe46 100644 --- a/packages/github/_dev/deploy/docker/gcs-mock-service/main.go +++ b/packages/github/_dev/deploy/docker/gcs-mock-service/main.go @@ -18,46 +18,6 @@ import ( "gopkg.in/yaml.v3" ) -// ObjectData stores the raw data and its content type. -type ObjectData struct { - Data []byte - ContentType string -} - -// The in-memory store to hold ObjectData structs. -var inMemoryStore = make(map[string]map[string]ObjectData) - -// GCSListResponse mimics the structure of a real GCS object list response. -type GCSListResponse struct { - Kind string `json:"kind"` - Items []GCSObject `json:"items"` -} - -// GCSObject mimics the structure of a GCS object resource with ContentType. -type GCSObject struct { - Kind string `json:"kind"` - Name string `json:"name"` - Bucket string `json:"bucket"` - Size string `json:"size"` - ContentType string `json:"contentType"` -} - -// Manifest represents the top-level structure of the YAML file -type Manifest struct { - Buckets map[string]Bucket `yaml:"buckets"` -} - -// Bucket represents each bucket and its files -type Bucket struct { - Files []File `yaml:"files"` -} - -// File represents each file entry inside a bucket -type File struct { - Path string `yaml:"path"` - ContentType string `yaml:"content-type"` -} - func main() { host := flag.String("host", "0.0.0.0", "host to listen on") port := flag.String("port", "4443", "port to listen on") @@ -79,13 +39,59 @@ func main() { fmt.Println("Store is empty. Create buckets and objects via API calls.") } - http.HandleFunc("/", handleRequests) + // setup HTTP handlers + mux := http.NewServeMux() + mux.HandleFunc("/health", healthHandler) + mux.HandleFunc("/storage/v1/b", handleCreateBucket) + mux.HandleFunc("/storage/v1/b/", func(w http.ResponseWriter, r *http.Request) { + log.Printf("Received request: %s %s", r.Method, r.URL.Path) + + if r.Method == "GET" { + if strings.HasSuffix(r.URL.Path, "/o") { + handleListObjects(w, r) + return + } + // route for getting an object (either path-style or API-style) + handleGetObject(w, r) + return + } + http.NotFound(w, r) + }) + mux.HandleFunc("/upload/storage/v1/b/", handleUploadObject) - if err := http.ListenAndServe(addr, nil); err != nil { + // fallback: path-style object access, e.g. /bucket/object + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + log.Printf("Received request: %s %s", r.Method, r.URL.Path) + + if r.Method == "GET" { + // route for getting an object (either path-style or API-style) + handleGetObject(w, r) + return + } + http.NotFound(w, r) + }) + + if err := http.ListenAndServe(addr, mux); err != nil { log.Fatalf("failed to start server: %v", err) } } +// readManifest reads and parses the YAML manifest file. +func readManifest(path string) (*Manifest, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("failed to read manifest: %w", err) + } + + var manifest Manifest + if err := yaml.Unmarshal(data, &manifest); err != nil { + return nil, fmt.Errorf("failed to parse manifest: %w", err) + } + + return &manifest, nil +} + +// processManifest creates buckets and uploads objects as specified in the manifest. func processManifest(manifest *Manifest) error { for bucketName, bucket := range manifest.Buckets { for _, file := range bucket.Files { @@ -108,54 +114,13 @@ func processManifest(manifest *Manifest) error { return nil } -func readManifest(path string) (*Manifest, error) { - data, err := os.ReadFile(path) - if err != nil { - return nil, fmt.Errorf("failed to read manifest: %w", err) - } - - var manifest Manifest - if err := yaml.Unmarshal(data, &manifest); err != nil { - return nil, fmt.Errorf("failed to parse manifest: %w", err) - } - - return &manifest, nil -} - -func handleRequests(w http.ResponseWriter, r *http.Request) { - path := r.URL.Path - log.Printf("Received request: %s %s", r.Method, path) - - switch r.Method { - case "GET": - if strings.HasPrefix(path, "/health") { - healthHandler(w, r) - return - } - if strings.HasPrefix(path, "/storage/v1/b/") && strings.HasSuffix(path, "/o") { - handleListObjects(w, r) - return - } - handleGetObject(w, r) - case "POST": - if path == "/storage/v1/b" { - handleCreateBucket(w, r) - return - } - if strings.HasPrefix(path, "/upload/storage/v1/b/") { - handleUploadObject(w, r) - return - } - default: - http.NotFound(w, r) - } -} - +// healthHandler responds with a simple "OK" message for health checks. func healthHandler(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) fmt.Fprint(w, "OK") } +// handleListObjects lists all objects in the specified bucket. func handleListObjects(w http.ResponseWriter, r *http.Request) { bucketName := strings.Split(strings.Trim(r.URL.Path, "/"), "/")[3] @@ -181,6 +146,7 @@ func handleListObjects(w http.ResponseWriter, r *http.Request) { http.Error(w, "not found", http.StatusNotFound) } +// handleGetObject retrieves a specific object from a bucket. func handleGetObject(w http.ResponseWriter, r *http.Request) { var bucketName, objectName string path := strings.Trim(r.URL.Path, "/") @@ -211,7 +177,13 @@ func handleGetObject(w http.ResponseWriter, r *http.Request) { http.Error(w, "not found", http.StatusNotFound) } +// handleCreateBucket creates a new bucket. func handleCreateBucket(w http.ResponseWriter, r *http.Request) { + if r.Method != "POST" { + http.NotFound(w, r) + return + } + var bucketInfo struct { Name string `json:"name"` } @@ -231,7 +203,13 @@ func handleCreateBucket(w http.ResponseWriter, r *http.Request) { json.NewEncoder(w).Encode(bucketInfo) } +// handleUploadObject uploads an object to a specified bucket. func handleUploadObject(w http.ResponseWriter, r *http.Request) { + if r.Method != "POST" { + http.NotFound(w, r) + return + } + pathParts := strings.Split(strings.Trim(r.URL.Path, "/"), "/") if len(pathParts) < 5 { http.Error(w, "invalid upload URL", http.StatusBadRequest) @@ -295,3 +273,43 @@ func uploadObject(bucketName, objectName string, data []byte, contentType string ContentType: contentType, }, nil } + +// The in-memory store to hold ObjectData structs. +var inMemoryStore = make(map[string]map[string]ObjectData) + +// ObjectData stores the raw data and its content type. +type ObjectData struct { + Data []byte + ContentType string +} + +// GCSListResponse mimics the structure of a real GCS object list response. +type GCSListResponse struct { + Kind string `json:"kind"` + Items []GCSObject `json:"items"` +} + +// GCSObject mimics the structure of a GCS object resource with ContentType. +type GCSObject struct { + Kind string `json:"kind"` + Name string `json:"name"` + Bucket string `json:"bucket"` + Size string `json:"size"` + ContentType string `json:"contentType"` +} + +// Manifest represents the top-level structure of the YAML file +type Manifest struct { + Buckets map[string]Bucket `yaml:"buckets"` +} + +// Bucket represents each bucket and its files +type Bucket struct { + Files []File `yaml:"files"` +} + +// File represents each file entry inside a bucket +type File struct { + Path string `yaml:"path"` + ContentType string `yaml:"content-type"` +} From 899a53edbe4a1c028541a76288c2ffe4b72c986e Mon Sep 17 00:00:00 2001 From: Shourie Ganguly Date: Thu, 18 Sep 2025 15:27:06 +0530 Subject: [PATCH 18/18] optimised the mockservice --- .../deploy/docker/gcs-mock-service/main.go | 88 ++++++++----------- 1 file changed, 35 insertions(+), 53 deletions(-) diff --git a/packages/github/_dev/deploy/docker/gcs-mock-service/main.go b/packages/github/_dev/deploy/docker/gcs-mock-service/main.go index 735d585fe46..391e75c7efe 100644 --- a/packages/github/_dev/deploy/docker/gcs-mock-service/main.go +++ b/packages/github/_dev/deploy/docker/gcs-mock-service/main.go @@ -41,41 +41,33 @@ func main() { // setup HTTP handlers mux := http.NewServeMux() + // health check mux.HandleFunc("/health", healthHandler) - mux.HandleFunc("/storage/v1/b", handleCreateBucket) - mux.HandleFunc("/storage/v1/b/", func(w http.ResponseWriter, r *http.Request) { - log.Printf("Received request: %s %s", r.Method, r.URL.Path) - - if r.Method == "GET" { - if strings.HasSuffix(r.URL.Path, "/o") { - handleListObjects(w, r) - return - } - // route for getting an object (either path-style or API-style) - handleGetObject(w, r) - return - } - http.NotFound(w, r) - }) - mux.HandleFunc("/upload/storage/v1/b/", handleUploadObject) - - // fallback: path-style object access, e.g. /bucket/object - mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { - log.Printf("Received request: %s %s", r.Method, r.URL.Path) - - if r.Method == "GET" { - // route for getting an object (either path-style or API-style) - handleGetObject(w, r) - return - } - http.NotFound(w, r) - }) - - if err := http.ListenAndServe(addr, mux); err != nil { + // standard gcs api calls + mux.HandleFunc("GET /storage/v1/b/{bucket}/o", handleListObjects) + mux.HandleFunc("GET /storage/v1/b/{bucket}/o/{object...}", handleGetObject) + mux.HandleFunc("POST /storage/v1/b", handleCreateBucket) + mux.HandleFunc("POST /upload/storage/v1/b/{bucket}/o", handleUploadObject) + mux.HandleFunc("POST /upload/storage/v1/b/{bucket}/o/{object...}", handleUploadObject) + // direct path-style gcs sdk calls + mux.HandleFunc("GET /{bucket}/o/{object...}", handleGetObject) + mux.HandleFunc("GET /{bucket}/{object...}", handleGetObject) + // debug: log all requests + loggedMux := loggingMiddleware(mux) + + if err := http.ListenAndServe(addr, loggedMux); err != nil { log.Fatalf("failed to start server: %v", err) } } +// loggingMiddleware logs incoming HTTP requests. +func loggingMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Printf("%s %s\n", r.Method, r.URL.Path) + next.ServeHTTP(w, r) + }) +} + // readManifest reads and parses the YAML manifest file. func readManifest(path string) (*Manifest, error) { data, err := os.ReadFile(path) @@ -122,7 +114,7 @@ func healthHandler(w http.ResponseWriter, r *http.Request) { // handleListObjects lists all objects in the specified bucket. func handleListObjects(w http.ResponseWriter, r *http.Request) { - bucketName := strings.Split(strings.Trim(r.URL.Path, "/"), "/")[3] + bucketName := r.PathValue("bucket") if bucket, ok := inMemoryStore[bucketName]; ok { response := GCSListResponse{ @@ -148,19 +140,8 @@ func handleListObjects(w http.ResponseWriter, r *http.Request) { // handleGetObject retrieves a specific object from a bucket. func handleGetObject(w http.ResponseWriter, r *http.Request) { - var bucketName, objectName string - path := strings.Trim(r.URL.Path, "/") - parts := strings.Split(path, "/") - - if strings.HasPrefix(path, "storage/v1/b/") { - if len(parts) >= 6 { - bucketName, objectName = parts[3], parts[5] - } - } else { - if len(parts) >= 2 { - bucketName, objectName = parts[0], parts[1] - } - } + bucketName := r.PathValue("bucket") + objectName := r.PathValue("object") if bucketName == "" || objectName == "" { http.Error(w, "not found: invalid URL format", http.StatusNotFound) @@ -179,7 +160,7 @@ func handleGetObject(w http.ResponseWriter, r *http.Request) { // handleCreateBucket creates a new bucket. func handleCreateBucket(w http.ResponseWriter, r *http.Request) { - if r.Method != "POST" { + if r.Method != http.MethodPost { http.NotFound(w, r) return } @@ -195,30 +176,31 @@ func handleCreateBucket(w http.ResponseWriter, r *http.Request) { http.Error(w, "bucket name is required", http.StatusBadRequest) return } + if err := createBucket(bucketInfo.Name); err != nil { http.Error(w, err.Error(), http.StatusConflict) return } + w.WriteHeader(http.StatusOK) json.NewEncoder(w).Encode(bucketInfo) } // handleUploadObject uploads an object to a specified bucket. func handleUploadObject(w http.ResponseWriter, r *http.Request) { - if r.Method != "POST" { + if r.Method != http.MethodPost { http.NotFound(w, r) return } - pathParts := strings.Split(strings.Trim(r.URL.Path, "/"), "/") - if len(pathParts) < 5 { - http.Error(w, "invalid upload URL", http.StatusBadRequest) - return - } - bucketName := pathParts[4] + bucketName := r.PathValue("bucket") objectName := r.URL.Query().Get("name") if objectName == "" { - http.Error(w, "missing 'name' query parameter", http.StatusBadRequest) + objectName = r.PathValue("object") + } + + if bucketName == "" || objectName == "" { + http.Error(w, "missing bucket or object name", http.StatusBadRequest) return }