From 088a6c732645b8ffcde7c051accfcdd58db11e76 Mon Sep 17 00:00:00 2001 From: stefans-elastic Date: Wed, 10 Sep 2025 16:28:09 +0300 Subject: [PATCH 1/3] [vsphere] add extra grok pattern to cover more log formats --- packages/vsphere/changelog.yml | 5 + .../_dev/test/pipeline/test-format-common.log | 5 +- .../test-format-common.log-expected.json | 109 +++++++++++++++++- .../elasticsearch/ingest_pipeline/default.yml | 3 +- packages/vsphere/manifest.yml | 2 +- 5 files changed, 117 insertions(+), 7 deletions(-) diff --git a/packages/vsphere/changelog.yml b/packages/vsphere/changelog.yml index 7fbd360cb31..10e29fca628 100644 --- a/packages/vsphere/changelog.yml +++ b/packages/vsphere/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.22.0" + changes: + - description: Add extra grok pattern to cover more log formats + type: enhancement + link: https://github.com/elastic/integrations/pull/1 - version: "1.21.0" changes: - description: Improve documentation diff --git a/packages/vsphere/data_stream/log/_dev/test/pipeline/test-format-common.log b/packages/vsphere/data_stream/log/_dev/test/pipeline/test-format-common.log index e371a0235d8..4be48df99c6 100644 --- a/packages/vsphere/data_stream/log/_dev/test/pipeline/test-format-common.log +++ b/packages/vsphere/data_stream/log/_dev/test/pipeline/test-format-common.log @@ -151,4 +151,7 @@ <14>1 2022-12-29T01:33:33.284655+00:00 vspherehost01 vpxd 6225 - - Event [162431924] [1-1] [2022-12-29T01:33:33.284478Z] [vim.event.EventEx] [info] [philipp] [] [162431924] [Failed login philipp from 192.168.11.1 at 12/29/2022 01:33:33 GMT in SSO] <14>1 2021-09-06T14:40:13.289354+00:00 vcenter vpxd 58650 - - Event [575793] [1-1] [2021-09-06T14:40:13.288346Z] [vim.event.UserLogoutSessionEvent] [info] [VSPHERE.LOCAL\Administrator] [] [575793] [User VSPHERE.LOCAL\Administrator@10.100.10.190 logged out (login time: Monday, 06 September, 2021 02:40:13 PM, number of API invocations: 75,133, user agent: Go-http-client/1.1)] <166>1 2024-09-18T21:30:05.155Z esxihost01 Hostd: info vsansystem[21254123] [vSAN@1234 sub=AccessChecker opId=011a11e2-7123] Shared secret from 192.168.0.1 logged in as VMware-client/6.5.0 -<166>1 2024-09-18T21:30:05.155Z esxihost01 Hostd: info vsansystem[21254123] [vSAN@1234 sub=AccessChecker opId=011a11e2-7123] SSL thumbprint logged in as VMware-client/6.5.0 +<166>1 2024-09-18T21:30:05.155Z esxihost01 Hostd: info vsansystem[21254123] [vSAN@1234 sub=AccessChecker opId=011a11e2-7123] SSL thumbprint logged in as VMware-client/6.5.0 +<110>1 2025-09-10T01:01:25.113Z PC-ESXI-VSAN-P01 envoy 21004234 - [proxy.disconnect@2345 key2=\"\\\"CP\\\"\" subject=\"\" ip=\"127.0.0.1\" priority=\"info\" vmw_vcenter=\"prod-vc02.sphere.com\" vmw_vcenter_id=\"550e8400-e29b-41d4-a716-446655440000\" vmw_vr_ops_id=\"b5cb8b7e-f1d8-4191-a5fe-391af4d592c3\" result=\"success\" vmw_cluster=\"PROD-VM01\" vmw_datacenter=\"TestCenter\" vmw_object_id=\"host-112233\" port=\"45296\" facility=\"13\" object=\"proxy\"] +<134>1 2025-09-10T15:43:11.026Z prod-vc01 vpxd-main - - [Originator@6884 key2=\"\\\"CP\\\"\" vmw_cluster=\"PROD-P01\" vmw_datacenter=\"TestCenter\" vmw_object_id=\"vm-112233\" vmw_host=\"esxi-p01.sphere.com\" priority=\"info\" vmw_vcenter=\"prod-vc01.sphere.com\" vmw_vcenter_id=\"550e8400-e29b-41d4-a716-446655440000\" facility=\"local1\" vmw_vr_ops_id=\"b5cb8b7e-f1d8-4191-a5fe-391af4d592c3\"] 2025-09-10T03:34:13.023+08:00 info vpxd[07317] [Originator@6875 sub=vpxLri opID=6598c432] [VpxLRO] -- FINISH lri-111222333 +<166>1 2025-09-10T12:15:33.834Z PC-ESXI-HCI-P01.sphere.com envoy-access 2188123 [Originator@6534 key2=\"\\\"CP\\\"\" vmw_cluster=\"PROD-VM01\" vmw_datacenter=\"TestCenter\" vmw_object_id=\"host-007\" priority=\"info\" vmw_vcenter=\"prod-vc02.sphere.com\" vmw_vcenter_id=\"550e8400-e29b-41d4-a716-446655440000\" facility=\"local1\" vmw_vr_ops_id=\"b5cb8b7e-f1d8-4191-a5fe-391af4d592c3\"] POST /sdk 200 via_upstream - 452 256 gzip 0 0 0 127.0.0.1:60792 HTTP/1.1 TLSv1.2 127.0.0.1:443 127.0.0.1:57833 HTTP/1.1 - 127.0.0.1:8307 - \"QueryNetworkHint\" diff --git a/packages/vsphere/data_stream/log/_dev/test/pipeline/test-format-common.log-expected.json b/packages/vsphere/data_stream/log/_dev/test/pipeline/test-format-common.log-expected.json index dc3fa8ce0e4..594c97e350f 100644 --- a/packages/vsphere/data_stream/log/_dev/test/pipeline/test-format-common.log-expected.json +++ b/packages/vsphere/data_stream/log/_dev/test/pipeline/test-format-common.log-expected.json @@ -5932,7 +5932,7 @@ "authentication" ], "kind": "event", - "original": "<166>1 2024-09-18T21:30:05.155Z esxihost01 Hostd: info vsansystem[21254123] [vSAN@1234 sub=AccessChecker opId=011a11e2-7123] SSL thumbprint logged in as VMware-client/6.5.0 ", + "original": "<166>1 2024-09-18T21:30:05.155Z esxihost01 Hostd: info vsansystem[21254123] [vSAN@1234 sub=AccessChecker opId=011a11e2-7123] SSL thumbprint logged in as VMware-client/6.5.0", "outcome": "success", "type": [ "info" @@ -5954,7 +5954,7 @@ } } }, - "message": "info vsansystem[21254123] [vSAN@1234 sub=AccessChecker opId=011a11e2-7123] SSL thumbprint logged in as VMware-client/6.5.0 ", + "message": "info vsansystem[21254123] [vSAN@1234 sub=AccessChecker opId=011a11e2-7123] SSL thumbprint logged in as VMware-client/6.5.0", "process": { "name": "Hostd" }, @@ -5969,8 +5969,109 @@ "name": "Other" }, "name": "Other", - "original": "VMware-client/6.5.0 " + "original": "VMware-client/6.5.0" } - } + }, + { + "@timestamp": "2025-09-10T01:01:25.113Z", + "ecs": { + "version": "8.11.0" + }, + "event": { + "kind": "event", + "original": "<110>1 2025-09-10T01:01:25.113Z PC-ESXI-VSAN-P01 envoy 21004234 - [proxy.disconnect@2345 key2=\\\"\\\\\\\"CP\\\\\\\"\\\" subject=\\\"\\\" ip=\\\"127.0.0.1\\\" priority=\\\"info\\\" vmw_vcenter=\\\"prod-vc02.sphere.com\\\" vmw_vcenter_id=\\\"550e8400-e29b-41d4-a716-446655440000\\\" vmw_vr_ops_id=\\\"b5cb8b7e-f1d8-4191-a5fe-391af4d592c3\\\" result=\\\"success\\\" vmw_cluster=\\\"PROD-VM01\\\" vmw_datacenter=\\\"TestCenter\\\" vmw_object_id=\\\"host-112233\\\" port=\\\"45296\\\" facility=\\\"13\\\" object=\\\"proxy\\\"]" + }, + "host": { + "name": "PC-ESXI-VSAN-P01" + }, + "log": { + "syslog": { + "facility": { + "code": 13, + "name": "Log audit" + }, + "priority": 110, + "severity": { + "code": 6, + "name": "Informational" + } + } + }, + "message": "[proxy.disconnect@2345 key2=\\\"\\\\\\\"CP\\\\\\\"\\\" subject=\\\"\\\" ip=\\\"127.0.0.1\\\" priority=\\\"info\\\" vmw_vcenter=\\\"prod-vc02.sphere.com\\\" vmw_vcenter_id=\\\"550e8400-e29b-41d4-a716-446655440000\\\" vmw_vr_ops_id=\\\"b5cb8b7e-f1d8-4191-a5fe-391af4d592c3\\\" result=\\\"success\\\" vmw_cluster=\\\"PROD-VM01\\\" vmw_datacenter=\\\"TestCenter\\\" vmw_object_id=\\\"host-112233\\\" port=\\\"45296\\\" facility=\\\"13\\\" object=\\\"proxy\\\"]", + "process": { + "name": "envoy", + "pid": 21004234 + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2025-09-10T15:43:11.026Z", + "ecs": { + "version": "8.11.0" + }, + "event": { + "kind": "event", + "original": "<134>1 2025-09-10T15:43:11.026Z prod-vc01 vpxd-main - - [Originator@6884 key2=\\\"\\\\\\\"CP\\\\\\\"\\\" vmw_cluster=\\\"PROD-P01\\\" vmw_datacenter=\\\"TestCenter\\\" vmw_object_id=\\\"vm-112233\\\" vmw_host=\\\"esxi-p01.sphere.com\\\" priority=\\\"info\\\" vmw_vcenter=\\\"prod-vc01.sphere.com\\\" vmw_vcenter_id=\\\"550e8400-e29b-41d4-a716-446655440000\\\" facility=\\\"local1\\\" vmw_vr_ops_id=\\\"b5cb8b7e-f1d8-4191-a5fe-391af4d592c3\\\"] 2025-09-10T03:34:13.023+08:00 info vpxd[07317] [Originator@6875 sub=vpxLri opID=6598c432] [VpxLRO] -- FINISH lri-111222333" + }, + "host": { + "name": "prod-vc01" + }, + "log": { + "syslog": { + "facility": { + "code": 16, + "name": "Local 0" + }, + "priority": 134, + "severity": { + "code": 6, + "name": "Informational" + } + } + }, + "message": "[Originator@6884 key2=\\\"\\\\\\\"CP\\\\\\\"\\\" vmw_cluster=\\\"PROD-P01\\\" vmw_datacenter=\\\"TestCenter\\\" vmw_object_id=\\\"vm-112233\\\" vmw_host=\\\"esxi-p01.sphere.com\\\" priority=\\\"info\\\" vmw_vcenter=\\\"prod-vc01.sphere.com\\\" vmw_vcenter_id=\\\"550e8400-e29b-41d4-a716-446655440000\\\" facility=\\\"local1\\\" vmw_vr_ops_id=\\\"b5cb8b7e-f1d8-4191-a5fe-391af4d592c3\\\"] 2025-09-10T03:34:13.023+08:00 info vpxd[07317] [Originator@6875 sub=vpxLri opID=6598c432] [VpxLRO] -- FINISH lri-111222333", + "process": { + "name": "vpxd-main" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2025-09-10T12:15:33.834Z", + "ecs": { + "version": "8.11.0" + }, + "event": { + "kind": "event", + "original": "<166>1 2025-09-10T12:15:33.834Z PC-ESXI-HCI-P01.sphere.com envoy-access 2188123 [Originator@6534 key2=\\\"\\\\\\\"CP\\\\\\\"\\\" vmw_cluster=\\\"PROD-VM01\\\" vmw_datacenter=\\\"TestCenter\\\" vmw_object_id=\\\"host-007\\\" priority=\\\"info\\\" vmw_vcenter=\\\"prod-vc02.sphere.com\\\" vmw_vcenter_id=\\\"550e8400-e29b-41d4-a716-446655440000\\\" facility=\\\"local1\\\" vmw_vr_ops_id=\\\"b5cb8b7e-f1d8-4191-a5fe-391af4d592c3\\\"] POST /sdk 200 via_upstream - 452 256 gzip 0 0 0 127.0.0.1:60792 HTTP/1.1 TLSv1.2 127.0.0.1:443 127.0.0.1:57833 HTTP/1.1 - 127.0.0.1:8307 - \\\"QueryNetworkHint\\\"" + }, + "host": { + "name": "PC-ESXI-HCI-P01.sphere.com" + }, + "log": { + "syslog": { + "facility": { + "code": 20, + "name": "Local 4" + }, + "priority": 166, + "severity": { + "code": 6, + "name": "Informational" + } + } + }, + "message": "[Originator@6534 key2=\\\"\\\\\\\"CP\\\\\\\"\\\" vmw_cluster=\\\"PROD-VM01\\\" vmw_datacenter=\\\"TestCenter\\\" vmw_object_id=\\\"host-007\\\" priority=\\\"info\\\" vmw_vcenter=\\\"prod-vc02.sphere.com\\\" vmw_vcenter_id=\\\"550e8400-e29b-41d4-a716-446655440000\\\" facility=\\\"local1\\\" vmw_vr_ops_id=\\\"b5cb8b7e-f1d8-4191-a5fe-391af4d592c3\\\"] POST /sdk 200 via_upstream - 452 256 gzip 0 0 0 127.0.0.1:60792 HTTP/1.1 TLSv1.2 127.0.0.1:443 127.0.0.1:57833 HTTP/1.1 - 127.0.0.1:8307 - \\\"QueryNetworkHint\\\"", + "process": { + "name": "envoy-access", + "pid": 2188123 + }, + "tags": [ + "preserve_original_event" + ] + } ] } \ No newline at end of file diff --git a/packages/vsphere/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/vsphere/data_stream/log/elasticsearch/ingest_pipeline/default.yml index 4007f39d190..43fcf605cd8 100644 --- a/packages/vsphere/data_stream/log/elasticsearch/ingest_pipeline/default.yml +++ b/packages/vsphere/data_stream/log/elasticsearch/ingest_pipeline/default.yml @@ -16,6 +16,7 @@ processors: (%{POSINT:process.pid:long}|-) - -%{SPACE}%{GREEDYDATA:message}" - "^(%{ECS_SYSLOG_PRIORITY})?%{TIMESTAMP_ISO8601:_tmp.timestamp}%{SPACE}%{HOST}%{SPACE}%{DATA:process.name}(?:\\[%{POSINT:process.pid:long}\\])?\\: %{GREEDYDATA:message}" + - "<%{NONNEGINT:log.syslog.priority:long}>%{INT} %{TIMESTAMP_ISO8601:_tmp.timestamp}%{SPACE}%{HOST}%{SPACE}%{NOTSPACE:process.name}%{SPACE}(%{POSINT:process.pid:long}|-)( -)?%{SPACE}%{GREEDYDATA:message}" - "^ \\(%{TIMESTAMP_ISO8601:_tmp.timestamp} %{GREEDYDATA:message}\\)%{GREEDYDATA:_tmp.drop}" pattern_definitions: ECS_SYSLOG_PRIORITY: "<%{NONNEGINT:log.syslog.priority:long}>(\\d )?" @@ -134,4 +135,4 @@ on_failure: ignore_failure: true - append: field: error.message - value: "{{ _ingest.on_failure_message }}" + value: "{{ _ingest.on_failure_message }}" \ No newline at end of file diff --git a/packages/vsphere/manifest.yml b/packages/vsphere/manifest.yml index da64362c1d4..9940c89fd32 100644 --- a/packages/vsphere/manifest.yml +++ b/packages/vsphere/manifest.yml @@ -1,7 +1,7 @@ title: VMware vSphere format_version: "3.0.2" name: vsphere -version: "1.21.0" +version: "1.22.0" description: This Elastic integration collects metrics and logs from vSphere/vCenter servers type: integration categories: From 5dc858385872534a055360f4fa14d983fe477070 Mon Sep 17 00:00:00 2001 From: stefans-elastic Date: Wed, 10 Sep 2025 17:00:00 +0300 Subject: [PATCH 2/3] pattern cleanup --- .../data_stream/log/elasticsearch/ingest_pipeline/default.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/vsphere/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/vsphere/data_stream/log/elasticsearch/ingest_pipeline/default.yml index 43fcf605cd8..e56b1e0bf79 100644 --- a/packages/vsphere/data_stream/log/elasticsearch/ingest_pipeline/default.yml +++ b/packages/vsphere/data_stream/log/elasticsearch/ingest_pipeline/default.yml @@ -16,7 +16,7 @@ processors: (%{POSINT:process.pid:long}|-) - -%{SPACE}%{GREEDYDATA:message}" - "^(%{ECS_SYSLOG_PRIORITY})?%{TIMESTAMP_ISO8601:_tmp.timestamp}%{SPACE}%{HOST}%{SPACE}%{DATA:process.name}(?:\\[%{POSINT:process.pid:long}\\])?\\: %{GREEDYDATA:message}" - - "<%{NONNEGINT:log.syslog.priority:long}>%{INT} %{TIMESTAMP_ISO8601:_tmp.timestamp}%{SPACE}%{HOST}%{SPACE}%{NOTSPACE:process.name}%{SPACE}(%{POSINT:process.pid:long}|-)( -)?%{SPACE}%{GREEDYDATA:message}" + - "^(%{ECS_SYSLOG_PRIORITY})?%{TIMESTAMP_ISO8601:_tmp.timestamp}%{SPACE}%{HOST}%{SPACE}%{NOTSPACE:process.name}%{SPACE}(%{POSINT:process.pid:long}|-)( -)?%{SPACE}%{GREEDYDATA:message}" - "^ \\(%{TIMESTAMP_ISO8601:_tmp.timestamp} %{GREEDYDATA:message}\\)%{GREEDYDATA:_tmp.drop}" pattern_definitions: ECS_SYSLOG_PRIORITY: "<%{NONNEGINT:log.syslog.priority:long}>(\\d )?" From 3335501c2876e1f73d1b2df74e9996a1ee20757d Mon Sep 17 00:00:00 2001 From: Mykola Kmet Date: Thu, 11 Sep 2025 11:35:45 +0300 Subject: [PATCH 3/3] Update packages/vsphere/changelog.yml --- packages/vsphere/changelog.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/vsphere/changelog.yml b/packages/vsphere/changelog.yml index 10e29fca628..5fca6170f03 100644 --- a/packages/vsphere/changelog.yml +++ b/packages/vsphere/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Add extra grok pattern to cover more log formats type: enhancement - link: https://github.com/elastic/integrations/pull/1 + link: https://github.com/elastic/integrations/pull/15274 - version: "1.21.0" changes: - description: Improve documentation