From d18380c2110368085ca9204786d75d5ca73f483d Mon Sep 17 00:00:00 2001 From: Mykola Kmet Date: Thu, 11 Sep 2025 14:48:00 +0300 Subject: [PATCH 1/8] small restructure of readme --- packages/airflow/_dev/build/docs/README.md | 12 ++++++------ packages/airflow/docs/README.md | 12 ++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/packages/airflow/_dev/build/docs/README.md b/packages/airflow/_dev/build/docs/README.md index 3b0487e1689..59c739e8057 100644 --- a/packages/airflow/_dev/build/docs/README.md +++ b/packages/airflow/_dev/build/docs/README.md @@ -10,7 +10,11 @@ Use the Airflow integration to: - Create informative visualizations to track usage trends, measure key metrics, and derive actionable business insights. - Monitor your workflows' performance and status in real-time. -## Data streams +## Compatibility + +The Airflow module is tested with Airflow `2.4.0`. It should work with versions `2.0.0` and later. + +## What data does this integration collect? The Airflow integration gathers metric data. @@ -23,17 +27,13 @@ Data stream: Note: - Users can monitor and view metrics within the ingested documents for Airflow in the `metrics-*` index pattern from `Discover`. -## Compatibility - -The Airflow module is tested with Airflow `2.4.0`. It should work with versions `2.0.0` and later. - ## What do I need to use this integration? Users require Elasticsearch to store and search user data, and Kibana to visualize and manage it. They can utilize the hosted Elasticsearch Service on Elastic Cloud, which is recommended, or self-manage the Elastic Stack on their own hardware. To ingest data from Airflow, users must have [StatsD](https://github.com/statsd/statsd) to receive the same. -## Setup +## How do I deploy this integration? For step-by-step instructions on how to set up an integration, see the [Getting started](https://www.elastic.co/guide/en/starting-with-the-elasticsearch-platform-and-its-solutions/current/getting-started-observability.html) guide. diff --git a/packages/airflow/docs/README.md b/packages/airflow/docs/README.md index b96ec7e91dc..5b38eee59b4 100644 --- a/packages/airflow/docs/README.md +++ b/packages/airflow/docs/README.md @@ -10,7 +10,11 @@ Use the Airflow integration to: - Create informative visualizations to track usage trends, measure key metrics, and derive actionable business insights. - Monitor your workflows' performance and status in real-time. -## Data streams +## Compatibility + +The Airflow module is tested with Airflow `2.4.0`. It should work with versions `2.0.0` and later. + +## What data does this integration collect? The Airflow integration gathers metric data. @@ -23,17 +27,13 @@ Data stream: Note: - Users can monitor and view metrics within the ingested documents for Airflow in the `metrics-*` index pattern from `Discover`. -## Compatibility - -The Airflow module is tested with Airflow `2.4.0`. It should work with versions `2.0.0` and later. - ## What do I need to use this integration? Users require Elasticsearch to store and search user data, and Kibana to visualize and manage it. They can utilize the hosted Elasticsearch Service on Elastic Cloud, which is recommended, or self-manage the Elastic Stack on their own hardware. To ingest data from Airflow, users must have [StatsD](https://github.com/statsd/statsd) to receive the same. -## Setup +## How do I deploy this integration? For step-by-step instructions on how to set up an integration, see the [Getting started](https://www.elastic.co/guide/en/starting-with-the-elasticsearch-platform-and-its-solutions/current/getting-started-observability.html) guide. From 231da488ecd9ef5738b08eb4701aacd8b0848027 Mon Sep 17 00:00:00 2001 From: Mykola Kmet Date: Thu, 11 Sep 2025 14:48:17 +0300 Subject: [PATCH 2/8] remove ambiguous sentence --- packages/airflow/_dev/build/docs/README.md | 2 -- packages/airflow/docs/README.md | 2 -- 2 files changed, 4 deletions(-) diff --git a/packages/airflow/_dev/build/docs/README.md b/packages/airflow/_dev/build/docs/README.md index 59c739e8057..8674d838a7e 100644 --- a/packages/airflow/_dev/build/docs/README.md +++ b/packages/airflow/_dev/build/docs/README.md @@ -31,8 +31,6 @@ Note: Users require Elasticsearch to store and search user data, and Kibana to visualize and manage it. They can utilize the hosted Elasticsearch Service on Elastic Cloud, which is recommended, or self-manage the Elastic Stack on their own hardware. -To ingest data from Airflow, users must have [StatsD](https://github.com/statsd/statsd) to receive the same. - ## How do I deploy this integration? For step-by-step instructions on how to set up an integration, see the [Getting started](https://www.elastic.co/guide/en/starting-with-the-elasticsearch-platform-and-its-solutions/current/getting-started-observability.html) guide. diff --git a/packages/airflow/docs/README.md b/packages/airflow/docs/README.md index 5b38eee59b4..271f5520a38 100644 --- a/packages/airflow/docs/README.md +++ b/packages/airflow/docs/README.md @@ -31,8 +31,6 @@ Note: Users require Elasticsearch to store and search user data, and Kibana to visualize and manage it. They can utilize the hosted Elasticsearch Service on Elastic Cloud, which is recommended, or self-manage the Elastic Stack on their own hardware. -To ingest data from Airflow, users must have [StatsD](https://github.com/statsd/statsd) to receive the same. - ## How do I deploy this integration? For step-by-step instructions on how to set up an integration, see the [Getting started](https://www.elastic.co/guide/en/starting-with-the-elasticsearch-platform-and-its-solutions/current/getting-started-observability.html) guide. From 53b67e5c27c05cb3c75d02cff954fd373897fe90 Mon Sep 17 00:00:00 2001 From: Mykola Kmet Date: Thu, 11 Sep 2025 14:57:09 +0300 Subject: [PATCH 3/8] add version 1.0.0 --- packages/airflow/changelog.yml | 5 +++++ packages/airflow/manifest.yml | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/airflow/changelog.yml b/packages/airflow/changelog.yml index 040f0788e8b..33622a6cbe3 100644 --- a/packages/airflow/changelog.yml +++ b/packages/airflow/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.0.0" + changes: + - description: Make Airflow package GA. + type: enhancement + link: https://github.com/elastic/integrations/pull/15287 - version: "0.11.0" changes: - description: Improve documentation diff --git a/packages/airflow/manifest.yml b/packages/airflow/manifest.yml index acaddd3b0ca..52d4323d844 100644 --- a/packages/airflow/manifest.yml +++ b/packages/airflow/manifest.yml @@ -1,6 +1,6 @@ name: airflow title: Airflow -version: "0.11.0" +version: "1.0.0" description: Airflow Integration. type: integration format_version: "3.0.0" From 138ca10145bdb87262c44ad63060f87f98d10f17 Mon Sep 17 00:00:00 2001 From: Mykola Kmet Date: Thu, 11 Sep 2025 14:57:36 +0300 Subject: [PATCH 4/8] remove "experimental" --- packages/airflow/data_stream/statsd/manifest.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/airflow/data_stream/statsd/manifest.yml b/packages/airflow/data_stream/statsd/manifest.yml index 23f50d6375d..e3c46a9d7dd 100644 --- a/packages/airflow/data_stream/statsd/manifest.yml +++ b/packages/airflow/data_stream/statsd/manifest.yml @@ -1,6 +1,5 @@ type: metrics title: Airflow metrics -release: experimental streams: - input: statsd/metrics enabled: true From 48bf43c6f63c239d4da7117c846df2ef3b4aa100 Mon Sep 17 00:00:00 2001 From: Mykola Kmet Date: Thu, 11 Sep 2025 14:57:51 +0300 Subject: [PATCH 5/8] update ECS to 8.17.0 --- packages/airflow/_dev/build/build.yml | 2 +- .../statsd/elasticsearch/ingest_pipeline/default.yml | 2 +- packages/airflow/data_stream/statsd/sample_event.json | 2 +- packages/airflow/docs/README.md | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/airflow/_dev/build/build.yml b/packages/airflow/_dev/build/build.yml index 2bfcfc223b0..d8553567e9c 100644 --- a/packages/airflow/_dev/build/build.yml +++ b/packages/airflow/_dev/build/build.yml @@ -1,3 +1,3 @@ dependencies: ecs: - reference: "git@v8.11.0" + reference: "git@v8.17.0" diff --git a/packages/airflow/data_stream/statsd/elasticsearch/ingest_pipeline/default.yml b/packages/airflow/data_stream/statsd/elasticsearch/ingest_pipeline/default.yml index 6dc9f1a0b6d..c7543195e49 100644 --- a/packages/airflow/data_stream/statsd/elasticsearch/ingest_pipeline/default.yml +++ b/packages/airflow/data_stream/statsd/elasticsearch/ingest_pipeline/default.yml @@ -3,7 +3,7 @@ description: Pipeline for processing airflow data processors: - set: field: ecs.version - value: "8.11.0" + value: "8.17.0" - rename: field: statsd target_field: airflow diff --git a/packages/airflow/data_stream/statsd/sample_event.json b/packages/airflow/data_stream/statsd/sample_event.json index 6fce2ee6aeb..ded759d48dc 100644 --- a/packages/airflow/data_stream/statsd/sample_event.json +++ b/packages/airflow/data_stream/statsd/sample_event.json @@ -24,7 +24,7 @@ "type": "metrics" }, "ecs": { - "version": "8.11.0" + "version": "8.17.0" }, "elastic_agent": { "id": "97400795-188c-4140-a1ee-0002078c785d", diff --git a/packages/airflow/docs/README.md b/packages/airflow/docs/README.md index 271f5520a38..d00ea9c85ba 100644 --- a/packages/airflow/docs/README.md +++ b/packages/airflow/docs/README.md @@ -93,7 +93,7 @@ An example event for `statsd` looks as following: "type": "metrics" }, "ecs": { - "version": "8.11.0" + "version": "8.17.0" }, "elastic_agent": { "id": "97400795-188c-4140-a1ee-0002078c785d", From 7ec7cedb9bc9ddc15732f350aa9795b0c7007437 Mon Sep 17 00:00:00 2001 From: Mykola Kmet Date: Thu, 11 Sep 2025 16:20:05 +0300 Subject: [PATCH 6/8] update compatibility section --- packages/airflow/_dev/build/docs/README.md | 2 +- packages/airflow/docs/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/airflow/_dev/build/docs/README.md b/packages/airflow/_dev/build/docs/README.md index 8674d838a7e..74687b9e3ad 100644 --- a/packages/airflow/_dev/build/docs/README.md +++ b/packages/airflow/_dev/build/docs/README.md @@ -12,7 +12,7 @@ Use the Airflow integration to: ## Compatibility -The Airflow module is tested with Airflow `2.4.0`. It should work with versions `2.0.0` and later. +The Airflow package is tested with Airflow `2.4.0` and `3.0.6`. It should work with any `2.*` and `3.*` versions. ## What data does this integration collect? diff --git a/packages/airflow/docs/README.md b/packages/airflow/docs/README.md index d00ea9c85ba..46af5b5ac3e 100644 --- a/packages/airflow/docs/README.md +++ b/packages/airflow/docs/README.md @@ -12,7 +12,7 @@ Use the Airflow integration to: ## Compatibility -The Airflow module is tested with Airflow `2.4.0`. It should work with versions `2.0.0` and later. +The Airflow package is tested with Airflow `2.4.0` and `3.0.6`. It should work with any `2.*` and `3.*` versions. ## What data does this integration collect? From 4055571a76060de68a0866b4422198988b6a104d Mon Sep 17 00:00:00 2001 From: Mykola Kmet Date: Fri, 12 Sep 2025 15:48:12 +0300 Subject: [PATCH 7/8] move data_stream parameters from package manifest.yml to data_stream manifest.yml --- .../_dev/test/system/test-default-config.yml | 8 +++++--- .../airflow/data_stream/statsd/manifest.yml | 15 +++++++++++++++ packages/airflow/manifest.yml | 18 ------------------ 3 files changed, 20 insertions(+), 21 deletions(-) diff --git a/packages/airflow/data_stream/statsd/_dev/test/system/test-default-config.yml b/packages/airflow/data_stream/statsd/_dev/test/system/test-default-config.yml index ea129816200..92f930893fe 100644 --- a/packages/airflow/data_stream/statsd/_dev/test/system/test-default-config.yml +++ b/packages/airflow/data_stream/statsd/_dev/test/system/test-default-config.yml @@ -1,4 +1,6 @@ service_notify_signal: SIGHUP -vars: - host: 0.0.0.0 - port: 8125 +vars: ~ +data_stream: + vars: + host: 0.0.0.0 + port: 8125 \ No newline at end of file diff --git a/packages/airflow/data_stream/statsd/manifest.yml b/packages/airflow/data_stream/statsd/manifest.yml index e3c46a9d7dd..775b9b68f98 100644 --- a/packages/airflow/data_stream/statsd/manifest.yml +++ b/packages/airflow/data_stream/statsd/manifest.yml @@ -5,5 +5,20 @@ streams: enabled: true title: Airflow metrics description: Collect Airflow metrics + vars: + - name: host + type: text + title: Host + multi: false + required: true + show_user: true + default: localhost + - name: port + type: text + title: Port + multi: false + required: true + show_user: true + default: "8125" elasticsearch: index_mode: "time_series" diff --git a/packages/airflow/manifest.yml b/packages/airflow/manifest.yml index 52d4323d844..fd25fa71e3c 100644 --- a/packages/airflow/manifest.yml +++ b/packages/airflow/manifest.yml @@ -16,32 +16,14 @@ icons: title: Airflow logo size: 32x32 type: image/svg+xml -vars: - - name: host - type: text - title: Host - multi: false - required: true - show_user: true - default: localhost - - name: port - type: text - title: Port - multi: false - required: true - show_user: true - default: "8125" policy_templates: - name: server title: Airflow metrics description: Collect metrics from Airflow instances - data_streams: - - statsd inputs: - type: "statsd/metrics" title: "Collect Airflow metrics" description: "Collecting Airflow metrics" - input_group: metrics owner: github: elastic/obs-infraobs-integrations type: elastic From 3c1cac1a01aea9850b4f21b01e491cd157b877cd Mon Sep 17 00:00:00 2001 From: Mykola Kmet Date: Thu, 25 Sep 2025 14:50:44 +0300 Subject: [PATCH 8/8] use ttl 60s instead of default (30s) --- packages/airflow/data_stream/statsd/agent/stream/stream.yml.hbs | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/airflow/data_stream/statsd/agent/stream/stream.yml.hbs b/packages/airflow/data_stream/statsd/agent/stream/stream.yml.hbs index 4834628aaaf..c488b457c01 100644 --- a/packages/airflow/data_stream/statsd/agent/stream/stream.yml.hbs +++ b/packages/airflow/data_stream/statsd/agent/stream/stream.yml.hbs @@ -1,6 +1,7 @@ metricsets: ["server"] host: {{host}} port: {{port}} +ttl: 60s statsd.mappings: - metric: '_start' labels: