Skip to content

Commit cc88c07

Browse files
tsdk02hyperswitch-bot[bot]Abhitator216ivor11lsampras
authored
feat(analytics): FRM Analytics (#4880)
Co-authored-by: hyperswitch-bot[bot] <148525504+hyperswitch-bot[bot]@users.noreply.github.com> Co-authored-by: Abhitator216 <[email protected]> Co-authored-by: Abhishek Kanojia <[email protected]> Co-authored-by: ivor-juspay <[email protected]> Co-authored-by: Sampras Lopes <[email protected]>
1 parent 7a1651d commit cc88c07

36 files changed

+1629
-78
lines changed

‎Cargo.lock

Lines changed: 3 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎api-reference/openapi_spec.json

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7228,7 +7228,6 @@
72287228
},
72297229
"CaptureStatus": {
72307230
"type": "string",
7231-
"description": "The status of the capture",
72327231
"enum": [
72337232
"started",
72347233
"charged",

‎config/config.example.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -623,6 +623,7 @@ source = "logs" # The event sink to push events supports kafka or logs (stdout)
623623

624624
[events.kafka]
625625
brokers = [] # Kafka broker urls for bootstrapping the client
626+
fraud_check_analytics_topic = "topic" # Kafka topic to be used for FraudCheck events
626627
intent_analytics_topic = "topic" # Kafka topic to be used for PaymentIntent events
627628
attempt_analytics_topic = "topic" # Kafka topic to be used for PaymentAttempt events
628629
refund_analytics_topic = "topic" # Kafka topic to be used for Refund events

‎config/development.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -615,6 +615,7 @@ source = "logs"
615615

616616
[events.kafka]
617617
brokers = ["localhost:9092"]
618+
fraud_check_analytics_topic= "hyperswitch-fraud-check-events"
618619
intent_analytics_topic = "hyperswitch-payment-intent-events"
619620
attempt_analytics_topic = "hyperswitch-payment-attempt-events"
620621
refund_analytics_topic = "hyperswitch-refund-events"

‎config/docker_compose.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -435,6 +435,7 @@ delay_between_retries_in_milliseconds = 500
435435

436436
[events.kafka]
437437
brokers = ["localhost:9092"]
438+
fraud_check_analytics_topic= "hyperswitch-fraud-check-events"
438439
intent_analytics_topic = "hyperswitch-payment-intent-events"
439440
attempt_analytics_topic = "hyperswitch-payment-attempt-events"
440441
refund_analytics_topic = "hyperswitch-refund-events"

‎crates/analytics/Cargo.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ api_models = { version = "0.1.0", path = "../api_models", features = [
1515
"errors",
1616
] }
1717
storage_impl = { version = "0.1.0", path = "../storage_impl", default-features = false }
18+
common_enums = { version = "0.1.0", path = "../common_enums" }
1819
common_utils = { version = "0.1.0", path = "../common_utils" }
1920
external_services = { version = "0.1.0", path = "../external_services", default-features = false }
2021
hyperswitch_interfaces = { version = "0.1.0", path = "../hyperswitch_interfaces" }

‎crates/analytics/docs/clickhouse/scripts/fraud_check.sql

Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
CREATE TABLE fraud_check_queue (
2+
`frm_id` String,
3+
`payment_id` String,
4+
`merchant_id` String,
5+
`attempt_id` String,
6+
`created_at` DateTime CODEC(T64, LZ4),
7+
`frm_name` LowCardinality(String),
8+
`frm_transaction_id` String,
9+
`frm_transaction_type` LowCardinality(String),
10+
`frm_status` LowCardinality(String),
11+
`frm_score` Int32,
12+
`frm_reason` LowCardinality(String),
13+
`frm_error` Nullable(String),
14+
`amount` UInt32,
15+
`currency` LowCardinality(String),
16+
`payment_method` LowCardinality(String),
17+
`payment_method_type` LowCardinality(String),
18+
`refund_transaction_id` Nullable(String),
19+
`metadata` Nullable(String),
20+
`modified_at` DateTime CODEC(T64, LZ4),
21+
`last_step` LowCardinality(String),
22+
`payment_capture_method` LowCardinality(String),
23+
`sign_flag` Int8
24+
) ENGINE = Kafka SETTINGS kafka_broker_list = 'kafka0:29092',
25+
kafka_topic_list = 'hyperswitch-fraud-check-events',
26+
kafka_group_name = 'hyper',
27+
kafka_format = 'JSONEachRow',
28+
kafka_handle_error_mode = 'stream';
29+
30+
CREATE TABLE fraud_check (
31+
`frm_id` String,
32+
`payment_id` String,
33+
`merchant_id` LowCardinality(String),
34+
`attempt_id` String,
35+
`created_at` DateTime DEFAULT now() CODEC(T64, LZ4),
36+
`frm_name` LowCardinality(String),
37+
`frm_transaction_id` String,
38+
`frm_transaction_type` LowCardinality(String),
39+
`frm_status` LowCardinality(String),
40+
`frm_score` Int32,
41+
`frm_reason` LowCardinality(String),
42+
`frm_error` Nullable(String),
43+
`amount` UInt32,
44+
`currency` LowCardinality(String),
45+
`payment_method` LowCardinality(String),
46+
`payment_method_type` LowCardinality(String),
47+
`refund_transaction_id` Nullable(String),
48+
`metadata` Nullable(String),
49+
`modified_at` DateTime DEFAULT now() CODEC(T64, LZ4),
50+
`last_step` LowCardinality(String),
51+
`payment_capture_method` LowCardinality(String),
52+
`sign_flag` Int8
53+
INDEX frmNameIndex frm_name TYPE bloom_filter GRANULARITY 1,
54+
INDEX frmStatusIndex frm_status TYPE bloom_filter GRANULARITY 1,
55+
INDEX paymentMethodIndex payment_method TYPE bloom_filter GRANULARITY 1,
56+
INDEX paymentMethodTypeIndex payment_method_type TYPE bloom_filter GRANULARITY 1,
57+
INDEX currencyIndex currency TYPE bloom_filter GRANULARITY 1
58+
) ENGINE = CollapsingMergeTree(sign_flag) PARTITION BY toStartOfDay(created_at)
59+
ORDER BY
60+
(created_at, merchant_id, attempt_id, frm_id) TTL created_at + toIntervalMonth(18) SETTINGS index_granularity = 8192;
61+
62+
CREATE MATERIALIZED VIEW fraud_check_mv TO fraud_check (
63+
`frm_id` String,
64+
`payment_id` String,
65+
`merchant_id` String,
66+
`attempt_id` String,
67+
`created_at` DateTime64(3),
68+
`frm_name` LowCardinality(String),
69+
`frm_transaction_id` String,
70+
`frm_transaction_type` LowCardinality(String),
71+
`frm_status` LowCardinality(String),
72+
`frm_score` Int32,
73+
`frm_reason` LowCardinality(String),
74+
`frm_error` Nullable(String),
75+
`amount` UInt32,
76+
`currency` LowCardinality(String),
77+
`payment_method` LowCardinality(String),
78+
`payment_method_type` LowCardinality(String),
79+
`refund_transaction_id` Nullable(String),
80+
`metadata` Nullable(String),
81+
`modified_at` DateTime64(3),
82+
`last_step` LowCardinality(String),
83+
`payment_capture_method` LowCardinality(String),
84+
`sign_flag` Int8
85+
) AS
86+
SELECT
87+
frm_id,
88+
payment_id,
89+
merchant_id,
90+
attempt_id,
91+
created_at,
92+
frm_name,
93+
frm_transaction_id,
94+
frm_transaction_type,
95+
frm_status,
96+
frm_score,
97+
frm_reason,
98+
frm_error,
99+
amount,
100+
currency,
101+
payment_method,
102+
payment_method_type,
103+
refund_transaction_id,
104+
metadata,
105+
modified_at,
106+
last_step,
107+
payment_capture_method,
108+
sign_flag
109+
FROM
110+
fraud_check_queue
111+
WHERE
112+
length(_error) = 0;
113+
114+
CREATE MATERIALIZED VIEW fraud_check_parse_errors (
115+
`topic` String,
116+
`partition` Int64,
117+
`offset` Int64,
118+
`raw` String,
119+
`error` String
120+
) ENGINE = MergeTree
121+
ORDER BY
122+
(topic, partition, offset) SETTINGS index_granularity = 8192 AS
123+
SELECT
124+
_topic AS topic,
125+
_partition AS partition,
126+
_offset AS offset,
127+
_raw_message AS raw,
128+
_error AS error
129+
FROM
130+
fraud_check_queue
131+
WHERE
132+
length(_error) > 0;

‎crates/analytics/src/clickhouse.rs

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ use time::PrimitiveDateTime;
99
use super::{
1010
active_payments::metrics::ActivePaymentsMetricRow,
1111
auth_events::metrics::AuthEventMetricRow,
12+
frm::{filters::FrmFilterRow, metrics::FrmMetricRow},
1213
health_check::HealthCheck,
1314
payment_intents::{filters::PaymentIntentFilterRow, metrics::PaymentIntentMetricRow},
1415
payments::{
@@ -130,6 +131,7 @@ impl AnalyticsDataSource for ClickhouseClient {
130131
match table {
131132
AnalyticsCollection::Payment
132133
| AnalyticsCollection::Refund
134+
| AnalyticsCollection::FraudCheck
133135
| AnalyticsCollection::PaymentIntent
134136
| AnalyticsCollection::Dispute => {
135137
TableEngine::CollapsingMergeTree { sign: "sign_flag" }
@@ -162,6 +164,8 @@ impl super::payment_intents::filters::PaymentIntentFilterAnalytics for Clickhous
162164
impl super::payment_intents::metrics::PaymentIntentMetricAnalytics for ClickhouseClient {}
163165
impl super::refunds::metrics::RefundMetricAnalytics for ClickhouseClient {}
164166
impl super::refunds::filters::RefundFilterAnalytics for ClickhouseClient {}
167+
impl super::frm::metrics::FrmMetricAnalytics for ClickhouseClient {}
168+
impl super::frm::filters::FrmFilterAnalytics for ClickhouseClient {}
165169
impl super::sdk_events::filters::SdkEventFilterAnalytics for ClickhouseClient {}
166170
impl super::sdk_events::metrics::SdkEventMetricAnalytics for ClickhouseClient {}
167171
impl super::sdk_events::events::SdkEventsFilterAnalytics for ClickhouseClient {}
@@ -290,6 +294,25 @@ impl TryInto<RefundFilterRow> for serde_json::Value {
290294
}
291295
}
292296

297+
impl TryInto<FrmMetricRow> for serde_json::Value {
298+
type Error = Report<ParsingError>;
299+
300+
fn try_into(self) -> Result<FrmMetricRow, Self::Error> {
301+
serde_json::from_value(self).change_context(ParsingError::StructParseFailure(
302+
"Failed to parse FrmMetricRow in clickhouse results",
303+
))
304+
}
305+
}
306+
307+
impl TryInto<FrmFilterRow> for serde_json::Value {
308+
type Error = Report<ParsingError>;
309+
310+
fn try_into(self) -> Result<FrmFilterRow, Self::Error> {
311+
serde_json::from_value(self).change_context(ParsingError::StructParseFailure(
312+
"Failed to parse FrmFilterRow in clickhouse results",
313+
))
314+
}
315+
}
293316
impl TryInto<DisputeMetricRow> for serde_json::Value {
294317
type Error = Report<ParsingError>;
295318

@@ -409,6 +432,7 @@ impl ToSql<ClickhouseClient> for AnalyticsCollection {
409432
match self {
410433
Self::Payment => Ok("payment_attempts".to_string()),
411434
Self::Refund => Ok("refunds".to_string()),
435+
Self::FraudCheck => Ok("fraud_check".to_string()),
412436
Self::SdkEvents => Ok("sdk_events_audit".to_string()),
413437
Self::SdkEventsAnalytics => Ok("sdk_events".to_string()),
414438
Self::ApiEvents => Ok("api_events_audit".to_string()),

‎crates/analytics/src/core.rs

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,11 @@ pub async fn get_domain_info(
2121
download_dimensions: None,
2222
dimensions: utils::get_refund_dimensions(),
2323
},
24+
AnalyticsDomain::Frm => GetInfoResponse {
25+
metrics: utils::get_frm_metrics_info(),
26+
download_dimensions: None,
27+
dimensions: utils::get_frm_dimensions(),
28+
},
2429
AnalyticsDomain::SdkEvents => GetInfoResponse {
2530
metrics: utils::get_sdk_event_metrics_info(),
2631
download_dimensions: None,

‎crates/analytics/src/frm.rs

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
pub mod accumulator;
2+
mod core;
3+
4+
pub mod filters;
5+
pub mod metrics;
6+
pub mod types;
7+
pub use accumulator::{FrmMetricAccumulator, FrmMetricsAccumulator};
8+
9+
pub use self::core::{get_filters, get_metrics};

0 commit comments

Comments
 (0)