feat(router): add incoming header request logs (#3939)

Author: sahkal

config/deployments/integration_test.toml

@@ -290,3 +290,6 @@ outgoing_enabled = true
 
 [webhook_source_verification_call]
 connectors_with_webhook_source_verification_call = "paypal"
+
+[unmasked_headers]
+keys = "user-agent"

config/deployments/production.toml

@@ -304,3 +304,6 @@ outgoing_enabled = true
 
 [webhook_source_verification_call]
 connectors_with_webhook_source_verification_call = "paypal"
+
+[unmasked_headers]
+keys = "user-agent"

config/deployments/sandbox.toml

@@ -306,3 +306,6 @@ outgoing_enabled = true
 
 [webhook_source_verification_call]
 connectors_with_webhook_source_verification_call = "paypal"
+
+[unmasked_headers]
+keys = "user-agent"

config/development.toml

@@ -571,3 +571,6 @@ enabled = true
 
 [file_storage]
 file_storage_backend = "file_system"
+
+[unmasked_headers]
+keys = "user-agent"

config/docker_compose.toml

@@ -430,3 +430,6 @@ source = "logs"
 
 [file_storage]
 file_storage_backend = "file_system"
+
+[unmasked_headers]
+keys = "user-agent"

crates/router/src/configs/secrets_transformers.rs

@@ -353,5 +353,6 @@ pub(crate) async fn fetch_raw_secrets(
         #[cfg(feature = "olap")]
         connector_onboarding,
         cors: conf.cors,
+        unmasked_headers: conf.unmasked_headers,
     }
 }

crates/router/src/configs/settings.rs

@@ -115,6 +115,13 @@ pub struct Settings<S: SecretState> {
     pub events: EventsConfig,
     #[cfg(feature = "olap")]
     pub connector_onboarding: SecretStateContainer<ConnectorOnboarding, S>,
+    pub unmasked_headers: UnmaskedHeaders,
+}
+
+#[derive(Debug, Deserialize, Clone, Default)]
+pub struct UnmaskedHeaders {
+    #[serde(deserialize_with = "deserialize_hashset")]
+    pub keys: HashSet<String>,
 }
 
 #[cfg(feature = "frm")]

crates/router/src/services/api.rs

@@ -1121,11 +1121,35 @@ where
 {
     let request_method = request.method().as_str();
     let url_path = request.path();
+
+    let unmasked_incoming_header_keys = state.conf().unmasked_headers.keys;
+
+    let incoming_request_header = request.headers();
+
+    let incoming_header_to_log: HashMap<String, http::header::HeaderValue> =
+        incoming_request_header
+            .iter()
+            .fold(HashMap::new(), |mut acc, (key, value)| {
+                let key = key.to_string();
+                if unmasked_incoming_header_keys.contains(&key.as_str().to_lowercase()) {
+                    acc.insert(key.clone(), value.clone());
+                } else {
+                    acc.insert(
+                        key.clone(),
+                        http::header::HeaderValue::from_static("**MASKED**"),
+                    );
+                }
+                acc
+            });
+
     tracing::Span::current().record("request_method", request_method);
     tracing::Span::current().record("request_url_path", url_path);
 
     let start_instant = Instant::now();
-    logger::info!(tag = ?Tag::BeginRequest, payload = ?payload);
+
+    logger::info!(
+        tag = ?Tag::BeginRequest, payload = ?payload,
+    headers = ?incoming_header_to_log);
 
     let server_wrap_util_res = metrics::request::record_request_time_metric(
         server_wrap_util(

loadtest/config/development.toml

@@ -293,3 +293,6 @@ enabled = true
 client_id = ""
 client_secret = ""
 partner_id = ""
+
+[unmasked_headers]
+keys = "user-agent"