Skip to content

Commit e2e911d

Browse files
TartanLeGrandTartanLeGrand
authored
feat(clickhouse): db cleanup (#1876)
1 parent 5ca2011 commit e2e911d

File tree

3 files changed

+314
-9
lines changed

3 files changed

+314
-9
lines changed

‎charts/sentry/templates/snuba/cleanup/cronjob-clickhouse-cleanup.yaml

Lines changed: 242 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,242 @@
1+
{{- if .Values.snuba.cleanup.enabled }}
2+
{{- $batchApiIsStable := eq (include "sentry.batch.isStable" .) "true" -}}
3+
apiVersion: {{ include "sentry.batch.apiVersion" . }}
4+
kind: CronJob
5+
metadata:
6+
name: {{ template "sentry.fullname" . }}-clickhouse-cleanup
7+
labels:
8+
app: {{ template "sentry.fullname" . }}
9+
chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}"
10+
release: "{{ .Release.Name }}"
11+
heritage: "{{ .Release.Service }}"
12+
spec:
13+
schedule: "{{ .Values.snuba.cleanup.schedule }}"
14+
successfulJobsHistoryLimit: {{ .Values.snuba.cleanup.successfulJobsHistoryLimit }}
15+
failedJobsHistoryLimit: {{ .Values.snuba.cleanup.failedJobsHistoryLimit }}
16+
concurrencyPolicy: "{{ .Values.snuba.cleanup.concurrencyPolicy }}"
17+
jobTemplate:
18+
spec:
19+
{{- if .Values.snuba.cleanup.activeDeadlineSeconds }}
20+
activeDeadlineSeconds: {{ .Values.snuba.cleanup.activeDeadlineSeconds }}
21+
{{- end}}
22+
template:
23+
metadata:
24+
annotations:
25+
checksum/configYml: {{ .Values.config.configYml | toYaml | toString | sha256sum }}
26+
checksum/config.yaml: {{ include "snuba.config" . | sha256sum }}
27+
{{- if .Values.snuba.cleanup.annotations }}
28+
{{ toYaml .Values.snuba.cleanup.annotations | indent 12 }}
29+
{{- end }}
30+
labels:
31+
app: {{ template "sentry.fullname" . }}
32+
release: "{{ .Release.Name }}"
33+
{{- if .Values.snuba.cleanup.podLabels }}
34+
{{ toYaml .Values.snuba.cleanup.podLabels | indent 12 }}
35+
{{- end }}
36+
spec:
37+
{{- if .Values.snuba.cleanup.affinity }}
38+
affinity:
39+
{{ toYaml .Values.snuba.cleanup.affinity | indent 12 }}
40+
{{- end }}
41+
{{- if .Values.snuba.cleanup.nodeSelector }}
42+
nodeSelector:
43+
{{ toYaml .Values.snuba.cleanup.nodeSelector | indent 12 }}
44+
{{- else if .Values.global.nodeSelector }}
45+
nodeSelector:
46+
{{ toYaml .Values.global.nodeSelector | indent 12 }}
47+
{{- end }}
48+
{{- if .Values.snuba.cleanup.tolerations }}
49+
tolerations:
50+
{{ toYaml .Values.snuba.cleanup.tolerations | indent 12 }}
51+
{{- else if .Values.global.tolerations }}
52+
tolerations:
53+
{{ toYaml .Values.global.tolerations | indent 12 }}
54+
{{- end }}
55+
{{- if .Values.dnsPolicy }}
56+
dnsPolicy: {{ .Values.dnsPolicy | quote }}
57+
{{- end }}
58+
{{- if .Values.dnsConfig }}
59+
dnsConfig:
60+
{{ toYaml .Values.dnsConfig | indent 12 }}
61+
{{- end }}
62+
{{- if .Values.images.snuba.imagePullSecrets }}
63+
imagePullSecrets:
64+
{{ toYaml .Values.images.snuba.imagePullSecrets | indent 12 }}
65+
{{- end }}
66+
{{- if .Values.snuba.cleanup.securityContext }}
67+
securityContext:
68+
{{ toYaml .Values.snuba.cleanup.securityContext | indent 12 }}
69+
{{- end }}
70+
containers:
71+
- name: {{ .Chart.Name }}-clickhouse-cleanup
72+
image: clickhouse/clickhouse-client:latest
73+
imagePullPolicy: IfNotPresent
74+
command: ["/bin/bash"]
75+
args:
76+
- "-c"
77+
- |
78+
set -e
79+
echo "Starting ClickHouse cleanup with retention period of {{ .Values.snuba.cleanup.retentionDays }} days"
80+
echo "Connecting to ClickHouse at $CLICKHOUSE_HOST:$CLICKHOUSE_PORT"
81+
82+
# Function to discover tables
83+
discover_tables() {
84+
echo "Auto-detecting tables in ClickHouse database..."
85+
86+
# Get all tables from the database
87+
ALL_TABLES=$(clickhouse-client \
88+
--host="$CLICKHOUSE_HOST" \
89+
--port="$CLICKHOUSE_PORT" \
90+
--user="$CLICKHOUSE_USER" \
91+
--database="$CLICKHOUSE_DATABASE" \
92+
--password="$CLICKHOUSE_PASSWORD" \
93+
--query="SHOW TABLES" 2>/dev/null || echo "")
94+
95+
if [ -z "$ALL_TABLES" ]; then
96+
echo "Warning: Could not retrieve table list, falling back to hardcoded list"
97+
return 1
98+
fi
99+
100+
echo "Found tables: $ALL_TABLES"
101+
102+
# Filter tables based on include/exclude patterns
103+
DISCOVERED_TABLES=()
104+
{{- range .Values.snuba.cleanup.tables.includePatterns }}
105+
INCLUDE_PATTERN="{{ . }}"
106+
{{- end }}
107+
{{- range .Values.snuba.cleanup.tables.excludePatterns }}
108+
EXCLUDE_PATTERN="{{ . }}"
109+
{{- end }}
110+
111+
for table in $ALL_TABLES; do
112+
# Check include patterns
113+
INCLUDE_MATCH=false
114+
{{- range .Values.snuba.cleanup.tables.includePatterns }}
115+
if echo "$table" | grep -E "{{ . }}" >/dev/null 2>&1; then
116+
INCLUDE_MATCH=true
117+
fi
118+
{{- end }}
119+
120+
# Check exclude patterns
121+
EXCLUDE_MATCH=false
122+
{{- range .Values.snuba.cleanup.tables.excludePatterns }}
123+
if echo "$table" | grep -E "{{ . }}" >/dev/null 2>&1; then
124+
EXCLUDE_MATCH=true
125+
fi
126+
{{- end }}
127+
128+
# Add to discovered tables if it matches include and doesn't match exclude
129+
if [ "$INCLUDE_MATCH" = true ] && [ "$EXCLUDE_MATCH" = false ]; then
130+
DISCOVERED_TABLES+=("$table")
131+
echo "Including table: $table"
132+
else
133+
echo "Excluding table: $table (include=$INCLUDE_MATCH, exclude=$EXCLUDE_MATCH)"
134+
fi
135+
done
136+
137+
if [ ${#DISCOVERED_TABLES[@]} -eq 0 ]; then
138+
echo "Warning: No tables matched the include/exclude patterns, falling back to hardcoded list"
139+
return 1
140+
fi
141+
142+
# Export discovered tables for cleanup
143+
printf '%s\n' "${DISCOVERED_TABLES[@]}"
144+
return 0
145+
}
146+
147+
# Determine which tables to clean up
148+
{{- if .Values.snuba.cleanup.tables.autoDetect }}
149+
echo "Table auto-detection enabled"
150+
if TABLES_TO_CLEANUP=$(discover_tables); then
151+
readarray -t TABLES <<< "$TABLES_TO_CLEANUP"
152+
echo "Using auto-detected tables: ${TABLES[*]}"
153+
else
154+
echo "Auto-detection failed, using fallback list"
155+
TABLES=(
156+
{{- range .Values.snuba.cleanup.tables.fallbackList }}
157+
"{{ . }}"
158+
{{- end }}
159+
)
160+
fi
161+
{{- else }}
162+
echo "Table auto-detection disabled, using configured fallback list"
163+
TABLES=(
164+
{{- range .Values.snuba.cleanup.tables.fallbackList }}
165+
"{{ . }}"
166+
{{- end }}
167+
)
168+
{{- end }}
169+
170+
echo "Tables to cleanup: ${TABLES[*]}"
171+
172+
# Execute cleanup for each table
173+
CLEANED_COUNT=0
174+
FAILED_COUNT=0
175+
176+
for table in "${TABLES[@]}"; do
177+
if [ -n "$table" ]; then
178+
echo "Cleaning up table: $table"
179+
if clickhouse-client \
180+
--host="$CLICKHOUSE_HOST" \
181+
--port="$CLICKHOUSE_PORT" \
182+
--user="$CLICKHOUSE_USER" \
183+
--database="$CLICKHOUSE_DATABASE" \
184+
--password="$CLICKHOUSE_PASSWORD" \
185+
--query="DELETE FROM $table WHERE timestamp < now() - INTERVAL {{ .Values.snuba.cleanup.retentionDays }} DAY"; then
186+
echo "Successfully cleaned up table: $table"
187+
CLEANED_COUNT=$((CLEANED_COUNT + 1))
188+
else
189+
echo "Warning: Failed to cleanup table $table (table may not exist or have timestamp column)"
190+
FAILED_COUNT=$((FAILED_COUNT + 1))
191+
fi
192+
fi
193+
done
194+
195+
echo "ClickHouse cleanup completed: $CLEANED_COUNT tables cleaned, $FAILED_COUNT failed"
196+
env:
197+
- name: CLICKHOUSE_HOST
198+
value: {{ include "sentry.clickhouse.host" . | quote }}
199+
- name: CLICKHOUSE_PORT
200+
value: {{ include "sentry.clickhouse.port" . | quote }}
201+
- name: CLICKHOUSE_USER
202+
value: "default"
203+
- name: CLICKHOUSE_DATABASE
204+
value: "default"
205+
- name: CLICKHOUSE_PASSWORD
206+
valueFrom:
207+
secretKeyRef:
208+
name: {{ template "sentry.fullname" . }}-clickhouse
209+
key: clickhouse-password
210+
optional: true
211+
{{- if .Values.snuba.cleanup.env }}
212+
{{ toYaml .Values.snuba.cleanup.env | indent 12 }}
213+
{{- end }}
214+
resources:
215+
{{ toYaml .Values.snuba.cleanup.resources | indent 14 }}
216+
{{- if .Values.snuba.cleanup.containerSecurityContext }}
217+
securityContext:
218+
{{ toYaml .Values.snuba.cleanup.containerSecurityContext | indent 14 }}
219+
{{- end }}
220+
{{- if .Values.snuba.cleanup.sidecars }}
221+
{{ toYaml .Values.snuba.cleanup.sidecars | indent 10 }}
222+
{{- end }}
223+
{{- if .Values.global.sidecars }}
224+
{{ toYaml .Values.global.sidecars | indent 10 }}
225+
{{- end }}
226+
restartPolicy: Never
227+
{{- if or .Values.snuba.cleanup.volumes .Values.global.volumes }}
228+
volumes:
229+
{{- if .Values.snuba.cleanup.volumes }}
230+
{{ toYaml .Values.snuba.cleanup.volumes | indent 10 }}
231+
{{- end }}
232+
{{- if .Values.global.volumes }}
233+
{{ toYaml .Values.global.volumes | indent 10 }}
234+
{{- end }}
235+
{{- end }}
236+
{{- if .Values.snuba.cleanup.priorityClassName }}
237+
priorityClassName: "{{ .Values.snuba.cleanup.priorityClassName }}"
238+
{{- end }}
239+
{{- if .Values.serviceAccount.enabled }}
240+
serviceAccountName: {{ .Values.serviceAccount.name }}-clickhouse-cleanup
241+
{{- end }}
242+
{{- end }}

‎charts/sentry/templates/snuba/cleanup/serviceaccount-clickhouse-cleanup.yaml

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
{{- if .Values.serviceAccount.enabled }}
2+
apiVersion: v1
3+
kind: ServiceAccount
4+
metadata:
5+
name: {{ .Values.serviceAccount.name }}-clickhouse-cleanup
6+
{{- if .Values.serviceAccount.annotations }}
7+
annotations: {{ toYaml .Values.serviceAccount.annotations | nindent 4 }}
8+
{{- end }}
9+
automountServiceAccountToken: {{ .Values.serviceAccount.automountServiceAccountToken }}
10+
{{- end }}

‎charts/sentry/values.yaml

Lines changed: 62 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -2019,6 +2019,68 @@ snuba:
20192019

20202020
rustConsumer: false
20212021

2022+
# ClickHouse database cleanup configuration
2023+
cleanup:
2024+
# -- Enable ClickHouse database cleanup cronjob
2025+
enabled: false
2026+
# -- Retention period in days for ClickHouse data cleanup
2027+
retentionDays: 30
2028+
# -- Cron schedule for cleanup job (daily at 2 AM by default)
2029+
schedule: "0 2 * * *"
2030+
# -- Number of successful job executions to retain
2031+
successfulJobsHistoryLimit: 5
2032+
# -- Number of failed job executions to retain
2033+
failedJobsHistoryLimit: 5
2034+
# -- Maximum time in seconds for cleanup job to run
2035+
activeDeadlineSeconds: 300
2036+
# -- Concurrency policy for cleanup job
2037+
concurrencyPolicy: Forbid
2038+
# -- Resource limits and requests for cleanup job
2039+
resources: {}
2040+
# Table discovery configuration
2041+
tables:
2042+
# -- Auto-detect tables instead of using a hardcoded list
2043+
autoDetect: true
2044+
# -- Include patterns for table names (regex patterns)
2045+
includePatterns:
2046+
- ".*_local$"
2047+
- ".*_raw_local$"
2048+
# -- Exclude patterns for table names (regex patterns)
2049+
excludePatterns:
2050+
- "^system\\."
2051+
- "^information_schema\\."
2052+
- "^INFORMATION_SCHEMA\\."
2053+
# -- Fallback hardcoded table list (used if autoDetect fails or is disabled)
2054+
fallbackList:
2055+
- "discover_local"
2056+
- "events_local"
2057+
- "events_ro_local"
2058+
- "outcomes_raw_local"
2059+
- "querylog_local"
2060+
- "sessions_raw_local"
2061+
- "transactions_local"
2062+
- "profiles_raw_local"
2063+
- "functions_raw_local"
2064+
- "replays_local"
2065+
- "generic_metric_sets_raw_local"
2066+
- "generic_metric_distributions_raw_local"
2067+
- "generic_metric_counters_raw_local"
2068+
- "spans_local"
2069+
- "group_attributes_raw_local"
2070+
- "generic_metric_gauges_raw_local"
2071+
- "profile_chunks_raw_local"
2072+
# podLabels: {}
2073+
# affinity: {}
2074+
# nodeSelector: {}
2075+
# tolerations: []
2076+
# env: []
2077+
# volumes: []
2078+
# sidecars: []
2079+
# containerSecurityContext: {}
2080+
# securityContext: {}
2081+
# annotations: {}
2082+
# volumeMounts: []
2083+
20222084
hooks:
20232085
enabled: true
20242086
preUpgrade: false
@@ -2174,15 +2236,6 @@ symbolicator:
21742236
# volumeMounts: []
21752237
# sidecars: []
21762238

2177-
# TODO The cleanup cronjob is not yet implemented
2178-
cleanup:
2179-
enabled: false
2180-
# podLabels: {}
2181-
# affinity: {}
2182-
# env: []
2183-
# volumes: []
2184-
# sidecars: []
2185-
21862239
auth:
21872240
register: true
21882241

0 commit comments

Comments
 (0)