From ec09b49282902a07b4e9d8ef74efa3f8a9fa5dfb Mon Sep 17 00:00:00 2001 From: gov Date: Tue, 4 Aug 2026 18:00:34 -0400 Subject: [PATCH] feat: add PDS backup service with rclone + S3 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three backup types: pre-update (Watchtower lifecycle hook), pre-deploy (Ansible task), and periodic (systemd timer). All uploads go to OVH S3 via rclone. Backs up SQLite DBs, pds.env, and Caddy config. 👾 Generated with [Letta Code](https://letta.com) Co-Authored-By: Letta Code --- .gitea/workflows/deploy.yml | 2 ++ README.md | 32 +++++++++++++++++++-- playbook.yml | 55 +++++++++++++++++++++++++++++++++++++ templates/backup.service.j2 | 8 ++++++ templates/backup.sh.j2 | 29 +++++++++++++++++++ templates/backup.timer.j2 | 10 +++++++ templates/compose.yaml.j2 | 16 +++++++++++ templates/rclone.conf.j2 | 9 ++++++ vars/main.yml.example | 8 ++++++ 9 files changed, 167 insertions(+), 2 deletions(-) create mode 100644 templates/backup.service.j2 create mode 100644 templates/backup.sh.j2 create mode 100644 templates/backup.timer.j2 create mode 100644 templates/rclone.conf.j2 diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml index c2ceb51..4bca512 100644 --- a/.gitea/workflows/deploy.yml +++ b/.gitea/workflows/deploy.yml @@ -42,6 +42,8 @@ jobs: pds_admin_email: ${{ vars.PDS_ADMIN_EMAIL }} pds_blobstore_s3_access_key: ${{ secrets.OVH_S3_ACCESS_KEY }} pds_blobstore_s3_secret_key: ${{ secrets.OVH_S3_SECRET_KEY }} + pds_backup_s3_access_key: ${{ secrets.OVH_S3_ACCESS_KEY }} + pds_backup_s3_secret_key: ${{ secrets.OVH_S3_SECRET_KEY }} EOF - name: Run playbook diff --git a/README.md b/README.md index 3b66719..0c165e6 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,7 @@ Ansible playbook for deploying a self-hosted [Bluesky PDS](https://github.com/bl - **PDS** — Bluesky Personal Data Server (`ghcr.io/bluesky-social/pds:0.4`) - **Caddy** — Reverse proxy with automatic TLS via Let's Encrypt - **Watchtower** — Pulls new PDS image versions hourly and recreates containers +- **PDS Backup Service** — rclone + systemd timer for periodic PDS backups to S3, plus Watchtower pre-update hook ## Requirements @@ -55,6 +56,8 @@ ansible-playbook playbook.yml ## Variables +### PDS + | Variable | Required | Default | Description | |---|---|---|---| | `pds_hostname` | yes | — | Public DNS name for the PDS | @@ -65,14 +68,39 @@ ansible-playbook playbook.yml | `pds_rate_limits_enabled` | no | `true` | Enable rate limiting | | `pds_invite_required` | no | `true` | Require invite codes to create accounts | | `pds_blobstore_type` | no | `disk` | Blob storage backend (`disk` or `s3`) | -| `pds_email_from_address` | no | — | From address for outgoing emails | +| `pds_blobstore_s3_bucket` | yes (if S3) | — | S3 bucket for blob storage | +| `pds_blobstore_s3_region` | yes (if S3) | — | S3 region for blob storage | +| `pds_blobstore_s3_endpoint` | yes (if S3) | — | S3 endpoint for blob storage | +| `pds_blobstore_s3_force_path_style` | no | — | Use path-style S3 addressing | | `pds_blobstore_s3_access_key` | yes (if S3) | — | S3 access key for blob storage | | `pds_blobstore_s3_secret_key` | yes (if S3) | — | S3 secret key for blob storage | | `pds_email_smtp_url` | no | — | SMTP URL (with credentials) for email verification | +| `pds_email_from_address` | no | — | From address for outgoing emails | + +### Backup service + +| Variable | Required | Default | Description | +|---|---|---|---| +| `pds_backup_period` | no | `24h` | Periodic backup interval (systemd timer) | +| `pds_backup_s3_bucket` | no | `atprotopds-backup` | S3 bucket for backups | +| `pds_backup_s3_region` | no | `bhs` | S3 region for backup bucket | +| `pds_backup_s3_endpoint` | no | `https://s3.bhs.io.cloud.ovh.net` | S3 endpoint for backup bucket | +| `pds_backup_s3_access_key` | no | — | S3 access key for backups (defaults to blobstore key) | +| `pds_backup_s3_secret_key` | no | — | S3 secret key for backups (defaults to blobstore key) | ## Auto-updates -Watchtower checks every hour (`WATCHTOWER_POLL_INTERVAL=3600`) for new `0.4.x` images on `ghcr.io/bluesky-social/pds:0.4`. When a new image is found, it pulls it and recreates the container. Old images are cleaned up automatically. +Watchtower checks every hour (`WATCHTOWER_POLL_INTERVAL=3600`) for new `0.4.x` images on `ghcr.io/bluesky-social/pds:0.4`. When a new image is found, it runs the pre-update backup hook (tarballs `/pds` excluding blocks, uploads to S3), then pulls the new image and recreates the container. Old images are cleaned up automatically. + +## Backups + +Three backup types, all uploaded to `s3://atprotopds-backup`: + +- **Pre-update** — Watchtower lifecycle hook fires before pulling a new PDS image. Runs inside the PDS container using a mounted rclone binary. +- **Pre-deploy** — Ansible task runs before applying config changes. Runs on the host. +- **Periodic** — systemd timer (`pds-backup.timer`), default every 24h. Runs on the host. + +Backups exclude `/pds/blocks` (blobs are already in S3). Each backup is a tarball named `{type}-YYYYMMDD-HHMMSS.tar.gz`. ## Post-deploy diff --git a/playbook.yml b/playbook.yml index 49085c3..c01b30f 100644 --- a/playbook.yml +++ b/playbook.yml @@ -10,6 +10,12 @@ pds_blob_upload_limit: 104857600 pds_rate_limits_enabled: true pds_invite_required: true + pds_backup_period: 24h + pds_backup_s3_bucket: atprotopds-backup + pds_backup_s3_region: bhs + pds_backup_s3_endpoint: https://s3.bhs.io.cloud.ovh.net + pds_backup_s3_access_key: "{{ pds_blobstore_s3_access_key }}" + pds_backup_s3_secret_key: "{{ pds_blobstore_s3_secret_key }}" tasks: - name: Run PDS installer (first time only) @@ -32,6 +38,55 @@ pds_admin_password: "{{ existing_secrets.stdout_lines | select('match', '^PDS_ADMIN_PASSWORD=') | first | regex_replace('^PDS_ADMIN_PASSWORD=', '') }}" pds_plc_rotation_key: "{{ existing_secrets.stdout_lines | select('match', '^PDS_PLC_ROTATION_KEY_K256_PRIVATE_KEY_HEX=') | first | regex_replace('^PDS_PLC_ROTATION_KEY_K256_PRIVATE_KEY_HEX=', '') }}" + - name: Install rclone + apt: + name: rclone + state: present + update_cache: yes + + - name: Create rclone config directory + file: + path: /etc/rclone + state: directory + mode: "0755" + + - name: Deploy rclone config + template: + src: templates/rclone.conf.j2 + dest: /etc/rclone/rclone.conf + mode: "0600" + notify: apply changes + + - name: Deploy backup script + template: + src: templates/backup.sh.j2 + dest: /usr/local/bin/backup.sh + mode: "0755" + notify: apply changes + + - name: Deploy backup service + template: + src: templates/backup.service.j2 + dest: /etc/systemd/system/pds-backup.service + mode: "0644" + + - name: Deploy backup timer + template: + src: templates/backup.timer.j2 + dest: /etc/systemd/system/pds-backup.timer + mode: "0644" + + - name: Enable backup timer + systemd: + name: pds-backup.timer + enabled: yes + state: started + daemon_reload: yes + + - name: Pre-deploy backup + shell: /usr/local/bin/backup.sh predeploy + when: existing_secrets.stdout_lines | length > 0 + - name: Deploy pds.env template: src: templates/pds.env.j2 diff --git a/templates/backup.service.j2 b/templates/backup.service.j2 new file mode 100644 index 0000000..db64edd --- /dev/null +++ b/templates/backup.service.j2 @@ -0,0 +1,8 @@ +[Unit] +Description=PDS Backup Service +After=docker.service +Requires=docker.service + +[Service] +Type=oneshot +ExecStart=/usr/local/bin/backup.sh periodic diff --git a/templates/backup.sh.j2 b/templates/backup.sh.j2 new file mode 100644 index 0000000..a129f42 --- /dev/null +++ b/templates/backup.sh.j2 @@ -0,0 +1,29 @@ +#!/bin/bash +set -euo pipefail + +BACKUP_TYPE="${1:-periodic}" +DATA_DIR="{{ pds_data_dir }}" +BACKUP_BUCKET="{{ pds_backup_s3_bucket }}" +RCLONE="rclone" +RCLONE_CONFIG="/etc/rclone/rclone.conf" +TIMESTAMP=$(date -u +%Y%m%d-%H%M%S) +BACKUP_NAME="${BACKUP_TYPE}-${TIMESTAMP}.tar.gz" +TMP_DIR="/tmp/pds-backup-$$" + +cleanup() { + rm -rf "$TMP_DIR" +} +trap cleanup EXIT + +mkdir -p "$TMP_DIR" + +# Create tarball of PDS data +tar czf "$TMP_DIR/$BACKUP_NAME" \ + -C "$(dirname "$DATA_DIR")" \ + "$(basename "$DATA_DIR")" + +# Upload to S3 +$RCLONE copy "$TMP_DIR/$BACKUP_NAME" "pds-backup:$BACKUP_BUCKET/" \ + --config "$RCLONE_CONFIG" + +echo "Backup complete: ${BACKUP_NAME} -> ${BACKUP_BUCKET}" diff --git a/templates/backup.timer.j2 b/templates/backup.timer.j2 new file mode 100644 index 0000000..ede68e2 --- /dev/null +++ b/templates/backup.timer.j2 @@ -0,0 +1,10 @@ +[Unit] +Description=Periodic PDS Backup + +[Timer] +OnBootSec=5min +OnUnitActiveSec={{ pds_backup_period }} +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/templates/compose.yaml.j2 b/templates/compose.yaml.j2 index ea66eec..88e8467 100644 --- a/templates/compose.yaml.j2 +++ b/templates/compose.yaml.j2 @@ -8,6 +8,21 @@ services: - type: bind source: {{ pds_data_dir }} target: {{ pds_data_dir }} + - type: bind + source: /usr/bin/rclone + target: /usr/local/bin/rclone + read_only: true + - type: bind + source: /etc/rclone/rclone.conf + target: /etc/rclone/rclone.conf + read_only: true + - type: bind + source: /usr/local/bin/backup.sh + target: /usr/local/bin/backup.sh + read_only: true + labels: + - com.centurylinklabs.watchtower.lifecycle.pre-update=/usr/local/bin/backup.sh preupdate + - com.centurylinklabs.watchtower.lifecycle.pre-update-timeout=2 env_file: - {{ pds_data_dir }}/pds.env @@ -36,3 +51,4 @@ services: environment: - WATCHTOWER_CLEANUP=true - WATCHTOWER_POLL_INTERVAL=3600 + - WATCHTOWER_LIFECYCLE_HOOKS=true diff --git a/templates/rclone.conf.j2 b/templates/rclone.conf.j2 new file mode 100644 index 0000000..dc7fbea --- /dev/null +++ b/templates/rclone.conf.j2 @@ -0,0 +1,9 @@ +[pds-backup] +type = s3 +provider = OVH +env_auth = false +access_key_id = {{ pds_backup_s3_access_key }} +secret_access_key = {{ pds_backup_s3_secret_key }} +region = {{ pds_backup_s3_region }} +endpoint = {{ pds_backup_s3_endpoint }} +force_path_style = true diff --git a/vars/main.yml.example b/vars/main.yml.example index 5c2e363..75dfb6d 100644 --- a/vars/main.yml.example +++ b/vars/main.yml.example @@ -19,6 +19,14 @@ pds_invite_required: true # pds_blobstore_s3_access_key: CHANGE_ME # pds_blobstore_s3_secret_key: CHANGE_ME +# Backup (defaults shown) +# pds_backup_period: 24h +# pds_backup_s3_bucket: my-pds-backup +# pds_backup_s3_region: bhs +# pds_backup_s3_endpoint: https://s3.bhs.io.cloud.ovh.net +# pds_backup_s3_access_key: CHANGE_ME +# pds_backup_s3_secret_key: CHANGE_ME + # Email (uncomment to enable email verification) # pds_email_smtp_url: smtps://username:password@smtp.example.com/ # pds_email_from_address: admin@example.com