From be08225aaff72baebd4d1e2596b935b17d462451 Mon Sep 17 00:00:00 2001 From: gov Date: Sun, 9 Aug 2026 23:15:35 -0400 Subject: [PATCH] feat: pds 0.5 with telemetry and branding changes --- .gitea/workflows/deploy.yml | 11 +++++ README.md | 61 ++++++++++++++++++++++-- playbook.yml | 70 +++++++++++++++++++++++++++- templates/grafana-dashboards.yml.j2 | 11 +++++ templates/grafana-datasources.yml.j2 | 12 +++++ templates/monitoring-compose.yaml.j2 | 66 ++++++++++++++++++++++++++ templates/pds.env.j2 | 48 +++++++++++++++++++ templates/prometheus.yml.j2 | 28 +++++++++++ vars/main.yml.example | 23 ++++++++- 9 files changed, 323 insertions(+), 7 deletions(-) create mode 100644 templates/grafana-dashboards.yml.j2 create mode 100644 templates/grafana-datasources.yml.j2 create mode 100644 templates/monitoring-compose.yaml.j2 create mode 100644 templates/prometheus.yml.j2 diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml index 4bca512..0ca0f1e 100644 --- a/.gitea/workflows/deploy.yml +++ b/.gitea/workflows/deploy.yml @@ -45,6 +45,17 @@ jobs: pds_backup_s3_access_key: ${{ secrets.OVH_S3_ACCESS_KEY }} pds_backup_s3_secret_key: ${{ secrets.OVH_S3_SECRET_KEY }} EOF + # Optional: enable telemetry/monitoring + if [ -n "${{ vars.PDS_TELEMETRY_ENABLED }}" ]; then + echo "pds_telemetry_enabled: ${{ vars.PDS_TELEMETRY_ENABLED }}" >> vars/main.yml + fi + # Optional: branding + if [ -n "${{ vars.PDS_SERVICE_NAME }}" ]; then + echo "pds_service_name: \"${{ vars.PDS_SERVICE_NAME }}\"" >> vars/main.yml + fi + if [ -n "${{ vars.PDS_PRIMARY_COLOR }}" ]; then + echo "pds_primary_color: \"${{ vars.PDS_PRIMARY_COLOR }}\"" >> vars/main.yml + fi - name: Run playbook run: | diff --git a/README.md b/README.md index 0c165e6..3e836b9 100644 --- a/README.md +++ b/README.md @@ -4,10 +4,11 @@ Ansible playbook for deploying a self-hosted [Bluesky PDS](https://github.com/bl ## What it deploys -- **PDS** — Bluesky Personal Data Server (`ghcr.io/bluesky-social/pds:0.4`) +- **PDS** — Bluesky Personal Data Server (`ghcr.io/bluesky-social/pds:0.5`) - **Caddy** — Reverse proxy with automatic TLS via Let's Encrypt - **Watchtower** — Pulls new PDS image versions hourly and recreates containers - **PDS Backup Service** — rclone + systemd timer for periodic PDS backups to S3, plus Watchtower pre-update hook +- **Monitoring** *(optional)* — Prometheus + Grafana + node_exporter stack with OpenTelemetry metrics and a pre-built PDS Overview dashboard ## Requirements @@ -63,8 +64,8 @@ ansible-playbook playbook.yml | `pds_hostname` | yes | — | Public DNS name for the PDS | | `pds_admin_email` | yes | — | Admin contact email (used by Caddy for Let's Encrypt) | | `pds_data_dir` | no | `/pds` | Data directory (must be `/pds` — the installer enforces this) | -| `pds_image_tag` | no | `0.4` | PDS Docker image tag | -| `pds_blob_upload_limit` | no | `104857600` | Max blob upload size in bytes | +| `pds_image_tag` | no | `0.5` | PDS Docker image tag | +| `pds_blob_upload_limit` | no | `314572800` | Max blob upload size in bytes (300MB) | | `pds_rate_limits_enabled` | no | `true` | Enable rate limiting | | `pds_invite_required` | no | `true` | Require invite codes to create accounts | | `pds_blobstore_type` | no | `disk` | Blob storage backend (`disk` or `s3`) | @@ -77,6 +78,35 @@ ansible-playbook playbook.yml | `pds_email_smtp_url` | no | — | SMTP URL (with credentials) for email verification | | `pds_email_from_address` | no | — | From address for outgoing emails | +### Branding + +All optional. Defaults to hostname + "PDS" with neutral colors. See the [PDS customization blog post](https://atproto.com/blog/pds-customization-and-metrics) for details. + +| Variable | Default | Description | +|---|---|---| +| `pds_service_name` | hostname + " PDS" | Display name shown on auth screens and emails | +| `pds_logo_url` | — | Logo URL (SVG recommended) | +| `pds_primary_color` | — | Primary color for buttons, links, accents (hex, e.g. `#7c3aed`) | +| `pds_error_color` | — | Error state color | +| `pds_warning_color` | — | Warning state color | +| `pds_info_color` | — | Info state color | +| `pds_success_color` | — | Success state color | +| `pds_home_url` | — | Home page link in footer | +| `pds_terms_of_service_url` | — | Terms of service link in footer | +| `pds_privacy_policy_url` | — | Privacy policy link in footer | +| `pds_support_url` | — | Support link in footer | +| `pds_background_light_url` | — | Background image for auth screens (light mode) | +| `pds_background_dark_url` | — | Background image for auth screens (dark mode) | + +### Telemetry & monitoring + +Requires PDS image `0.5+` (telemetry added in `@atproto/pds` 0.5.26). When enabled, deploys Prometheus + Grafana + node_exporter alongside the PDS. All services bind to `127.0.0.1` — access via SSH tunnel. + +| Variable | Default | Description | +|---|---|---| +| `pds_telemetry_enabled` | `false` | Enable OpenTelemetry + monitoring stack | +| `pds_grafana_image_tag` | `13.1.1` | Grafana Docker image tag (pinned — no floating major tag) | + ### Backup service | Variable | Required | Default | Description | @@ -102,6 +132,28 @@ Three backup types, all uploaded to `s3://atprotopds-backup`: Backups exclude `/pds/blocks` (blobs are already in S3). Each backup is a tarball named `{type}-YYYYMMDD-HHMMSS.tar.gz`. +## Monitoring + +Set `pds_telemetry_enabled: true` in `vars/main.yml` to enable OpenTelemetry metrics and deploy the monitoring stack (Prometheus + Grafana + node_exporter). The PDS pushes metrics over OTLP to Prometheus, which receives them directly (no collector needed). + +**Access Grafana via SSH tunnel:** + +```bash +ssh -L 3001:localhost:3001 sysadmin@your-server +``` + +Then open `http://localhost:3001` and log in with `admin`/`admin` (change on first login). The PDS Overview dashboard is provisioned automatically in a folder named "PDS". + +**Ports (all bound to 127.0.0.1):** + +| Service | Port | Purpose | +|---|---|---| +| Grafana | 3001 | Dashboard UI | +| Prometheus | 9090 | Metrics query API | +| node_exporter | 9100 | Host metrics | + +Prometheus retains 15 days of data by default. Data is stored in Docker volumes, not under `/pds`, so it's not included in PDS backups. + ## Post-deploy Check that the PDS is healthy: @@ -145,6 +197,9 @@ Auto-deploys on push to `main`. Set up these in Gitea repo settings (Settings > | `SERVER_IP` | Target server IP address | | `PDS_HOSTNAME` | Public DNS name for the PDS (e.g., `pds.example.com`) | | `PDS_ADMIN_EMAIL` | Admin contact email | +| `PDS_TELEMETRY_ENABLED` | *(optional)* Set to `true` to enable telemetry + monitoring stack | +| `PDS_SERVICE_NAME` | *(optional)* PDS display name for branding | +| `PDS_PRIMARY_COLOR` | *(optional)* Primary color (hex) for branding | **Secrets** (sensitive credentials): diff --git a/playbook.yml b/playbook.yml index 46f4152..85588b3 100644 --- a/playbook.yml +++ b/playbook.yml @@ -6,8 +6,8 @@ - vars/main.yml vars: pds_data_dir: /pds - pds_image_tag: "0.4" - pds_blob_upload_limit: 104857600 + pds_image_tag: "0.5" + pds_blob_upload_limit: 314572800 pds_rate_limits_enabled: true pds_invite_required: true pds_backup_period: 24h @@ -16,6 +16,7 @@ pds_backup_s3_endpoint: https://s3.bhs.io.cloud.ovh.net pds_backup_s3_access_key: "{{ pds_blobstore_s3_access_key }}" pds_backup_s3_secret_key: "{{ pds_blobstore_s3_secret_key }}" + pds_grafana_image_tag: "13.1.1" tasks: - name: Run PDS installer (first time only) @@ -135,6 +136,71 @@ changed_when: false failed_when: pds_backup_exit_code.stdout != "0" + # ── Monitoring stack (telemetry) ────────────────────────────── + # Deploys Prometheus + Grafana + node_exporter when pds_telemetry_enabled is true. + # All services bind to 127.0.0.1 — access via SSH tunnel: + # ssh -L 3001:localhost:3001 sysadmin@ + # Then open http://localhost:3001 (admin/admin, change on first login) + + - name: Create monitoring directories + file: + path: "{{ item }}" + state: directory + mode: "0755" + loop: + - "{{ pds_data_dir }}/monitoring" + - "{{ pds_data_dir }}/monitoring/grafana/provisioning/datasources" + - "{{ pds_data_dir }}/monitoring/grafana/provisioning/dashboards" + - "{{ pds_data_dir }}/monitoring/dashboards" + when: pds_telemetry_enabled is defined and pds_telemetry_enabled | bool + + - name: Deploy monitoring compose + template: + src: templates/monitoring-compose.yaml.j2 + dest: "{{ pds_data_dir }}/monitoring/compose.yaml" + mode: "0644" + when: pds_telemetry_enabled is defined and pds_telemetry_enabled | bool + notify: restart monitoring + + - name: Deploy Prometheus config + template: + src: templates/prometheus.yml.j2 + dest: "{{ pds_data_dir }}/monitoring/prometheus.yml" + mode: "0644" + when: pds_telemetry_enabled is defined and pds_telemetry_enabled | bool + notify: restart monitoring + + - name: Deploy Grafana datasource provisioning + template: + src: templates/grafana-datasources.yml.j2 + dest: "{{ pds_data_dir }}/monitoring/grafana/provisioning/datasources/prometheus.yml" + mode: "0644" + when: pds_telemetry_enabled is defined and pds_telemetry_enabled | bool + notify: restart monitoring + + - name: Deploy Grafana dashboard provisioning + template: + src: templates/grafana-dashboards.yml.j2 + dest: "{{ pds_data_dir }}/monitoring/grafana/provisioning/dashboards/pds.yml" + mode: "0644" + when: pds_telemetry_enabled is defined and pds_telemetry_enabled | bool + notify: restart monitoring + + - name: Download PDS Overview dashboard + get_url: + url: https://raw.githubusercontent.com/bluesky-social/pds/main/monitoring/dashboards/pds-overview.json + dest: "{{ pds_data_dir }}/monitoring/dashboards/pds-overview.json" + mode: "0644" + when: pds_telemetry_enabled is defined and pds_telemetry_enabled | bool + notify: restart monitoring + + - name: Start monitoring stack + shell: cd {{ pds_data_dir }}/monitoring && docker compose up -d + when: pds_telemetry_enabled is defined and pds_telemetry_enabled | bool + handlers: - name: apply changes shell: cd {{ pds_data_dir }} && docker compose up -d && docker restart caddy + + - name: restart monitoring + shell: cd {{ pds_data_dir }}/monitoring && docker compose up -d --force-recreate diff --git a/templates/grafana-dashboards.yml.j2 b/templates/grafana-dashboards.yml.j2 new file mode 100644 index 0000000..55059bd --- /dev/null +++ b/templates/grafana-dashboards.yml.j2 @@ -0,0 +1,11 @@ +apiVersion: 1 + +providers: + - name: PDS + type: file + folder: PDS + updateIntervalSeconds: 30 + allowUiUpdates: false + options: + path: /var/lib/grafana/dashboards + foldersFromFilesStructure: false diff --git a/templates/grafana-datasources.yml.j2 b/templates/grafana-datasources.yml.j2 new file mode 100644 index 0000000..82e8b82 --- /dev/null +++ b/templates/grafana-datasources.yml.j2 @@ -0,0 +1,12 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + uid: pds-prometheus + type: prometheus + access: proxy + url: http://127.0.0.1:9090 + isDefault: true + editable: false + jsonData: + timeInterval: 15s diff --git a/templates/monitoring-compose.yaml.j2 b/templates/monitoring-compose.yaml.j2 new file mode 100644 index 0000000..3eeebf9 --- /dev/null +++ b/templates/monitoring-compose.yaml.j2 @@ -0,0 +1,66 @@ +name: pds-monitoring + +# Self-contained monitoring stack for a self-hosted PDS. +# Deployed by Ansible to {{ pds_data_dir }}/monitoring/compose.yaml +# +# All services bind to 127.0.0.1 — nothing is exposed to the internet. +# Reach Grafana via SSH tunnel: +# ssh -L 3001:localhost:3001 you@your-pds-host +# Then open http://localhost:3001 (admin/admin, change on first login) + +services: + node-exporter: + container_name: pds-node-exporter + image: prom/node-exporter:v1 + network_mode: host + pid: host + restart: unless-stopped + command: + - '--path.procfs=/host/proc' + - '--path.sysfs=/host/sys' + - '--path.rootfs=/rootfs' + - '--collector.filesystem.mount-points-exclude=^/(dev|proc|sys|run|var/lib/docker/.+)($$|/)' + - '--web.listen-address=127.0.0.1:9100' + volumes: + - /proc:/host/proc:ro + - /sys:/host/sys:ro + - /:/rootfs:ro + + prometheus: + container_name: pds-prometheus + image: prom/prometheus:v3 + network_mode: host + restart: unless-stopped + command: + - '--config.file=/etc/prometheus/prometheus.yml' + - '--storage.tsdb.path=/prometheus' + - '--storage.tsdb.retention.time=15d' + - '--web.listen-address=127.0.0.1:9090' + # Lets the PDS push OTLP metrics directly — no collector needed + - '--web.enable-otlp-receiver' + volumes: + - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro + - prometheus-data:/prometheus + + grafana: + container_name: pds-grafana + # Grafana publishes no floating major tag — pinned to exact version + image: grafana/grafana:{{ pds_grafana_image_tag }} + network_mode: host + depends_on: + - prometheus + restart: unless-stopped + environment: + GF_SERVER_HTTP_ADDR: 127.0.0.1 + GF_SERVER_HTTP_PORT: 3001 + GF_ANALYTICS_REPORTING_ENABLED: false + GF_ANALYTICS_CHECK_FOR_UPDATES: false + GF_USERS_ALLOW_SIGN_UP: false + volumes: + - ./grafana/provisioning:/etc/grafana/provisioning:ro + - ./dashboards:/var/lib/grafana/dashboards:ro + - grafana-data:/var/lib/grafana + +volumes: + prometheus-data: + grafana-data: diff --git a/templates/pds.env.j2 b/templates/pds.env.j2 index a462a1f..6f795a9 100644 --- a/templates/pds.env.j2 +++ b/templates/pds.env.j2 @@ -33,3 +33,51 @@ PDS_EMAIL_SMTP_URL={{ pds_email_smtp_url }} {% if pds_email_from_address is defined %} PDS_EMAIL_FROM_ADDRESS={{ pds_email_from_address }} {% endif %} +{% if pds_service_name is defined %} +PDS_SERVICE_NAME={{ pds_service_name }} +{% endif %} +{% if pds_logo_url is defined %} +PDS_LOGO_URL={{ pds_logo_url }} +{% endif %} +{% if pds_primary_color is defined %} +PDS_PRIMARY_COLOR={{ pds_primary_color }} +{% endif %} +{% if pds_error_color is defined %} +PDS_ERROR_COLOR={{ pds_error_color }} +{% endif %} +{% if pds_warning_color is defined %} +PDS_WARNING_COLOR={{ pds_warning_color }} +{% endif %} +{% if pds_info_color is defined %} +PDS_INFO_COLOR={{ pds_info_color }} +{% endif %} +{% if pds_success_color is defined %} +PDS_SUCCESS_COLOR={{ pds_success_color }} +{% endif %} +{% if pds_home_url is defined %} +PDS_HOME_URL={{ pds_home_url }} +{% endif %} +{% if pds_terms_of_service_url is defined %} +PDS_TERMS_OF_SERVICE_URL={{ pds_terms_of_service_url }} +{% endif %} +{% if pds_privacy_policy_url is defined %} +PDS_PRIVACY_POLICY_URL={{ pds_privacy_policy_url }} +{% endif %} +{% if pds_support_url is defined %} +PDS_SUPPORT_URL={{ pds_support_url }} +{% endif %} +{% if pds_background_light_url is defined %} +PDS_BACKGROUND_LIGHT_URL={{ pds_background_light_url }} +{% endif %} +{% if pds_background_dark_url is defined %} +PDS_BACKGROUND_DARK_URL={{ pds_background_dark_url }} +{% endif %} +{% if pds_telemetry_enabled is defined and pds_telemetry_enabled | bool %} +NODE_OPTIONS=--import=@atproto/pds/telemetry +OTEL_SERVICE_NAME=pds +OTEL_EXPORTER_OTLP_METRICS_ENDPOINT=http://localhost:9090/api/v1/otlp/v1/metrics +OTEL_EXPORTER_OTLP_METRICS_PROTOCOL=http/protobuf +OTEL_METRIC_EXPORT_INTERVAL=15000 +OTEL_SEMCONV_STABILITY_OPT_IN=http +OTEL_NODE_RESOURCE_DETECTORS=env,host,os,process,serviceinstance,container +{% endif %} diff --git a/templates/prometheus.yml.j2 b/templates/prometheus.yml.j2 new file mode 100644 index 0000000..26715f8 --- /dev/null +++ b/templates/prometheus.yml.j2 @@ -0,0 +1,28 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + +# Host metrics are scraped. PDS metrics are *pushed* by the PDS over OTLP to +# /api/v1/otlp/v1/metrics (enabled by --web.enable-otlp-receiver in +# compose.yaml), so the PDS needs no scrape job here. +scrape_configs: + - job_name: node + static_configs: + - targets: ['127.0.0.1:9100'] + + - job_name: prometheus + static_configs: + - targets: ['127.0.0.1:9090'] + +otlp: + # Keep OTel resource attributes that identify which PDS a series came from + promote_resource_attributes: + - service.name + - service.version + - service.instance.id + - host.name + +storage: + tsdb: + # OTLP is a push protocol, so samples can arrive slightly out of order + out_of_order_time_window: 30m diff --git a/vars/main.yml.example b/vars/main.yml.example index 75dfb6d..27b8cfa 100644 --- a/vars/main.yml.example +++ b/vars/main.yml.example @@ -5,8 +5,8 @@ pds_admin_email: admin@example.com # Optional (defaults shown) # pds_data_dir must be /pds — the installer enforces this pds_data_dir: /pds -pds_image_tag: "0.4" -pds_blob_upload_limit: 104857600 +pds_image_tag: "0.5" +pds_blob_upload_limit: 314572800 pds_rate_limits_enabled: true pds_invite_required: true @@ -30,3 +30,22 @@ pds_invite_required: true # Email (uncomment to enable email verification) # pds_email_smtp_url: smtps://username:password@smtp.example.com/ # pds_email_from_address: admin@example.com + +# Branding (all optional — defaults to hostname + "PDS" with neutral colors) +# pds_service_name: "Example Social" +# pds_logo_url: "https://example.com/logo.svg" +# pds_primary_color: "#7c3aed" +# pds_error_color: "#dc2626" +# pds_warning_color: "#f59e0b" +# pds_info_color: "#3b82f6" +# pds_success_color: "#16a34a" +# pds_home_url: "https://example.com" +# pds_terms_of_service_url: "https://example.com/tos" +# pds_privacy_policy_url: "https://example.com/privacy" +# pds_support_url: "https://example.com/support" +# pds_background_light_url: "https://example.com/bg-light.png" +# pds_background_dark_url: "https://example.com/bg-dark.png" + +# Telemetry & monitoring (OpenTelemetry + Prometheus + Grafana) +# Requires PDS image 0.5+ (telemetry added in @atproto/pds 0.5.26) +# pds_telemetry_enabled: true