From 81ed6c90338ba35391f3aa0816d4c87212622cb8 Mon Sep 17 00:00:00 2001 From: Douglas J Hunley Date: Tue, 21 Jul 2026 13:29:56 -0400 Subject: [PATCH 01/12] feat(sudoers): let supabase-admin-agent invoke pgbackrest reconcile as root MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The salt subcommand's systemd service runs as the unprivileged supabase-admin-agent OS user, but /etc/pgbackrest/conf.d is pgbackrest:postgres 02770 — only root/adminapi can write there. Adds a NOPASSWD sudoers entry for `pgbackrest reconcile`, mirroring the existing salt-call entry, so the salt cycle can elevate just this one command (via sudo) to keep async-archiving config in sync with billing tier (INDATA-996) without granting the supabase-admin-agent user any broader group membership. --- .../supabase-admin-agent.sudoers.conf | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ansible/files/supabase_admin_agent_config/supabase-admin-agent.sudoers.conf b/ansible/files/supabase_admin_agent_config/supabase-admin-agent.sudoers.conf index df65f11370..1894047dae 100644 --- a/ansible/files/supabase_admin_agent_config/supabase-admin-agent.sudoers.conf +++ b/ansible/files/supabase_admin_agent_config/supabase-admin-agent.sudoers.conf @@ -1,2 +1,7 @@ %supabase-admin-agent ALL= NOPASSWD: /usr/bin/salt-call %supabase-admin-agent ALL= NOPASSWD: /usr/bin/gpg --homedir /etc/salt/gpgkeys --import, /usr/bin/gpg --homedir /etc/salt/gpgkeys --list-secret-keys * +# Lets the salt subcommand (running as the unprivileged supabase-admin-agent +# user) reconcile pgBackRest's async-archiving config with the instance's +# billing tier on each Salt cycle (INDATA-996) — /etc/pgbackrest/conf.d is +# only writable by root/adminapi. +%supabase-admin-agent ALL= NOPASSWD: /opt/supabase-admin-agent/supabase-admin-agent pgbackrest reconcile From 6a0ed91a58388afe629e36cc5dd9ed928d7372dc Mon Sep 17 00:00:00 2001 From: Douglas J Hunley Date: Wed, 22 Jul 2026 12:39:14 -0400 Subject: [PATCH 02/12] fix(ansible): root-own supabase-admin-agent binary path, not the low-priv agent MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The new sudoers rule in this PR grants the supabase-admin-agent user NOPASSWD root exec of /opt/supabase-admin-agent/supabase-admin-agent, but the config dir, extracted archive, and symlink were all owned by that same low-priv user/group. Since sudo matches on path only (not content/hash), that account could replace the binary and use the new sudo grant to run arbitrary code as root. Own the dir/binary/symlink as root:root instead, matching the existing pgdata-chown/pgdata-signal pattern in this file — the agent process only needs to execute it, never write to it. --- ansible/tasks/internal/supabase-admin-agent.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/ansible/tasks/internal/supabase-admin-agent.yml b/ansible/tasks/internal/supabase-admin-agent.yml index 69e7861291..af08e135b2 100644 --- a/ansible/tasks/internal/supabase-admin-agent.yml +++ b/ansible/tasks/internal/supabase-admin-agent.yml @@ -15,7 +15,8 @@ - name: supabase-admin-agent - config dir file: path: /opt/supabase-admin-agent - owner: supabase-admin-agent + owner: root + group: root state: directory - name: supabase-admin-agent - gpg dir @@ -90,7 +91,8 @@ remote_src: yes src: /tmp/supabase-admin-agent.tar.gz dest: /opt/supabase-admin-agent/ - owner: supabase-admin-agent + owner: root + group: root extra_opts: - --strip-components=1 @@ -99,7 +101,8 @@ path: /opt/supabase-admin-agent/supabase-admin-agent src: "/opt/supabase-admin-agent/supabase-admin-agent-linux-{{ arch }}" state: link - owner: supabase-admin-agent + owner: root + group: root mode: "0755" force: yes From 63a434a037d10d6c3bd4d062680694d6ab2c9d7f Mon Sep 17 00:00:00 2001 From: Douglas J Hunley Date: Thu, 23 Jul 2026 11:05:46 -0400 Subject: [PATCH 03/12] fix(ansible): move pgBackRest spool-path off the 10GB root volume MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit spool-path was never set explicitly, so pgBackRest defaulted to /var/spool/pgbackrest, which lands on the AMI's root volume (10GB, shared with the OS/systemd/journal) rather than the /data EBS volume PGDATA lives on. archive-get's async replica-catch-up queue can hold several 16MB+ WAL segments at once; once archive-async is enabled fleet-wide (INDATA-996) that risks filling the root volume and taking the instance down. Points spool-path at /data/pgbackrest_spool instead, and updates the directory-creation task to match. Global option, set once at provisioning — no supabase-admin-agent changes needed. --- ansible/files/pgbackrest_config/pgbackrest.conf | 2 ++ ansible/tasks/setup-pgbackrest.yml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/ansible/files/pgbackrest_config/pgbackrest.conf b/ansible/files/pgbackrest_config/pgbackrest.conf index f54336c4e8..ea136a8d3d 100644 --- a/ansible/files/pgbackrest_config/pgbackrest.conf +++ b/ansible/files/pgbackrest_config/pgbackrest.conf @@ -13,6 +13,8 @@ log-level-console = info log-level-file = detail log-subprocess = y resume = n +# Explicit vs. the /var/spool/pgbackrest default, which sits on the 10GB AMI root volume that archive-get's replica-catch-up queue (multi-segment WAL) could fill; /data is the dedicated EBS volume. +spool-path = /data/pgbackrest_spool start-fast = y # Note: the [supabase] stanza (pg1-path, pg1-socket-path, pg1-user) has been # removed from this file. supabase-admin-agent owns that stanza and writes it diff --git a/ansible/tasks/setup-pgbackrest.yml b/ansible/tasks/setup-pgbackrest.yml index 340fb7d963..d93bdcfc58 100644 --- a/ansible/tasks/setup-pgbackrest.yml +++ b/ansible/tasks/setup-pgbackrest.yml @@ -58,7 +58,7 @@ # (running as the pgbackrest user) cannot read conf files created by adminapi. - {dir: /etc/pgbackrest/conf.d, mode: '02770'} - {dir: /var/lib/pgbackrest} - - {dir: /var/spool/pgbackrest} + - {dir: /data/pgbackrest_spool} - {dir: /var/log/pgbackrest} loop_control: loop_var: backrest_dir From 52d321e37506f9dfdee67814f5a485539798b329 Mon Sep 17 00:00:00 2001 From: Douglas J Hunley Date: Thu, 23 Jul 2026 11:18:54 -0400 Subject: [PATCH 04/12] Revert "fix(ansible): move pgBackRest spool-path off the 10GB root volume" This reverts commit 63a434a037d10d6c3bd4d062680694d6ab2c9d7f. --- ansible/files/pgbackrest_config/pgbackrest.conf | 2 -- ansible/tasks/setup-pgbackrest.yml | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/ansible/files/pgbackrest_config/pgbackrest.conf b/ansible/files/pgbackrest_config/pgbackrest.conf index ea136a8d3d..f54336c4e8 100644 --- a/ansible/files/pgbackrest_config/pgbackrest.conf +++ b/ansible/files/pgbackrest_config/pgbackrest.conf @@ -13,8 +13,6 @@ log-level-console = info log-level-file = detail log-subprocess = y resume = n -# Explicit vs. the /var/spool/pgbackrest default, which sits on the 10GB AMI root volume that archive-get's replica-catch-up queue (multi-segment WAL) could fill; /data is the dedicated EBS volume. -spool-path = /data/pgbackrest_spool start-fast = y # Note: the [supabase] stanza (pg1-path, pg1-socket-path, pg1-user) has been # removed from this file. supabase-admin-agent owns that stanza and writes it diff --git a/ansible/tasks/setup-pgbackrest.yml b/ansible/tasks/setup-pgbackrest.yml index d93bdcfc58..340fb7d963 100644 --- a/ansible/tasks/setup-pgbackrest.yml +++ b/ansible/tasks/setup-pgbackrest.yml @@ -58,7 +58,7 @@ # (running as the pgbackrest user) cannot read conf files created by adminapi. - {dir: /etc/pgbackrest/conf.d, mode: '02770'} - {dir: /var/lib/pgbackrest} - - {dir: /data/pgbackrest_spool} + - {dir: /var/spool/pgbackrest} - {dir: /var/log/pgbackrest} loop_control: loop_var: backrest_dir From 5cebbd2985b2f523353565240da699ea1ca94e1e Mon Sep 17 00:00:00 2001 From: Douglas J Hunley Date: Thu, 23 Jul 2026 12:25:47 -0400 Subject: [PATCH 05/12] docs(ansible): collapse stacked sudoers comment, note root-ownership rationale Single-line comments only per updated style rules: collapses the 4-line sudoers rationale into one line, and adds a why-comment on the new root:root ownership for /opt/supabase-admin-agent so a future edit doesn't revert it back to the unprivileged service account. --- .../supabase-admin-agent.sudoers.conf | 5 +---- ansible/tasks/internal/supabase-admin-agent.yml | 1 + 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/ansible/files/supabase_admin_agent_config/supabase-admin-agent.sudoers.conf b/ansible/files/supabase_admin_agent_config/supabase-admin-agent.sudoers.conf index 1894047dae..81a4bbd259 100644 --- a/ansible/files/supabase_admin_agent_config/supabase-admin-agent.sudoers.conf +++ b/ansible/files/supabase_admin_agent_config/supabase-admin-agent.sudoers.conf @@ -1,7 +1,4 @@ %supabase-admin-agent ALL= NOPASSWD: /usr/bin/salt-call %supabase-admin-agent ALL= NOPASSWD: /usr/bin/gpg --homedir /etc/salt/gpgkeys --import, /usr/bin/gpg --homedir /etc/salt/gpgkeys --list-secret-keys * -# Lets the salt subcommand (running as the unprivileged supabase-admin-agent -# user) reconcile pgBackRest's async-archiving config with the instance's -# billing tier on each Salt cycle (INDATA-996) — /etc/pgbackrest/conf.d is -# only writable by root/adminapi. +# supabase-admin-agent needs sudo here since /etc/pgbackrest/conf.d is root/adminapi-only, but the salt subcommand runs unprivileged (INDATA-996) %supabase-admin-agent ALL= NOPASSWD: /opt/supabase-admin-agent/supabase-admin-agent pgbackrest reconcile diff --git a/ansible/tasks/internal/supabase-admin-agent.yml b/ansible/tasks/internal/supabase-admin-agent.yml index af08e135b2..b75ec9512d 100644 --- a/ansible/tasks/internal/supabase-admin-agent.yml +++ b/ansible/tasks/internal/supabase-admin-agent.yml @@ -15,6 +15,7 @@ - name: supabase-admin-agent - config dir file: path: /opt/supabase-admin-agent + # /opt/supabase-admin-agent (dir, unpacked binary, symlink) is all root-owned so the service account can run it via sudoers but never modify it (INDATA-996) owner: root group: root state: directory From 8d6593d250bcf7d9bbb39024b95a3a549884e3e0 Mon Sep 17 00:00:00 2001 From: Douglas J Hunley Date: Fri, 7 Aug 2026 23:39:07 -0400 Subject: [PATCH 06/12] fix(ansible): correct pgbackrest conf.d permissions in sudoers comment The comment described /etc/pgbackrest/conf.d as root/adminapi-only, but setup-pgbackrest.yml creates it pgbackrest:postgres 02770 -- the agent just isn't in either group. Contradicted the PR's own accurate description of the same path. --- .../supabase-admin-agent.sudoers.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/files/supabase_admin_agent_config/supabase-admin-agent.sudoers.conf b/ansible/files/supabase_admin_agent_config/supabase-admin-agent.sudoers.conf index 81a4bbd259..1df600f61e 100644 --- a/ansible/files/supabase_admin_agent_config/supabase-admin-agent.sudoers.conf +++ b/ansible/files/supabase_admin_agent_config/supabase-admin-agent.sudoers.conf @@ -1,4 +1,4 @@ %supabase-admin-agent ALL= NOPASSWD: /usr/bin/salt-call %supabase-admin-agent ALL= NOPASSWD: /usr/bin/gpg --homedir /etc/salt/gpgkeys --import, /usr/bin/gpg --homedir /etc/salt/gpgkeys --list-secret-keys * -# supabase-admin-agent needs sudo here since /etc/pgbackrest/conf.d is root/adminapi-only, but the salt subcommand runs unprivileged (INDATA-996) +# supabase-admin-agent needs sudo here since /etc/pgbackrest/conf.d is pgbackrest:postgres 02770 and the agent is in neither group, but the salt subcommand runs unprivileged (INDATA-996) %supabase-admin-agent ALL= NOPASSWD: /opt/supabase-admin-agent/supabase-admin-agent pgbackrest reconcile From 8601dfe78d9614a2518ac8b5d65629761d4ee4df Mon Sep 17 00:00:00 2001 From: Douglas J Hunley Date: Sat, 8 Aug 2026 17:33:00 -0400 Subject: [PATCH 07/12] fix(ansible): pin explicit mode on supabase-admin-agent config dir and unpack tasks Owner/group alone left permissions umask- and tarball-derived instead of deterministic; a UMASK 027 hardening pass would yield 0750 and break the salt service, postgres archive_command, and adminapi execution paths. --- ansible/tasks/internal/supabase-admin-agent.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ansible/tasks/internal/supabase-admin-agent.yml b/ansible/tasks/internal/supabase-admin-agent.yml index 3a2d363b04..6caaa62d0b 100644 --- a/ansible/tasks/internal/supabase-admin-agent.yml +++ b/ansible/tasks/internal/supabase-admin-agent.yml @@ -18,6 +18,7 @@ # /opt/supabase-admin-agent (dir, unpacked binary, symlink) is all root-owned so the service account can run it via sudoers but never modify it (INDATA-996) owner: root group: root + mode: "0755" state: directory - name: supabase-admin-agent - gpg dir @@ -78,6 +79,7 @@ dest: /opt/supabase-admin-agent/ owner: root group: root + mode: "0755" extra_opts: - --strip-components=1 From fde63fa0698db78b705933489d41ae31eac049a9 Mon Sep 17 00:00:00 2001 From: Douglas J Hunley Date: Sat, 8 Aug 2026 17:47:16 -0400 Subject: [PATCH 08/12] fix(ansible): validate sudoers.d file syntax before install A malformed /etc/sudoers.d/supabase-admin-agent breaks sudo fleet-wide, including the adminapi entries pgBackRest restore depends on. visudo -csf rejects a bad file before it's ever installed. --- ansible/tasks/internal/supabase-admin-agent.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/ansible/tasks/internal/supabase-admin-agent.yml b/ansible/tasks/internal/supabase-admin-agent.yml index 6caaa62d0b..e21d7537ba 100644 --- a/ansible/tasks/internal/supabase-admin-agent.yml +++ b/ansible/tasks/internal/supabase-admin-agent.yml @@ -33,6 +33,7 @@ src: files/supabase_admin_agent_config/supabase-admin-agent.sudoers.conf dest: /etc/sudoers.d/supabase-admin-agent mode: "0440" + validate: "visudo -csf %s" - name: supabase-admin-agent - pgbackrest helper scripts dir file: From b235ed9ed24e4c5d7fad28c3cef7f863f6741426 Mon Sep 17 00:00:00 2001 From: Douglas J Hunley Date: Sat, 8 Aug 2026 18:18:34 -0400 Subject: [PATCH 09/12] docs(ansible): soften supabase-admin-agent trust-boundary comment The comment implied this change alone stops the agent from ever gaining root; the salt-call NOPASSWD entry above already grants that group effectively unrestricted root. Reworded to describe what this change actually closes: binary tampering ahead of the new sudo grant. --- ansible/tasks/internal/supabase-admin-agent.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/tasks/internal/supabase-admin-agent.yml b/ansible/tasks/internal/supabase-admin-agent.yml index e21d7537ba..c385ccb75d 100644 --- a/ansible/tasks/internal/supabase-admin-agent.yml +++ b/ansible/tasks/internal/supabase-admin-agent.yml @@ -15,7 +15,7 @@ - name: supabase-admin-agent - config dir file: path: /opt/supabase-admin-agent - # /opt/supabase-admin-agent (dir, unpacked binary, symlink) is all root-owned so the service account can run it via sudoers but never modify it (INDATA-996) + # /opt/supabase-admin-agent (dir, unpacked binary, symlink) is all root-owned so the pgbackrest-reconcile sudo grant below can't be used to swap the binary first; the agent group already has effectively unrestricted root via the salt-call entry, this just closes one more tampering path (INDATA-996) owner: root group: root mode: "0755" From bc96d78558bc0148613c808689d00ef0e8d7de35 Mon Sep 17 00:00:00 2001 From: Douglas J Hunley Date: Sat, 8 Aug 2026 18:20:15 -0400 Subject: [PATCH 10/12] docs(ansible): drop bare Linear issue tag from non-TODO comments Per convention, comments only cite a Linear issue when tagged as a TODO for follow-up work, not as a general reference to the ticket a change is part of. --- .../supabase-admin-agent.sudoers.conf | 2 +- ansible/tasks/internal/supabase-admin-agent.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/files/supabase_admin_agent_config/supabase-admin-agent.sudoers.conf b/ansible/files/supabase_admin_agent_config/supabase-admin-agent.sudoers.conf index 1df600f61e..a159191e26 100644 --- a/ansible/files/supabase_admin_agent_config/supabase-admin-agent.sudoers.conf +++ b/ansible/files/supabase_admin_agent_config/supabase-admin-agent.sudoers.conf @@ -1,4 +1,4 @@ %supabase-admin-agent ALL= NOPASSWD: /usr/bin/salt-call %supabase-admin-agent ALL= NOPASSWD: /usr/bin/gpg --homedir /etc/salt/gpgkeys --import, /usr/bin/gpg --homedir /etc/salt/gpgkeys --list-secret-keys * -# supabase-admin-agent needs sudo here since /etc/pgbackrest/conf.d is pgbackrest:postgres 02770 and the agent is in neither group, but the salt subcommand runs unprivileged (INDATA-996) +# supabase-admin-agent needs sudo here since /etc/pgbackrest/conf.d is pgbackrest:postgres 02770 and the agent is in neither group, but the salt subcommand runs unprivileged %supabase-admin-agent ALL= NOPASSWD: /opt/supabase-admin-agent/supabase-admin-agent pgbackrest reconcile diff --git a/ansible/tasks/internal/supabase-admin-agent.yml b/ansible/tasks/internal/supabase-admin-agent.yml index c385ccb75d..31887df6cc 100644 --- a/ansible/tasks/internal/supabase-admin-agent.yml +++ b/ansible/tasks/internal/supabase-admin-agent.yml @@ -15,7 +15,7 @@ - name: supabase-admin-agent - config dir file: path: /opt/supabase-admin-agent - # /opt/supabase-admin-agent (dir, unpacked binary, symlink) is all root-owned so the pgbackrest-reconcile sudo grant below can't be used to swap the binary first; the agent group already has effectively unrestricted root via the salt-call entry, this just closes one more tampering path (INDATA-996) + # /opt/supabase-admin-agent (dir, unpacked binary, symlink) is all root-owned so the pgbackrest-reconcile sudo grant below can't be used to swap the binary first; the agent group already has effectively unrestricted root via the salt-call entry, this just closes one more tampering path owner: root group: root mode: "0755" From 9ba8836ca5456fb7b29273e9225318475b8fc249 Mon Sep 17 00:00:00 2001 From: Douglas J Hunley Date: Sat, 8 Aug 2026 18:22:26 -0400 Subject: [PATCH 11/12] refactor(ansible): drop no-op owner/group from symlink task state: link with the file module's default follow: yes applies owner/group to the link target, not the symlink itself, and the kernel ignores symlink ownership for access control regardless. The target is already root-owned by the unpack task above; these two lines re-asserted nothing. --- ansible/tasks/internal/supabase-admin-agent.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/ansible/tasks/internal/supabase-admin-agent.yml b/ansible/tasks/internal/supabase-admin-agent.yml index 31887df6cc..af37e53a8f 100644 --- a/ansible/tasks/internal/supabase-admin-agent.yml +++ b/ansible/tasks/internal/supabase-admin-agent.yml @@ -89,8 +89,6 @@ path: /opt/supabase-admin-agent/supabase-admin-agent src: "/opt/supabase-admin-agent/supabase-admin-agent-linux-{{ platform }}" state: link - owner: root - group: root mode: "0755" force: yes From 01304875996b54e9e34f7e9849748af4460c95c5 Mon Sep 17 00:00:00 2001 From: Douglas J Hunley Date: Sat, 8 Aug 2026 18:32:03 -0400 Subject: [PATCH 12/12] docs(ansible): clarify subject of sudoers comment's contrast clause Previous wording's trailing clause read as describing the entry it was attached to; reworded so 'the salt subcommand above' unambiguously refers to the entry on the preceding line. --- .../supabase-admin-agent.sudoers.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/files/supabase_admin_agent_config/supabase-admin-agent.sudoers.conf b/ansible/files/supabase_admin_agent_config/supabase-admin-agent.sudoers.conf index a159191e26..183e246b95 100644 --- a/ansible/files/supabase_admin_agent_config/supabase-admin-agent.sudoers.conf +++ b/ansible/files/supabase_admin_agent_config/supabase-admin-agent.sudoers.conf @@ -1,4 +1,4 @@ %supabase-admin-agent ALL= NOPASSWD: /usr/bin/salt-call %supabase-admin-agent ALL= NOPASSWD: /usr/bin/gpg --homedir /etc/salt/gpgkeys --import, /usr/bin/gpg --homedir /etc/salt/gpgkeys --list-secret-keys * -# supabase-admin-agent needs sudo here since /etc/pgbackrest/conf.d is pgbackrest:postgres 02770 and the agent is in neither group, but the salt subcommand runs unprivileged +# pgbackrest reconcile writes /etc/pgbackrest/conf.d (pgbackrest:postgres 02770) and the agent is in neither group, so this subcommand needs root; the salt subcommand above stays unprivileged %supabase-admin-agent ALL= NOPASSWD: /opt/supabase-admin-agent/supabase-admin-agent pgbackrest reconcile