From fc3b306bb218ab7fecba8cc9b81e0aef337964c7 Mon Sep 17 00:00:00 2001
From: Manuel Mendez <i@m.mmlb.dev>
Date: Tue, 18 Aug 2026 11:45:40 -0400
Subject: [PATCH] DNM DEBUG DISK USAGE

---
 ansible/tasks/stage2-setup-postgres.yml       | 27 +++++++++++++++++--
 .../scripts/surrogate-bootstrap-nix.sh        |  6 +++++
 scripts/nix-provision.sh                      | 15 +++++++++++
 3 files changed, 46 insertions(+), 2 deletions(-)

diff --git a/ansible/tasks/stage2-setup-postgres.yml b/ansible/tasks/stage2-setup-postgres.yml
index 3019fa829..7bebc5805 100644
--- a/ansible/tasks/stage2-setup-postgres.yml
+++ b/ansible/tasks/stage2-setup-postgres.yml
@@ -62,9 +62,18 @@
   when: stage2
   become: true
   block:
+    - name: Check disk usage before nix based installs
+      ansible.builtin.command: df -h /
+      register: df_before_nix_install
+      changed_when: false
+
+    - name: Show disk usage before nix based installs
+      ansible.builtin.debug:
+        var: df_before_nix_install.stdout_lines
+
     - name: Install packages from nix binary cache
       ansible.builtin.shell: |
-        sudo -u postgres bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install github:supabase/postgres/{{ git_commit_sha }}#{{ nix_item }}"
+        sudo -u postgres bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install github:supabase/postgres/{{ git_commit_sha }}#{{ nix_item }}; rc=$?; df -h / >&2; exit $rc"
       loop:
         - "{{ psql_version }}/bin"
         - pg_prove
@@ -73,10 +82,24 @@
         - "{{ postgresql_version }}_src"
       loop_control:
         loop_var: 'nix_item'
+      register: nix_install_result
+      ignore_errors: true
+
+    - name: Show df output for each nix install attempt
+      ansible.builtin.debug:
+        msg: "{{ item.stderr_lines }}"
+      loop: "{{ nix_install_result.results }}"
+      loop_control:
+        label: "{{ item.nix_item }}"
+
+    - name: Fail if any nix install failed
+      ansible.builtin.fail:
+        msg: "nix profile install failed for: {{ nix_install_result.results | selectattr('failed', 'defined') | selectattr('failed') | map(attribute='nix_item') | list }}"
+      when: nix_install_result is failed
 
     - name: Install supascan for baseline validation
       ansible.builtin.shell: |
-        sudo -u ubuntu bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install github:supabase/postgres/{{ git_commit_sha }}#supascan"
+        sudo -u ubuntu bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install github:supabase/postgres/{{ git_commit_sha }}#supascan && df -h / >&2"
 
     - name: nix collect garbage after supascan install
       ansible.builtin.shell:
diff --git a/ebssurrogate/scripts/surrogate-bootstrap-nix.sh b/ebssurrogate/scripts/surrogate-bootstrap-nix.sh
index f2cba1b04..06b22a83c 100755
--- a/ebssurrogate/scripts/surrogate-bootstrap-nix.sh
+++ b/ebssurrogate/scripts/surrogate-bootstrap-nix.sh
@@ -448,8 +448,14 @@ format_build_partition
 #pull_docker
 setup_chroot_environment
 #download_ccache
+df /mnt
+df -h /mnt
+du -x -h --max-depth=2 /mnt | sort -rh | head -30
 execute_playbook
 update_systemd_services
 #upload_ccache
 clean_system
+df /mnt
+df -h /mnt
+du -x -h --max-depth=2 /mnt | sort -rh | head -30
 umount_reset_mappings
diff --git a/scripts/nix-provision.sh b/scripts/nix-provision.sh
index 8a69f7156..f1560e6e4 100644
--- a/scripts/nix-provision.sh
+++ b/scripts/nix-provision.sh
@@ -66,7 +66,22 @@ function cleanup_packages {
 	# sudo add-apt-repository --yes --remove ppa:ansible/ansible
 }
 
+function show_disk_usage {
+	echo "disk usage post $1:"
+	sudo df /
+	sudo df -h /
+	sudo du -x -h --max-depth=2 / | sort -rh | head -30
+}
+
+# Snapshot disk usage even when a step below fails; errexit would otherwise skip
+# the remaining show_disk_usage calls and we'd lose the state we want to debug.
+trap 'show_disk_usage EXIT' EXIT
+
+show_disk_usage "start"
 install_packages
+show_disk_usage "install_packages"
 install_nix
+show_disk_usage "install_nix"
 execute_stage2_playbook
+show_disk_usage "ansible"
 cleanup_packages