Add package definitions, build script, and test suite

.github/workflows/test.yml

@@ -0,0 +1,78 @@
+name: Test
+
+on:
+  pull_request:
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
+      - name: shellcheck
+        run: shellcheck -x scripts/build-deb.sh tests/run-tests.sh tests/lib.sh tests/container-prep.sh tests/build-legacy-deb.sh tests/scenarios/*.sh
+
+  go:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
+      - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0
+        with:
+          go-version: "1.26"
+      - run: go vet ./...
+      - run: go test ./...
+
+  build-packages:
+    name: Build all packages
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
+      - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0
+        with:
+          go-version: "1.26"
+      - name: Build stub binaries
+        run: |
+          for arch in amd64 arm64; do
+            CGO_ENABLED=0 GOOS=linux GOARCH="$arch" go build -trimpath -ldflags '-s -w' -o "dist/stub-$arch" ./tests/stub
+          done
+      - name: Build every package definition
+        run: |
+          for env in packages/*/package.env; do
+            project=$(basename "$(dirname "$env")")
+            for arch in amd64 arm64; do
+              ./scripts/build-deb.sh --project "$project" --version 0.0.1 --arch "$arch" --binary "dist/stub-$arch" --output dist
+            done
+          done
+          ls -l dist/*.deb
+      - name: Assert package contents are owned by root
+        run: |
+          # the runner is not root, so this catches a dropped
+          # --root-owner-group in build-deb.sh that other gates miss because
+          # they build debs as root inside the test containers
+          for deb in dist/*.deb; do
+            if dpkg-deb -c "$deb" | awk '{print $2}' | grep -vqx 'root/root'; then
+              echo "non root owner in $deb" >&2
+              dpkg-deb -c "$deb" >&2
+              exit 1
+            fi
+          done
+
+  test-packages:
+    name: Test packages
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        # oldest and newest of each distro bracket the systemd and dpkg
+        # version spread
+        image:
+          - debian:bullseye
+          - debian:trixie
+          - ubuntu:jammy
+          - ubuntu:questing
+    steps:
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
+      - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0
+        with:
+          go-version: "1.26"
+      - name: Run packaging tests
+        run: tests/run-tests.sh --image ${{ matrix.image }} --project hostd

.gitignore

@@ -1 +1,3 @@
-.DS_Store
+.DS_Store
+dist/
+*.deb

packages/hostd/package.env

@@ -0,0 +1,5 @@
+# package settings for hostd, read by scripts/build-deb.sh
+PKG_NAME=hostd
+PKG_DESCRIPTION="host daemon for the Sia network"
+PKG_LONG_DESCRIPTION="hostd is the next generation Sia host daemon. It connects storage to the Sia network and manages contracts, pricing, and sector storage."
+PKG_HOMEPAGE=https://github.com/SiaFoundation/hostd

packages/renterd/package.env

@@ -0,0 +1,5 @@
+# package settings for renterd, read by scripts/build-deb.sh
+PKG_NAME=renterd
+PKG_DESCRIPTION="renter daemon for the Sia network"
+PKG_LONG_DESCRIPTION="renterd is the Sia renter daemon. It forms contracts with hosts on the Sia network and stores data with erasure coded redundancy."
+PKG_HOMEPAGE=https://github.com/SiaFoundation/renterd

packages/s3d/package.env

@@ -0,0 +1,7 @@
+# package settings for s3d, read by scripts/build-deb.sh
+PKG_NAME=s3d
+# debian package name. binary, service, and data dir still use PKG_NAME
+PKG_DEBIAN_NAME=sia-s3d
+PKG_DESCRIPTION="S3-compatible gateway for the Sia network"
+PKG_LONG_DESCRIPTION="s3d translates the AWS S3 API to the Sia storage network. Applications that speak S3 can store data on Sia without modification."
+PKG_HOMEPAGE=https://github.com/SiaFoundation/s3d

packages/walletd/package.env

@@ -0,0 +1,5 @@
+# package settings for walletd, read by scripts/build-deb.sh
+PKG_NAME=walletd
+PKG_DESCRIPTION="wallet daemon for the Sia network"
+PKG_LONG_DESCRIPTION="walletd is the Sia wallet daemon. It manages addresses and transactions on the Sia network and serves a wallet API."
+PKG_HOMEPAGE=https://github.com/SiaFoundation/walletd

scripts/build-deb.sh

@@ -0,0 +1,132 @@
+#!/usr/bin/env bash
+# builds a .deb for one Sia daemon from an already-built linux binary.
+# package settings come from packages/<project>/package.env.
+# the workflow and tests both use this, so they build the same thing.
+set -euo pipefail
+
+usage() {
+    echo "usage: $0 --project <name> --version <version> --arch <amd64|arm64> --binary <path> --output <dir>" >&2
+    exit 1
+}
+
+PROJECT='' VERSION='' ARCH='' BINARY='' OUTPUT=''
+while [ $# -gt 0 ]; do
+    case "$1" in
+    --project) PROJECT=$2; shift 2 ;;
+    --version) VERSION=$2; shift 2 ;;
+    --arch) ARCH=$2; shift 2 ;;
+    --binary) BINARY=$2; shift 2 ;;
+    --output) OUTPUT=$2; shift 2 ;;
+    *) usage ;;
+    esac
+done
+if [ -z "$PROJECT" ] || [ -z "$VERSION" ] || [ -z "$ARCH" ] || [ -z "$BINARY" ] || [ -z "$OUTPUT" ]; then
+    usage
+fi
+
+case "$ARCH" in
+amd64|arm64) ;;
+*) echo "unsupported architecture: $ARCH" >&2; exit 1 ;;
+esac
+[ -f "$BINARY" ] || { echo "binary not found: $BINARY" >&2; exit 1; }
+
+ROOT=$(cd "$(dirname "$0")/.." && pwd)
+PKG_DIR=$ROOT/packages/$PROJECT
+TEMPLATES=$ROOT/scripts/templates
+
+[ -f "$PKG_DIR/package.env" ] || { echo "missing $PKG_DIR/package.env, add a package definition first" >&2; exit 1; }
+# shellcheck source=/dev/null
+. "$PKG_DIR/package.env"
+: "${PKG_NAME:?package.env must set PKG_NAME}"
+: "${PKG_DESCRIPTION:?package.env must set PKG_DESCRIPTION}"
+: "${PKG_LONG_DESCRIPTION:?package.env must set PKG_LONG_DESCRIPTION}"
+: "${PKG_HOMEPAGE:?package.env must set PKG_HOMEPAGE}"
+[ "$PKG_NAME" = "$PROJECT" ] || { echo "PKG_NAME '$PKG_NAME' does not match project '$PROJECT'" >&2; exit 1; }
+# apt package name.
+# the installed binary, service, and data dir still use PKG_NAME.
+# defaults to PKG_NAME unless package.env overrides it.
+: "${PKG_DEBIAN_NAME:=$PKG_NAME}"
+
+# preinst needs hashes for old stock units.
+# ignore Description because the old workflow used free text there.
+# the unit text lives in one shared file so tests and packages use the same bytes.
+# shellcheck source=scripts/legacy-unit.sh
+. "$ROOT/scripts/legacy-unit.sh"
+LEGACY_MD5_A=$(legacy_unit "$PKG_NAME" "" a | md5sum | cut -d' ' -f1)
+LEGACY_MD5_B=$(legacy_unit "$PKG_NAME" "" b | md5sum | cut -d' ' -f1)
+PKG_LEGACY_MD5S="\"$LEGACY_MD5_A\"|\"$LEGACY_MD5_B\""
+
+# control files need wrapped long descriptions.
+# continuation lines start with a space; empty lines become ".".
+PKG_LONG_DESCRIPTION=$(printf '%s\n' "$PKG_LONG_DESCRIPTION" | fold -s -w 76 | sed -e 's/[[:space:]]*$//' -e 's/^$/./' -e 's/^/ /')
+
+# quote replacements so bash treats &, backslashes, and other chars as plain text.
+render() {
+    local content
+    content=$(cat "$1")
+    content=${content//@PKG_NAME@/"$PKG_NAME"}
+    content=${content//@PKG_DEBIAN_NAME@/"$PKG_DEBIAN_NAME"}
+    content=${content//@PKG_VERSION@/"$VERSION"}
+    content=${content//@PKG_ARCH@/"$ARCH"}
+    content=${content//@PKG_DESCRIPTION@/"$PKG_DESCRIPTION"}
+    content=${content//@PKG_LONG_DESCRIPTION@/"$PKG_LONG_DESCRIPTION"}
+    content=${content//@PKG_HOMEPAGE@/"$PKG_HOMEPAGE"}
+    content=${content//@PKG_INSTALLED_SIZE@/"${PKG_INSTALLED_SIZE:-0}"}
+    content=${content//@PKG_LEGACY_MD5S@/"$PKG_LEGACY_MD5S"}
+    printf '%s\n' "$content"
+}
+
+STAGE=$(mktemp -d)
+trap 'rm -rf "$STAGE"' EXIT
+chmod 755 "$STAGE"
+
+# payload
+install -Dm755 "$BINARY" "$STAGE/usr/bin/$PKG_NAME"
+
+# a package can override this by adding packages/<p>/<p>.service.
+install -dm755 "$STAGE/usr/lib/systemd/system"
+if [ -f "$PKG_DIR/$PKG_NAME.service" ]; then
+    install -m644 "$PKG_DIR/$PKG_NAME.service" "$STAGE/usr/lib/systemd/system/$PKG_NAME.service"
+else
+    render "$TEMPLATES/service.tmpl" > "$STAGE/usr/lib/systemd/system/$PKG_NAME.service"
+    chmod 644 "$STAGE/usr/lib/systemd/system/$PKG_NAME.service"
+fi
+
+# the daemon owns config via "<name> config".
+# the package ships no config, so dpkg never overwrites it.
+install -dm755 "$STAGE/var/lib/$PKG_NAME"
+
+install -dm755 "$STAGE/usr/share/doc/$PKG_DEBIAN_NAME"
+render "$TEMPLATES/copyright.tmpl" > "$STAGE/usr/share/doc/$PKG_DEBIAN_NAME/copyright"
+chmod 644 "$STAGE/usr/share/doc/$PKG_DEBIAN_NAME/copyright"
+# dpkg treats a version containing a hyphen as non native, which must name its
+# changelog changelog.Debian.gz; versions without one use changelog.gz.
+case "$VERSION" in
+*-*) CHANGELOG=changelog.Debian.gz ;;
+*)   CHANGELOG=changelog.gz ;;
+esac
+# honor SOURCE_DATE_EPOCH so identical inputs produce a byte identical package
+CHANGELOG_DATE=$(date -R -u -d "@${SOURCE_DATE_EPOCH:-$(date +%s)}")
+{
+    printf '%s (%s) stable; urgency=medium\n\n' "$PKG_DEBIAN_NAME" "$VERSION"
+    printf '  * Packaged upstream release %s.\n\n' "$VERSION"
+    printf ' -- The Sia Foundation <hello@sia.tech>  %s\n' "$CHANGELOG_DATE"
+} | gzip -9n > "$STAGE/usr/share/doc/$PKG_DEBIAN_NAME/$CHANGELOG"
+chmod 644 "$STAGE/usr/share/doc/$PKG_DEBIAN_NAME/$CHANGELOG"
+
+# control
+PKG_INSTALLED_SIZE=$(du -sk "$STAGE" | cut -f1)
+install -dm755 "$STAGE/DEBIAN"
+render "$TEMPLATES/control.tmpl" > "$STAGE/DEBIAN/control"
+for script in preinst postinst prerm postrm; do
+    render "$TEMPLATES/$script.tmpl" > "$STAGE/DEBIAN/$script"
+    chmod 755 "$STAGE/DEBIAN/$script"
+done
+(cd "$STAGE" && find . -type f -not -path './DEBIAN/*' | LC_ALL=C sort | sed 's|^\./||' | xargs md5sum > DEBIAN/md5sums)
+chmod 644 "$STAGE/DEBIAN/md5sums" "$STAGE/DEBIAN/control"
+
+# use xz so bullseye can install the package.
+# newer dpkg may default to zstd, which bullseye cannot unpack.
+mkdir -p "$OUTPUT"
+dpkg-deb --root-owner-group -Zxz --build "$STAGE" "$OUTPUT/${PKG_DEBIAN_NAME}_${VERSION}_${ARCH}.deb" > /dev/null
+echo "built $OUTPUT/${PKG_DEBIAN_NAME}_${VERSION}_${ARCH}.deb"

scripts/legacy-unit.sh

@@ -0,0 +1,28 @@
+# shellcheck shell=bash
+# one copy of the old systemd unit.
+# build-deb.sh hashes it for migrations.
+# tests/build-legacy-deb.sh writes it into the fake old package.
+# keeping one copy means the hash and test package cannot disagree.
+#
+# empty desc means no Description line.
+# preinst ignores Description because the old workflow got it from free text.
+# variant a is older; variant b adds TimeoutStopSec.
+legacy_unit() {
+    local name=$1 desc=$2 variant=$3
+    printf '%s\n' '[Unit]'
+    [ -n "$desc" ] && printf 'Description=%s\n' "$desc"
+    printf '%s\n' \
+        '' \
+        'After=network.target' \
+        '[Service]' \
+        "ExecStart=/usr/bin/$name" \
+        "WorkingDirectory=/var/lib/$name" \
+        'Restart=always' \
+        'RestartSec=15'
+    [ "$variant" = b ] && printf 'TimeoutStopSec=120\n'
+    printf '%s\n' \
+        '' \
+        '[Install]' \
+        'WantedBy=multi-user.target' \
+        "Alias=$name.service"
+}

scripts/templates/control.tmpl

@@ -0,0 +1,11 @@
+Package: @PKG_DEBIAN_NAME@
+Version: @PKG_VERSION@
+Architecture: @PKG_ARCH@
+Maintainer: The Sia Foundation <hello@sia.tech>
+Installed-Size: @PKG_INSTALLED_SIZE@
+Depends: ca-certificates
+Section: net
+Priority: optional
+Homepage: @PKG_HOMEPAGE@
+Description: @PKG_DESCRIPTION@
+@PKG_LONG_DESCRIPTION@

scripts/templates/copyright.tmpl

@@ -0,0 +1,26 @@
+Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
+Upstream-Name: @PKG_NAME@
+Source: @PKG_HOMEPAGE@
+
+Files: *
+Copyright: 2024-2026 The Sia Foundation
+License: MIT
+
+License: MIT
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+ .
+ The above copyright notice and this permission notice shall be included in
+ all copies or substantial portions of the Software.
+ .
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ THE SOFTWARE.

scripts/templates/postinst.tmpl

@@ -0,0 +1,66 @@
+#!/bin/sh
+set -e
+
+PKG=@PKG_NAME@
+UNIT=@PKG_NAME@.service
+LEGACY_UNIT=/etc/systemd/system/$UNIT
+MARKER=/run/sia-linux/$PKG.legacy-upgrade
+
+# config exists if the daemon wrote it in either supported place.
+config_present() {
+    [ -f "/var/lib/$PKG/$PKG.yml" ] || [ -f "/etc/$PKG/$PKG.yml" ]
+}
+
+case "$1" in
+configure)
+    # put back a user-edited old unit.
+    # keeping it in /etc makes it override our packaged unit.
+    if [ -e "$LEGACY_UNIT.dpkg-bak" ] && [ ! -e "$LEGACY_UNIT" ]; then
+        mv "$LEGACY_UNIT.dpkg-bak" "$LEGACY_UNIT"
+        echo "$PKG: preserved your modified $LEGACY_UNIT" >&2
+        echo "$PKG: it overrides the packaged unit in /usr/lib/systemd/system" >&2
+        echo "$PKG: consider moving your changes to a drop-in via 'systemctl edit $PKG'" >&2
+        echo "$PKG: and removing $LEGACY_UNIT to use the packaged unit" >&2
+    fi
+
+    if [ -d /run/systemd/system ]; then
+        systemctl daemon-reload || true
+
+        if [ -e "$MARKER" ]; then
+            # first upgrade from the old layout.
+            # we cannot know if it used to run, so only start it when config exists.
+            if config_present; then
+                systemctl enable "$UNIT" 2>/dev/null || true
+                # old systemd cannot enable this preserved unit.
+                # create the wants link ourselves.
+                if ! systemctl is-enabled --quiet "$UNIT" 2>/dev/null && [ -e "$LEGACY_UNIT" ]; then
+                    mkdir -p /etc/systemd/system/multi-user.target.wants || true
+                    ln -sf "$LEGACY_UNIT" "/etc/systemd/system/multi-user.target.wants/$UNIT" || true
+                    systemctl daemon-reload || true
+                fi
+                systemctl restart "$UNIT" || true
+            else
+                echo "$PKG: service was not restarted because no config was found; run 'systemctl enable --now $PKG' if it should be running" >&2
+            fi
+        elif [ -n "$2" ]; then
+            # normal upgrade or reinstall: restart if running, do not change enablement.
+            systemctl try-restart "$UNIT" || true
+        fi
+        # fresh installs stay off until the admin creates config.
+    fi
+    rm -f "$MARKER"
+
+    # no config on a fresh install: show the setup hint.
+    if [ -z "${2:-}" ] && ! config_present; then
+        echo "$PKG: run '$PKG config' to configure it, then 'systemctl enable --now $PKG'" >&2
+    fi
+
+    if [ -f "/var/lib/$PKG/$PKG.yml" ]; then
+        echo "$PKG: note: /var/lib/$PKG/$PKG.yml exists and takes precedence over /etc/$PKG/$PKG.yml" >&2
+    fi
+    ;;
+abort-upgrade|abort-remove|abort-deconfigure)
+    ;;
+esac
+
+exit 0