Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .github/workflows/release-chocolatey-channel.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,9 @@ on:
expected_source_sha: {required: true, type: string}
release_id: {required: true, type: string}
release_ref: {required: true, type: string}
secrets:
CHOCOLATEY_API_KEY:
required: false

permissions: {}

Expand Down
21 changes: 21 additions & 0 deletions .github/workflows/release-linux-repository-build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,32 @@ on:
expected_source_sha: {required: true, type: string}
release_id: {required: true, type: string}
release_ref: {required: true, type: string}
secrets:
RMUX_APT_GPG_PRIVATE_KEY:
required: false
RMUX_APT_GPG_KEY:
required: false
RMUX_RPM_GPG_PRIVATE_KEY:
required: false
RMUX_RPM_GPG_KEY:
required: false
RMUX_RPM_REPO_GPG_PRIVATE_KEY:
required: false
RMUX_RPM_REPO_GPG_KEY:
required: false
outputs:
repository_artifact_id:
value: ${{ jobs.build.outputs.repository_artifact_id }}
repository_artifact_digest:
value: ${{ jobs.build.outputs.repository_artifact_digest }}
workflow_dispatch:
inputs:
receipt_run_id: {required: true, type: string}
payload_artifact_id: {required: true, type: string}
payload_artifact_digest: {required: true, type: string}
expected_source_sha: {required: true, type: string}
release_id: {required: true, type: string}
release_ref: {required: true, type: string}

permissions: {}

Expand Down
3 changes: 3 additions & 0 deletions .github/workflows/release-linux-repository-publish.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,9 @@ on:
expected_source_sha: {required: true, type: string}
release_id: {required: true, type: string}
release_ref: {required: true, type: string}
secrets:
RMUX_DOWNSTREAM_APP_PRIVATE_KEY:
required: false

permissions: {}

Expand Down
3 changes: 3 additions & 0 deletions .github/workflows/release-owned-repository-channel.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,9 @@ on:
expected_source_sha: {required: true, type: string}
release_id: {required: true, type: string}
release_ref: {required: true, type: string}
secrets:
RMUX_DOWNSTREAM_APP_PRIVATE_KEY:
required: false

permissions: {}

Expand Down
3 changes: 3 additions & 0 deletions .github/workflows/release-snap-channel.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,9 @@ on:
expected_source_sha: {required: true, type: string}
release_id: {required: true, type: string}
release_ref: {required: true, type: string}
secrets:
SNAPCRAFT_STORE_CREDENTIALS:
required: false

permissions: {}

Expand Down
55 changes: 49 additions & 6 deletions scripts/release/downstream_workflow_contract.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,13 +64,29 @@ def _validate_reusable_workflow(path: Path, *, require_repository_guard: bool) -
text = path.read_text(encoding="utf-8")
if "on:\n workflow_call:" not in text or "permissions: {}" not in text:
raise ValueError(f"{path.name} must remain reusable and default-deny")
allows_signing_recovery = path.name == "release-linux-repository-build.yml"
for forbidden in (
"\n push:",
"\n workflow_dispatch:",
"larger-runner",
):
if forbidden in text:
raise ValueError(f"{path.name} contains forbidden value {forbidden}")
if allows_signing_recovery:
dispatch = text.split("\n workflow_dispatch:\n", 1)
if len(dispatch) != 2 or "\npermissions: {}\n" not in dispatch[1]:
raise ValueError("Linux repository signer lost its recovery trigger")
for name in (
"expected_source_sha",
"payload_artifact_digest",
"payload_artifact_id",
"receipt_run_id",
"release_id",
"release_ref",
):
if f" {name}:" not in dispatch[1]:
raise ValueError(f"Linux repository recovery lost input {name}")
elif "\n workflow_dispatch:" in text:
raise ValueError(f"{path.name} gained a mutation-capable dispatch trigger")
if "runs-on: self-hosted" in text or "\n - self-hosted" in text:
raise ValueError(f"{path.name} gained a self-hosted runner label")
if require_repository_guard and (
Expand All @@ -80,6 +96,35 @@ def _validate_reusable_workflow(path: Path, *, require_repository_guard: bool) -
raise ValueError(f"{path.name} does not reject external callers")


def _validate_worker_secret_declarations(root: Path) -> None:
workflows = root / ".github/workflows"
expected = {
"release-chocolatey-channel.yml": ("CHOCOLATEY_API_KEY",),
"release-linux-repository-build.yml": (
"RMUX_APT_GPG_PRIVATE_KEY",
"RMUX_APT_GPG_KEY",
"RMUX_RPM_GPG_PRIVATE_KEY",
"RMUX_RPM_GPG_KEY",
"RMUX_RPM_REPO_GPG_PRIVATE_KEY",
"RMUX_RPM_REPO_GPG_KEY",
),
"release-linux-repository-publish.yml": ("RMUX_DOWNSTREAM_APP_PRIVATE_KEY",),
"release-owned-repository-channel.yml": ("RMUX_DOWNSTREAM_APP_PRIVATE_KEY",),
"release-snap-channel.yml": ("SNAPCRAFT_STORE_CREDENTIALS",),
}
for name, secret_names in expected.items():
text = (workflows / name).read_text(encoding="utf-8")
call = text.split("\n workflow_dispatch:", 1)[0]
if "\n secrets:\n" not in call:
raise ValueError(f"{name} does not declare protected environment secrets")
for secret_name in secret_names:
declaration = f" {secret_name}:\n required: false"
if call.count(declaration) != 1:
raise ValueError(f"{name} lost secret declaration {secret_name}")
if text.count(f"secrets.{secret_name}") < 1:
raise ValueError(f"{name} no longer consumes secret {secret_name}")


def _calls_workflow(line: str, workflow: str) -> bool:
normalized = line.strip().removeprefix("- ").strip()
if not normalized.startswith("uses:"):
Expand Down Expand Up @@ -138,8 +183,7 @@ def _validate_receipt_origin(main: str) -> None:
"${{ runner.temp }}/rmux-downstream/publication-receipt-predicate.json"
)
nested_predicate = (
"${{ runner.temp }}/rmux-downstream/receipt/"
"publication-receipt-predicate.json"
"${{ runner.temp }}/rmux-downstream/receipt/publication-receipt-predicate.json"
)
if main.count(flat_predicate) != 1 or nested_predicate in main:
raise ValueError("downstream plan artifact must flatten its receipt predicate")
Expand Down Expand Up @@ -419,13 +463,12 @@ def validate_downstream_workflows(root: Path) -> None:
)
for path in _worker_paths(root):
_validate_reusable_workflow(path, require_repository_guard=False)
_validate_worker_secret_declarations(root)
_validate_callers(root, paths)
main = paths[0].read_text(encoding="utf-8")
_validate_receipt_origin(main)
_validate_channel_orchestration(main)
_validate_payload_prepare(
root / ".github/workflows/release-downstream-prepare.yml"
)
_validate_payload_prepare(root / ".github/workflows/release-downstream-prepare.yml")
_validate_channel_result_action(_channel_result_action_path(root))
for path in paths[1:]:
_validate_retry(path)
Expand Down
69 changes: 62 additions & 7 deletions scripts/release/publish-crate-set.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

import argparse
import hashlib
import json
import os
import re
import subprocess
Expand Down Expand Up @@ -50,7 +51,38 @@ def registry_url(name: str, version: str, *, download: bool = False) -> str:
return f"https://crates.io/api/v1/crates/{name}/{version}{suffix}"


def registry_version_exists(name: str, version: str) -> bool:
request = urllib.request.Request(
registry_url(name, version),
headers={"User-Agent": "rmux-release-writer/1 (security@rmux.io)"},
)
try:
with urllib.request.urlopen(request, timeout=30) as response:
data = response.read(2 * 1024 * 1024 + 1)
except urllib.error.HTTPError as error:
if error.code == 404:
return False
raise ValueError(f"crates.io lookup failed with HTTP {error.code}") from error
if len(data) > 2 * 1024 * 1024:
raise ValueError("crates.io metadata exceeds the release size limit")
try:
metadata = json.loads(data)
except (UnicodeDecodeError, json.JSONDecodeError) as error:
raise ValueError("crates.io metadata is invalid") from error
observed = metadata.get("version") if isinstance(metadata, dict) else None
if (
not isinstance(observed, dict)
or observed.get("crate") != name
or observed.get("num") != version
or observed.get("dl_path") != f"/api/v1/crates/{name}/{version}/download"
):
raise ValueError("crates.io metadata identity differs")
return True


def registry_bytes(name: str, version: str) -> bytes | None:
if not registry_version_exists(name, version):
return None
request = urllib.request.Request(
registry_url(name, version, download=True),
headers={"User-Agent": "rmux-release-writer/1 (security@rmux.io)"},
Expand All @@ -59,9 +91,7 @@ def registry_bytes(name: str, version: str) -> bytes | None:
with urllib.request.urlopen(request, timeout=30) as response:
data = response.read(16 * 1024 * 1024 + 1)
except urllib.error.HTTPError as error:
if error.code == 404:
return None
raise ValueError(f"crates.io lookup failed with HTTP {error.code}") from error
raise ValueError(f"crates.io download failed with HTTP {error.code}") from error
if len(data) > 16 * 1024 * 1024:
raise ValueError("crates.io package exceeds the release size limit")
return data
Expand All @@ -86,6 +116,27 @@ def run_cargo(args: list[str], *, token: str, target_dir: Path) -> None:
raise ValueError(f"cargo {' '.join(args[:2])} failed: {detail}")


def generated_package_paths(target_dir: Path, filename: str) -> tuple[Path, Path]:
package_dir = target_dir / "package"
return package_dir / filename, package_dir / "tmp-crate" / filename


def remove_generated_package(target_dir: Path, filename: str) -> None:
for path in generated_package_paths(target_dir, filename):
path.unlink(missing_ok=True)


def generated_package(target_dir: Path, filename: str) -> Path:
files = [
path
for path in generated_package_paths(target_dir, filename)
if path.is_file() and not path.is_symlink()
]
if len(files) != 1:
raise ValueError(f"Cargo generated an unexpected package set: {filename}")
return files[0]


def wait_for_exact(name: str, version: str, expected: Path) -> None:
expected_hash = file_hash(expected)
for attempt in range(19):
Expand Down Expand Up @@ -125,21 +176,25 @@ def execute(args: argparse.Namespace) -> None:
if hashlib.sha256(existing).hexdigest() != package["sha256"]:
raise ValueError(f"existing crates.io bytes differ: {name}")
continue
generated = cargo_target / "package" / package["file"]
generated.unlink(missing_ok=True)
remove_generated_package(cargo_target, package["file"])
run_cargo(
["publish", "--dry-run", "--locked", "--package", name],
token=token,
target_dir=cargo_target,
)
if not generated.is_file() or file_hash(generated) != file_hash(canonical):
if file_hash(generated_package(cargo_target, package["file"])) != file_hash(
canonical
):
raise ValueError(f"Cargo package bytes differ from the candidate: {name}")
remove_generated_package(cargo_target, package["file"])
run_cargo(
["publish", "--locked", "--package", name],
token=token,
target_dir=cargo_target,
)
if not generated.is_file() or file_hash(generated) != file_hash(canonical):
if file_hash(generated_package(cargo_target, package["file"])) != file_hash(
canonical
):
raise ValueError(f"published Cargo package bytes changed: {name}")
mutated = True
wait_for_exact(name, version, canonical)
Expand Down
79 changes: 79 additions & 0 deletions tests/release_downstream_workflows_spec.rs
Original file line number Diff line number Diff line change
Expand Up @@ -269,6 +269,85 @@ fn downstream_writers_keep_the_python_310_runtime_contract() {
}
}

#[cfg(unix)]
#[test]
fn crates_writer_distinguishes_missing_versions_from_download_denials() {
let fixture = r#"
import importlib.util
import pathlib
import sys
import tempfile
import urllib.error

root = pathlib.Path.cwd()
scripts = root / "scripts" / "release"
sys.path.insert(0, str(scripts))
spec = importlib.util.spec_from_file_location(
"publish_crate_set", scripts / "publish-crate-set.py"
)
module = importlib.util.module_from_spec(spec)
assert spec.loader is not None
spec.loader.exec_module(module)

calls = []
class Response:
def __init__(self, body):
self.body = body
def __enter__(self):
return self
def __exit__(self, *_args):
return False
def read(self, _limit):
return self.body

def missing(request, timeout):
calls.append((request.full_url, timeout))
raise urllib.error.HTTPError(request.full_url, 404, "missing", {}, None)

module.urllib.request.urlopen = missing
assert module.registry_bytes("rmux", "0.9.1") is None
assert calls == [("https://crates.io/api/v1/crates/rmux/0.9.1", 30)]

def forbidden(request, timeout):
raise urllib.error.HTTPError(request.full_url, 403, "forbidden", {}, None)

module.urllib.request.urlopen = forbidden
try:
module.registry_bytes("rmux", "0.9.1")
except ValueError as error:
assert "lookup failed with HTTP 403" in str(error)
else:
raise AssertionError("metadata access denial was treated as a missing version")

with tempfile.TemporaryDirectory() as directory:
target = pathlib.Path(directory)
filename = "rmux-types-0.9.1.crate"
direct, temporary = module.generated_package_paths(target, filename)
temporary.parent.mkdir(parents=True)
temporary.write_bytes(b"canonical")
assert module.generated_package(target, filename) == temporary
direct.write_bytes(b"duplicate")
try:
module.generated_package(target, filename)
except ValueError as error:
assert "unexpected package set" in str(error)
else:
raise AssertionError("ambiguous Cargo package output was accepted")
module.remove_generated_package(target, filename)
assert not direct.exists() and not temporary.exists()
"#;
let output = Command::new("python3")
.args(["-c", fixture])
.current_dir(repo_root())
.output()
.expect("run crates.io metadata regression fixture");
assert!(
output.status.success(),
"{}",
String::from_utf8_lossy(&output.stderr)
);
}

#[test]
fn downstream_json_writer_uses_canonical_lf_bytes_on_every_platform() {
let nonce = SystemTime::now()
Expand Down
Loading