sync.py

import os
import subprocess
import sys
import time
from re import sub
from urllib.parse import urlparse

import requests

GITHUB_TOKEN = os.environ["MIRROR_PAT"]
GITHUB_API = "https://api.github.com"
SESSION = requests.Session()
SESSION.headers.update(
    {
        "Authorization": f"token {GITHUB_TOKEN}",
        "Accept": "application/vnd.github+json",
    }
)

ORG = "OpenActionMirrors"
CATALOGUE_URL = "https://openactionapi.github.io/plugins/catalogue.json"


def run(cmd, cwd=None, capture_output=False):
    s = sub("ghp_.*?:", "", " ".join(cmd))
    print(f"Running: {s}", flush=True)
    if capture_output:
        result = subprocess.run(
            cmd, cwd=cwd, capture_output=True, text=True, check=True
        )
        return result.stdout
    else:
        subprocess.check_call(cmd, cwd=cwd)
        return None


def get_owner_repo_from_url(url: str):
    """
    Convert https://github.com/OpenActionPlugins/counter(.git)?
    to OpenActionPlugins/counter
    """
    parsed = urlparse(url)
    if parsed.netloc != "github.com":
        raise ValueError(f"Unsupported repository host in URL: {url}")
    # Path like /OpenActionPlugins/counter or /OpenActionPlugins/counter.git
    parts = parsed.path.strip("/").split("/")
    if len(parts) < 2:
        raise ValueError(f"Unexpected GitHub URL path: {url}")
    owner, repo = parts[0], parts[1]
    if repo.endswith(".git"):
        repo = repo[:-4]
    return f"{owner}/{repo}"


def ensure_repo_exists_and_update_description(mirror_name, upstream_full):
    """
    Ensure OpenActionMirrors/mirror_name exists, with a description that
    clearly marks it as a mirror of upstream_full.
    """
    owner = ORG
    repo = mirror_name

    r = SESSION.get(f"{GITHUB_API}/repos/{owner}/{repo}")
    description = f"Mirror of https://github.com/{upstream_full}"

    if r.status_code == 404:
        print(f"Creating mirror repo {owner}/{repo}")
        r = SESSION.post(
            f"{GITHUB_API}/orgs/{owner}/repos",
            json={
                "name": repo,
                "private": False,
                "has_issues": False,
                "has_wiki": False,
                "has_projects": False,
                "description": description,
            },
        )
        r.raise_for_status()
    else:
        r.raise_for_status()
        data = r.json()
        # Update description if different
        if data.get("description") != description:
            print(f"Updating description for {owner}/{repo}")
            ur = SESSION.patch(
                f"{GITHUB_API}/repos/{owner}/{repo}", json={"description": description}
            )
            ur.raise_for_status()


def sync_git(upstream_full, mirror_name, workdir="mirrors"):
    """
    upstream_full: 'OpenActionPlugins/counter'
    mirror_name: 'me.amankhanna.oacounter'
    """
    os.makedirs(workdir, exist_ok=True)
    upstream_owner, upstream_repo = upstream_full.split("/")

    local_path = os.path.join(workdir, f"{mirror_name}.git")

    upstream_url = f"https://github.com/{upstream_owner}/{upstream_repo}.git"
    mirror_url = (
        f"https://{GITHUB_TOKEN}:x-oauth-basic@github.com/{ORG}/{mirror_name}.git"
    )

    needs_push = False
    if not os.path.exists(local_path):
        run(["git", "clone", "--mirror", upstream_url, local_path])
        needs_push = True
    else:
        run(["git", "remote", "set-url", "origin", upstream_url], cwd=local_path)
        fetch_output = run(
            ["git", "fetch", "--prune", "origin"], cwd=local_path, capture_output=True
        )
        # If fetch output is not empty, there were updates
        if fetch_output and fetch_output.strip():
            needs_push = True

    if needs_push:
        run(
            ["git", "remote", "set-url", "--push", "origin", mirror_url], cwd=local_path
        )
        run(
            [
                "git",
                "push",
                "--prune",
                mirror_url,
                "+refs/heads/*:refs/heads/*",
                "+refs/tags/*:refs/tags/*",
            ],
            cwd=local_path,
        )
    else:
        print(f"No upstream changes detected for {mirror_name}, skipping push")


def get_releases(owner_repo):
    releases = []
    page = 1
    while True:
        r = SESSION.get(
            f"{GITHUB_API}/repos/{owner_repo}/releases",
            params={"per_page": 100, "page": page},
        )
        r.raise_for_status()
        batch = r.json()
        if not batch:
            break
        releases.extend(batch)
        page += 1
    return releases


def find_release_by_tag(releases, tag_name):
    for rel in releases:
        if rel["tag_name"] == tag_name:
            return rel
    return None


def sync_release_assets(upstream_rel, mirror_rel, mirror_repo):
    upstream_assets = upstream_rel.get("assets", [])
    mirror_assets = mirror_rel.get("assets", [])
    mirror_asset_names = {a["name"] for a in mirror_assets}

    for asset in upstream_assets:
        name = asset["name"]
        if name in mirror_asset_names:
            continue

        print(f"Copying asset {name} to {mirror_repo} release {mirror_rel['tag_name']}")
        download_url = asset["browser_download_url"]

        # Download the asset as raw bytes
        dr = SESSION.get(download_url, stream=True)
        try:
            dr.raise_for_status()
            content = dr.content  # Load into memory; fine for typical plugin sizes
        finally:
            dr.close()

        upload_url = (
            f"https://uploads.github.com/repos/{mirror_repo}/releases/"
            f"{mirror_rel['id']}/assets"
        )
        params = {"name": name}
        headers = {
            "Authorization": f"token {GITHUB_TOKEN}",
            "Accept": "application/vnd.github+json",
            "Content-Type": "application/octet-stream",
        }

        ur = requests.post(upload_url, params=params, headers=headers, data=content)
        if ur.status_code == 422:
            # Often means "already exists" or "invalid name"; log and continue
            print(
                f"Warning: upload of asset {name} to {mirror_repo} failed with 422; body: {ur.text}"
            )
            continue

        ur.raise_for_status()
        time.sleep(1)


def sync_releases(upstream_full, mirror_name):
    upstream_repo = upstream_full
    mirror_repo = f"{ORG}/{mirror_name}"
    print(f"Syncing releases {upstream_repo} -> {mirror_repo}")
    upstream_rels = get_releases(upstream_repo)
    mirror_rels = get_releases(mirror_repo)

    for urel in upstream_rels[::-1]:
        tag = urel["tag_name"]
        mrel = find_release_by_tag(mirror_rels, tag)

        if mrel is None:
            print(f"Creating release {tag} in {mirror_repo}")
            payload = {
                "tag_name": urel["tag_name"],
                "name": urel.get("name") or urel["tag_name"],
                "body": urel.get("body") or "",
                "draft": urel.get("draft", False),
                "prerelease": urel.get("prerelease", False),
            }
            r = SESSION.post(f"{GITHUB_API}/repos/{mirror_repo}/releases", json=payload)
            r.raise_for_status()
            mrel = r.json()
            mirror_rels.append(mrel)
        elif (
            urel.get("name", urel["tag_name"]) != mrel.get("name", mrel["tag_name"])
            or urel.get("body", "") != mrel.get("body", "")
            or urel.get("draft", False) != mrel.get("draft", False)
            or urel.get("prerelease", False) != mrel.get("prerelease", False)
        ):
            # Keep metadata roughly in sync, but only update if changed
            print(f"Updating release metadata for {tag} in {mirror_repo}")
            payload = {
                "name": urel.get("name", urel["tag_name"]),
                "body": urel.get("body", ""),
                "draft": urel.get("draft", False),
                "prerelease": urel.get("prerelease", False),
            }
            r = SESSION.patch(
                f"{GITHUB_API}/repos/{mirror_repo}/releases/{mrel['id']}", json=payload
            )
            r.raise_for_status()

        sync_release_assets(urel, mrel, mirror_repo)


def load_catalogue():
    print(f"Downloading catalogue from {CATALOGUE_URL}")
    r = SESSION.get(CATALOGUE_URL)
    r.raise_for_status()
    return r.json()


def iter_entries_from_catalogue(catalogue):
    """
    Yields tuples:
      (reverse_dns_id, upstream_full, display_name)
    """
    for reverse_dns_id, data in catalogue.items():
        repo_url = data.get("repository")
        if not repo_url:
            continue
        try:
            upstream_full = get_owner_repo_from_url(repo_url)
        except ValueError as e:
            print(f"Skipping {reverse_dns_id}: {e}", file=sys.stderr)
            continue
        name = data.get("name", reverse_dns_id)
        yield reverse_dns_id, upstream_full, name


def main():
    catalogue = load_catalogue()

    for reverse_dns_id, upstream_full, display_name in iter_entries_from_catalogue(
        catalogue
    ):
        print(
            f"=== Sync {upstream_full} -> {ORG}/{reverse_dns_id} ({display_name}) ==="
        )
        try:
            ensure_repo_exists_and_update_description(reverse_dns_id, upstream_full)
        except requests.HTTPError as e:
            print(f"Failed to ensure repo {reverse_dns_id}: {e}", file=sys.stderr)
            continue

        try:
            sync_git(upstream_full, reverse_dns_id)
        except subprocess.CalledProcessError as e:
            print(f"Git sync failed for {upstream_full}: {e}", file=sys.stderr)

        try:
            sync_releases(upstream_full, reverse_dns_id)
        except requests.HTTPError as e:
            print(f"Release sync failed for {upstream_full}: {e}", file=sys.stderr)

    return 0


if __name__ == "__main__":
    raise SystemExit(main())