diff --git a/.github/workflows/build-ruby-from-source.yml b/.github/workflows/build-ruby-from-source.yml new file mode 100644 index 0000000..e4b0ecc --- /dev/null +++ b/.github/workflows/build-ruby-from-source.yml @@ -0,0 +1,260 @@ +name: Build Ruby from Source + +on: + workflow_dispatch: + inputs: + version: + description: 'Ruby version to build (leave empty to auto-detect gaps)' + required: false + type: string + dry_run: + description: 'Only detect gaps, do not build' + required: false + type: boolean + default: false + workflow_run: + workflows: ["Mirror Sync"] + types: [completed] + +jobs: + detect-gaps: + name: Detect gaps + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.detect.outputs.matrix }} + has_gaps: ${{ steps.detect.outputs.has_gaps }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version-file: 'go.mod' + + - name: Build gap detector + run: | + cd scripts/detect-ruby-gaps + go build -o detect-ruby-gaps . + + - name: Detect gaps + id: detect + env: + R2_ENDPOINT: https://${{ secrets.CLOUDFLARE_ACCOUNT_ID }}.r2.cloudflarestorage.com + R2_BUCKET: ${{ secrets.CLOUDFLARE_R2_BUILDS_BUCKET }} + R2_ACCESS_KEY: ${{ secrets.CLOUDFLARE_R2_ACCESS_KEY_ID }} + R2_SECRET_KEY: ${{ secrets.CLOUDFLARE_R2_SECRET_ACCESS_KEY }} + run: | + ARGS="" + if [ -n "${{ inputs.version }}" ]; then + ARGS="--version=${{ inputs.version }}" + else + ARGS="--r2-endpoint=$R2_ENDPOINT --r2-bucket=$R2_BUCKET --r2-access-key=$R2_ACCESS_KEY --r2-secret-key=$R2_SECRET_KEY" + fi + + MATRIX=$(./scripts/detect-ruby-gaps/detect-ruby-gaps $ARGS) + echo "matrix=$MATRIX" >> "$GITHUB_OUTPUT" + + # Check if there are any gaps + COUNT=$(echo "$MATRIX" | jq '.include | length') + if [ "$COUNT" -gt 0 ]; then + echo "has_gaps=true" >> "$GITHUB_OUTPUT" + echo "Found $COUNT gaps to fill" + else + echo "has_gaps=false" >> "$GITHUB_OUTPUT" + echo "No gaps detected" + fi + + - name: Summary + run: | + echo "## Gap Detection Results" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + MATRIX='${{ steps.detect.outputs.matrix }}' + COUNT=$(echo "$MATRIX" | jq '.include | length') + echo "Found **$COUNT** version+platform gaps" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + if [ "$COUNT" -gt 0 ]; then + echo "| Version | Platform | Runner |" >> $GITHUB_STEP_SUMMARY + echo "|---------|----------|--------|" >> $GITHUB_STEP_SUMMARY + echo "$MATRIX" | jq -r '.include[] | "| \(.version) | \(.platform) | \(.runner) |"' >> $GITHUB_STEP_SUMMARY + fi + + build: + name: Build Ruby ${{ matrix.version }} (${{ matrix.platform }}) + needs: detect-gaps + if: needs.detect-gaps.outputs.has_gaps == 'true' && inputs.dry_run != true + runs-on: ${{ matrix.runner }} + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.detect-gaps.outputs.matrix) }} + steps: + # ─── Unix builds (Linux & macOS) ─── + - name: Install ruby-build (Unix) + if: matrix.build_os != 'windows' + run: | + git clone https://github.com/rbenv/ruby-build.git "$RUNNER_TEMP/ruby-build" + echo "$RUNNER_TEMP/ruby-build/bin" >> "$GITHUB_PATH" + + - name: Install build dependencies (Linux) + if: matrix.build_os == 'linux' + run: | + sudo apt-get update + sudo apt-get install -y autoconf bison build-essential libssl-dev libyaml-dev \ + libreadline-dev zlib1g-dev libncurses5-dev libffi-dev libgdbm-dev libgmp-dev rustc + + - name: Install build dependencies (macOS) + if: matrix.build_os == 'darwin' + run: | + brew install openssl@3 readline libyaml gmp rust + + - name: Build Ruby (Unix) + if: matrix.build_os != 'windows' + env: + RUBY_CONFIGURE_OPTS: >- + ${{ matrix.platform == 'darwin-arm64' + && '--disable-shared --disable-install-doc' + || '--enable-shared --disable-install-doc' }} + CPPFLAGS: "-DENABLE_PATH_CHECK=0" + run: | + ruby-build ${{ matrix.version }} "$RUNNER_TEMP/ruby-install/ruby" + + - name: Verify Ruby (Unix) + if: matrix.build_os != 'windows' + run: | + "$RUNNER_TEMP/ruby-install/ruby/bin/ruby" --version + + - name: Package Ruby (Unix) + if: matrix.build_os != 'windows' + run: | + cd "$RUNNER_TEMP/ruby-install" + tar -czf "$RUNNER_TEMP/ruby-${{ matrix.version }}-${{ matrix.platform }}.tar.gz" ruby/ + + # ─── Windows builds (MSYS2/MinGW) ─── + - name: Setup MSYS2 (Windows) + if: matrix.build_os == 'windows' + uses: msys2/setup-msys2@v2 + with: + msystem: ${{ matrix.arch == 'amd64' && 'MINGW64' || 'MINGW32' }} + update: true + install: >- + base-devel + ${{ matrix.arch == 'amd64' + && 'mingw-w64-x86_64-toolchain mingw-w64-x86_64-openssl mingw-w64-x86_64-libyaml mingw-w64-x86_64-libffi mingw-w64-x86_64-readline mingw-w64-x86_64-zlib mingw-w64-x86_64-gmp' + || 'mingw-w64-i686-toolchain mingw-w64-i686-openssl mingw-w64-i686-libyaml mingw-w64-i686-libffi mingw-w64-i686-readline mingw-w64-i686-zlib mingw-w64-i686-gmp' }} + autoconf bison git wget + + - name: Build Ruby (Windows) + if: matrix.build_os == 'windows' + shell: msys2 {0} + run: | + VERSION="${{ matrix.version }}" + MAJOR_MINOR="${VERSION%.*}" + INSTALL_DIR="$RUNNER_TEMP/ruby-install/ruby-$VERSION" + + # Download Ruby source + wget -q "https://cache.ruby-lang.org/pub/ruby/${MAJOR_MINOR}/ruby-${VERSION}.tar.gz" + tar xf "ruby-${VERSION}.tar.gz" + cd "ruby-${VERSION}" + + # Configure and build + ./configure --prefix="$(cygpath -u "$INSTALL_DIR")" --enable-shared --disable-install-doc + make -j$(nproc) + make install + + - name: Verify Ruby (Windows) + if: matrix.build_os == 'windows' + shell: msys2 {0} + run: | + "$RUNNER_TEMP/ruby-install/ruby-${{ matrix.version }}/bin/ruby.exe" --version + + - name: Package Ruby (Windows) + if: matrix.build_os == 'windows' + run: | + cd "$env:RUNNER_TEMP\ruby-install" + 7z a "$env:RUNNER_TEMP\ruby-${{ matrix.version }}-${{ matrix.platform }}.7z" "ruby-${{ matrix.version }}" + + # ─── Upload to R2 (all platforms) ─── + - name: Determine archive path + id: archive + shell: bash + run: | + if [ "${{ matrix.build_os }}" = "windows" ]; then + ARCHIVE="$RUNNER_TEMP/ruby-${{ matrix.version }}-${{ matrix.platform }}.7z" + EXT=".7z" + else + ARCHIVE="$RUNNER_TEMP/ruby-${{ matrix.version }}-${{ matrix.platform }}.tar.gz" + EXT=".tar.gz" + fi + echo "path=$ARCHIVE" >> "$GITHUB_OUTPUT" + echo "ext=$EXT" >> "$GITHUB_OUTPUT" + + - name: Calculate SHA256 + id: checksum + shell: bash + run: | + if [ "${{ matrix.build_os }}" = "windows" ]; then + SHA256=$(sha256sum "${{ steps.archive.outputs.path }}" | awk '{print $1}') + else + SHA256=$(shasum -a 256 "${{ steps.archive.outputs.path }}" | awk '{print $1}') + fi + echo "sha256=$SHA256" >> "$GITHUB_OUTPUT" + + - name: Create metadata + shell: bash + run: | + SIZE=$(wc -c < "${{ steps.archive.outputs.path }}" | tr -d ' ') + cat > "$RUNNER_TEMP/meta.json" << EOF + { + "sha256": "${{ steps.checksum.outputs.sha256 }}", + "sha256_source": "dtvem", + "source_url": "built-from-source", + "mirrored_at": "$(date -u +%Y-%m-%dT%H:%M:%SZ)", + "size": $SIZE + } + EOF + + - name: Install AWS CLI (macOS) + if: matrix.build_os == 'darwin' + run: brew install awscli + + - name: Upload binary to R2 + shell: bash + env: + AWS_ACCESS_KEY_ID: ${{ secrets.CLOUDFLARE_R2_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.CLOUDFLARE_R2_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: auto + R2_ENDPOINT: https://${{ secrets.CLOUDFLARE_ACCOUNT_ID }}.r2.cloudflarestorage.com + R2_BUCKET: ${{ secrets.CLOUDFLARE_R2_BUILDS_BUCKET }} + run: | + aws s3 cp "${{ steps.archive.outputs.path }}" \ + "s3://$R2_BUCKET/ruby/${{ matrix.version }}/${{ matrix.platform }}${{ steps.archive.outputs.ext }}" \ + --endpoint-url "$R2_ENDPOINT" + + - name: Upload metadata to R2 + shell: bash + env: + AWS_ACCESS_KEY_ID: ${{ secrets.CLOUDFLARE_R2_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.CLOUDFLARE_R2_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: auto + R2_ENDPOINT: https://${{ secrets.CLOUDFLARE_ACCOUNT_ID }}.r2.cloudflarestorage.com + R2_BUCKET: ${{ secrets.CLOUDFLARE_R2_BUILDS_BUCKET }} + run: | + aws s3 cp "$RUNNER_TEMP/meta.json" \ + "s3://$R2_BUCKET/ruby/${{ matrix.version }}/${{ matrix.platform }}.meta.json" \ + --endpoint-url "$R2_ENDPOINT" \ + --content-type "application/json" + + trigger-manifests: + name: Trigger manifest regeneration + needs: build + if: needs.build.result == 'success' + runs-on: ubuntu-latest + steps: + - name: Trigger manifest generation + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh workflow run generate-manifests-from-r2.yml \ + --repo CodingWithCalvin/dtvem.cli \ + --field runtime=ruby diff --git a/schemas/manifest.schema.json b/schemas/manifest.schema.json index 76c98d0..1703ba0 100644 --- a/schemas/manifest.schema.json +++ b/schemas/manifest.schema.json @@ -64,6 +64,10 @@ "type": "string", "enum": ["upstream", "dtvem"], "description": "Origin of the SHA256 checksum: 'upstream' if from the original provider, 'dtvem' if generated by us during mirroring" + }, + "source": { + "type": "string", + "description": "Build source indicator: 'built-from-source' when the binary was compiled by dtvem rather than obtained from upstream" } } } diff --git a/scripts/detect-ruby-gaps/go.mod b/scripts/detect-ruby-gaps/go.mod new file mode 100644 index 0000000..a826c74 --- /dev/null +++ b/scripts/detect-ruby-gaps/go.mod @@ -0,0 +1,27 @@ +module github.com/CodingWithCalvin/dtvem.cli/scripts/detect-ruby-gaps + +go 1.23.0 + +require ( + github.com/aws/aws-sdk-go-v2 v1.32.6 + github.com/aws/aws-sdk-go-v2/config v1.28.6 + github.com/aws/aws-sdk-go-v2/credentials v1.17.47 + github.com/aws/aws-sdk-go-v2/service/s3 v1.71.0 +) + +require ( + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.21 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.25 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.25 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.25 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.6 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.6 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.6 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.24.7 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.6 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.33.2 // indirect + github.com/aws/smithy-go v1.22.1 // indirect +) diff --git a/scripts/detect-ruby-gaps/go.sum b/scripts/detect-ruby-gaps/go.sum new file mode 100644 index 0000000..8c425bd --- /dev/null +++ b/scripts/detect-ruby-gaps/go.sum @@ -0,0 +1,36 @@ +github.com/aws/aws-sdk-go-v2 v1.32.6 h1:7BokKRgRPuGmKkFMhEg/jSul+tB9VvXhcViILtfG8b4= +github.com/aws/aws-sdk-go-v2 v1.32.6/go.mod h1:P5WJBrYqqbWVaOxgH0X/FYYD47/nooaPOZPlQdmiN2U= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 h1:lL7IfaFzngfx0ZwUGOZdsFFnQ5uLvR0hWqqhyE7Q9M8= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7/go.mod h1:QraP0UcVlQJsmHfioCrveWOC1nbiWUl3ej08h4mXWoc= +github.com/aws/aws-sdk-go-v2/config v1.28.6 h1:D89IKtGrs/I3QXOLNTH93NJYtDhm8SYa9Q5CsPShmyo= +github.com/aws/aws-sdk-go-v2/config v1.28.6/go.mod h1:GDzxJ5wyyFSCoLkS+UhGB0dArhb9mI+Co4dHtoTxbko= +github.com/aws/aws-sdk-go-v2/credentials v1.17.47 h1:48bA+3/fCdi2yAwVt+3COvmatZ6jUDNkDTIsqDiMUdw= +github.com/aws/aws-sdk-go-v2/credentials v1.17.47/go.mod h1:+KdckOejLW3Ks3b0E3b5rHsr2f9yuORBum0WPnE5o5w= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.21 h1:AmoU1pziydclFT/xRV+xXE/Vb8fttJCLRPv8oAkprc0= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.21/go.mod h1:AjUdLYe4Tgs6kpH4Bv7uMZo7pottoyHMn4eTcIcneaY= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.25 h1:s/fF4+yDQDoElYhfIVvSNyeCydfbuTKzhxSXDXCPasU= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.25/go.mod h1:IgPfDv5jqFIzQSNbUEMoitNooSMXjRSDkhXv8jiROvU= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.25 h1:ZntTCl5EsYnhN/IygQEUugpdwbhdkom9uHcbCftiGgA= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.25/go.mod h1:DBdPrgeocww+CSl1C8cEV8PN1mHMBhuCDLpXezyvWkE= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.25 h1:r67ps7oHCYnflpgDy2LZU0MAQtQbYIOqNNnqGO6xQkE= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.25/go.mod h1:GrGY+Q4fIokYLtjCVB/aFfCVL6hhGUFl8inD18fDalE= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 h1:iXtILhvDxB6kPvEXgsDhGaZCSC6LQET5ZHSdJozeI0Y= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1/go.mod h1:9nu0fVANtYiAePIBh2/pFUSwtJ402hLnp854CNoDOeE= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.6 h1:HCpPsWqmYQieU7SS6E9HXfdAMSud0pteVXieJmcpIRI= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.6/go.mod h1:ngUiVRCco++u+soRRVBIvBZxSMMvOVMXA4PJ36JLfSw= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.6 h1:50+XsN70RS7dwJ2CkVNXzj7U2L1HKP8nqTd3XWEXBN4= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.6/go.mod h1:WqgLmwY7so32kG01zD8CPTJWVWM+TzJoOVHwTg4aPug= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.6 h1:BbGDtTi0T1DYlmjBiCr/le3wzhA37O8QTC5/Ab8+EXk= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.6/go.mod h1:hLMJt7Q8ePgViKupeymbqI0la+t9/iYFBjxQCFwuAwI= +github.com/aws/aws-sdk-go-v2/service/s3 v1.71.0 h1:nyuzXooUNJexRT0Oy0UQY6AhOzxPxhtt4DcBIHyCnmw= +github.com/aws/aws-sdk-go-v2/service/s3 v1.71.0/go.mod h1:sT/iQz8JK3u/5gZkT+Hmr7GzVZehUMkRZpOaAwYXeGY= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.7 h1:rLnYAfXQ3YAccocshIH5mzNNwZBkBo+bP6EhIxak6Hw= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.7/go.mod h1:ZHtuQJ6t9A/+YDuxOLnbryAmITtr8UysSny3qcyvJTc= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.6 h1:JnhTZR3PiYDNKlXy50/pNeix9aGMo6lLpXwJ1mw8MD4= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.6/go.mod h1:URronUEGfXZN1VpdktPSD1EkAL9mfrV+2F4sjH38qOY= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.2 h1:s4074ZO1Hk8qv65GqNXqDjmkf4HSQqJukaLuuW0TpDA= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.2/go.mod h1:mVggCnIWoM09jP71Wh+ea7+5gAp53q+49wDFs1SW5z8= +github.com/aws/smithy-go v1.22.1 h1:/HPHZQ0g7f4eUeK6HKglFz8uwVfZKgoI25rb/J+dnro= +github.com/aws/smithy-go v1.22.1/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= diff --git a/scripts/detect-ruby-gaps/main.go b/scripts/detect-ruby-gaps/main.go new file mode 100644 index 0000000..1acdb89 --- /dev/null +++ b/scripts/detect-ruby-gaps/main.go @@ -0,0 +1,419 @@ +package main + +import ( + "context" + "encoding/json" + "flag" + "fmt" + "net/http" + "os" + "regexp" + "sort" + "strings" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" +) + +// Platform describes a target platform with runner metadata for GitHub Actions +type Platform struct { + Name string + Runner string + BuildOS string + Arch string +} + +// MatrixEntry represents a single entry in the GitHub Actions matrix +type MatrixEntry struct { + Version string `json:"version"` + Platform string `json:"platform"` + Runner string `json:"runner"` + BuildOS string `json:"build_os"` + Arch string `json:"arch"` +} + +// MatrixOutput is the JSON structure consumed by GitHub Actions fromJson() +type MatrixOutput struct { + Include []MatrixEntry `json:"include"` +} + +var allPlatforms = []Platform{ + {Name: "linux-amd64", Runner: "ubuntu-latest", BuildOS: "linux", Arch: "amd64"}, + {Name: "linux-arm64", Runner: "ubuntu-24.04-arm", BuildOS: "linux", Arch: "arm64"}, + {Name: "darwin-amd64", Runner: "macos-13", BuildOS: "darwin", Arch: "amd64"}, + {Name: "darwin-arm64", Runner: "macos-latest", BuildOS: "darwin", Arch: "arm64"}, + {Name: "windows-amd64", Runner: "windows-latest", BuildOS: "windows", Arch: "amd64"}, + {Name: "windows-386", Runner: "windows-latest", BuildOS: "windows", Arch: "386"}, +} + +var ( + versionFlag = flag.String("version", "", "Force all 6 platforms for this version (no R2 check)") + r2Endpoint = flag.String("r2-endpoint", "", "R2 endpoint URL") + r2Bucket = flag.String("r2-bucket", "", "R2 bucket name") + r2AccessKey = flag.String("r2-access-key", "", "R2 access key ID") + r2SecretKey = flag.String("r2-secret-key", "", "R2 secret access key") +) + +// githubRelease represents a GitHub release +type githubRelease struct { + TagName string `json:"tag_name"` + Assets []githubAsset `json:"assets"` +} + +// githubAsset represents a GitHub release asset +type githubAsset struct { + Name string `json:"name"` + BrowserDownloadURL string `json:"browser_download_url"` +} + +// httpClient is a shared HTTP client with reasonable timeouts +var httpClient = &http.Client{ + Timeout: 60 * time.Second, +} + +func main() { + flag.Parse() + + // If --version is provided, output all 6 platforms without R2 check + if *versionFlag != "" { + matrix := buildMatrixForVersion(*versionFlag) + outputMatrix(matrix) + return + } + + // Otherwise, detect gaps by comparing upstream vs R2 + if *r2Endpoint == "" || *r2Bucket == "" || *r2AccessKey == "" || *r2SecretKey == "" { + fmt.Fprintln(os.Stderr, "Error: R2 credentials required (--r2-endpoint, --r2-bucket, --r2-access-key, --r2-secret-key)") + fmt.Fprintln(os.Stderr, " Or use --version=X to force all platforms for a specific version") + os.Exit(1) + } + + // Fetch known versions from upstream sources + fmt.Fprintln(os.Stderr, "Fetching Ruby versions from upstream sources...") + knownVersions, err := fetchKnownVersions() + if err != nil { + fmt.Fprintf(os.Stderr, "Error fetching upstream versions: %v\n", err) + os.Exit(1) + } + fmt.Fprintf(os.Stderr, "Found %d unique Ruby versions from upstream\n", len(knownVersions)) + + // Fetch existing metadata from R2 + fmt.Fprintln(os.Stderr, "Fetching existing metadata from R2...") + existingMeta, err := fetchExistingMeta() + if err != nil { + fmt.Fprintf(os.Stderr, "Error fetching R2 metadata: %v\n", err) + os.Exit(1) + } + fmt.Fprintf(os.Stderr, "Found %d existing metadata entries in R2\n", len(existingMeta)) + + // Compute gaps + matrix := computeGaps(knownVersions, existingMeta) + fmt.Fprintf(os.Stderr, "Detected %d gaps\n", len(matrix.Include)) + + outputMatrix(matrix) +} + +// buildMatrixForVersion creates a matrix with all 6 platforms for a given version +func buildMatrixForVersion(version string) *MatrixOutput { + matrix := &MatrixOutput{} + for _, p := range allPlatforms { + // Exclude darwin-arm64 for versions < 3.1.0 + if p.Name == "darwin-arm64" && !supportsARM64Darwin(version) { + continue + } + matrix.Include = append(matrix.Include, MatrixEntry{ + Version: version, + Platform: p.Name, + Runner: p.Runner, + BuildOS: p.BuildOS, + Arch: p.Arch, + }) + } + return matrix +} + +// supportsARM64Darwin returns true if the version supports ARM64 macOS (>= 3.1.0) +func supportsARM64Darwin(version string) bool { + parts := strings.SplitN(version, ".", 3) + if len(parts) < 2 { + return false + } + major := 0 + minor := 0 + fmt.Sscanf(parts[0], "%d", &major) + fmt.Sscanf(parts[1], "%d", &minor) + + if major > 3 { + return true + } + if major == 3 && minor >= 1 { + return true + } + return false +} + +// fetchKnownVersions queries upstream sources and returns a sorted list of unique versions +func fetchKnownVersions() ([]string, error) { + versionSet := make(map[string]bool) + + // Fetch from RubyInstaller (Windows builds) + fmt.Fprintln(os.Stderr, " Fetching from RubyInstaller...") + installerVersions, err := fetchRubyInstallerVersions() + if err != nil { + fmt.Fprintf(os.Stderr, " Warning: failed to fetch from RubyInstaller: %v\n", err) + } else { + for _, v := range installerVersions { + versionSet[v] = true + } + fmt.Fprintf(os.Stderr, " Found %d versions from RubyInstaller\n", len(installerVersions)) + } + + // Fetch from ruby-builder (Linux/macOS builds) + fmt.Fprintln(os.Stderr, " Fetching from ruby-builder...") + builderVersions, err := fetchRubyBuilderVersions() + if err != nil { + fmt.Fprintf(os.Stderr, " Warning: failed to fetch from ruby-builder: %v\n", err) + } else { + for _, v := range builderVersions { + versionSet[v] = true + } + fmt.Fprintf(os.Stderr, " Found %d versions from ruby-builder\n", len(builderVersions)) + } + + // Filter: only >= 2.7.0, exclude preview/rc + var versions []string + for v := range versionSet { + if isPreRelease(v) { + continue + } + if !isAtLeast270(v) { + continue + } + versions = append(versions, v) + } + + sort.Strings(versions) + return versions, nil +} + +// rubyInstallerPattern matches filenames like: rubyinstaller-3.2.2-1-x64.7z +var rubyInstallerPattern = regexp.MustCompile( + `^rubyinstaller-(\d+\.\d+\.\d+)-\d+-([^.]+)\.(7z|zip)$`, +) + +func fetchRubyInstallerVersions() ([]string, error) { + url := "https://api.github.com/repos/oneclick/rubyinstaller2/releases?per_page=100" + resp, err := httpGetWithRetry(url, 3) + if err != nil { + return nil, fmt.Errorf("fetching releases: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + return nil, fmt.Errorf("HTTP %d", resp.StatusCode) + } + + var releases []githubRelease + if err := json.NewDecoder(resp.Body).Decode(&releases); err != nil { + return nil, fmt.Errorf("parsing releases: %w", err) + } + + seen := make(map[string]bool) + var versions []string + for _, release := range releases { + for _, asset := range release.Assets { + matches := rubyInstallerPattern.FindStringSubmatch(asset.Name) + if matches == nil { + continue + } + version := matches[1] + if !seen[version] { + seen[version] = true + versions = append(versions, version) + } + } + } + + return versions, nil +} + +// rubyBuilderPattern matches filenames like: ruby-3.2.2-ubuntu-22.04.tar.gz +var rubyBuilderPattern = regexp.MustCompile( + `^ruby-(\d+\.\d+\.\d+)-([^.]+(?:\.[^.]+)?(?:-arm64)?)\.(tar\.gz)$`, +) + +func fetchRubyBuilderVersions() ([]string, error) { + url := "https://api.github.com/repos/ruby/ruby-builder/releases/tags/toolcache" + resp, err := httpGetWithRetry(url, 3) + if err != nil { + return nil, fmt.Errorf("fetching release: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + return nil, fmt.Errorf("HTTP %d", resp.StatusCode) + } + + var release githubRelease + if err := json.NewDecoder(resp.Body).Decode(&release); err != nil { + return nil, fmt.Errorf("parsing release: %w", err) + } + + seen := make(map[string]bool) + var versions []string + for _, asset := range release.Assets { + matches := rubyBuilderPattern.FindStringSubmatch(asset.Name) + if matches == nil { + continue + } + version := matches[1] + if !seen[version] { + seen[version] = true + versions = append(versions, version) + } + } + + return versions, nil +} + +// isPreRelease returns true if the version contains a pre-release suffix (e.g., "-preview1", "-rc1") +func isPreRelease(version string) bool { + return strings.Contains(version, "-") +} + +// isAtLeast270 returns true if the version is >= 2.7.0 +func isAtLeast270(version string) bool { + parts := strings.SplitN(version, ".", 3) + if len(parts) < 2 { + return false + } + major := 0 + minor := 0 + fmt.Sscanf(parts[0], "%d", &major) + fmt.Sscanf(parts[1], "%d", &minor) + + if major > 2 { + return true + } + if major == 2 && minor >= 7 { + return true + } + return false +} + +// fetchExistingMeta lists all ruby/**/*.meta.json keys in R2 +func fetchExistingMeta() (map[string]bool, error) { + client, err := createS3Client() + if err != nil { + return nil, fmt.Errorf("creating S3 client: %w", err) + } + + keys := make(map[string]bool) + paginator := s3.NewListObjectsV2Paginator(client, &s3.ListObjectsV2Input{ + Bucket: r2Bucket, + Prefix: aws.String("ruby/"), + }) + + for paginator.HasMorePages() { + page, err := paginator.NextPage(context.Background()) + if err != nil { + return nil, fmt.Errorf("listing objects: %w", err) + } + for _, obj := range page.Contents { + key := *obj.Key + if strings.HasSuffix(key, ".meta.json") { + keys[key] = true + } + } + } + + return keys, nil +} + +// metaKeyPattern matches paths like "ruby/3.2.10/linux-amd64.meta.json" +var metaKeyPattern = regexp.MustCompile(`^ruby/([^/]+)/([^/]+)\.meta\.json$`) + +// computeGaps determines which version+platform pairs are missing from R2 +func computeGaps(versions []string, existingMeta map[string]bool) *MatrixOutput { + matrix := &MatrixOutput{} + + for _, version := range versions { + for _, p := range allPlatforms { + // Exclude darwin-arm64 for versions < 3.1.0 + if p.Name == "darwin-arm64" && !supportsARM64Darwin(version) { + continue + } + + metaKey := fmt.Sprintf("ruby/%s/%s.meta.json", version, p.Name) + if !existingMeta[metaKey] { + matrix.Include = append(matrix.Include, MatrixEntry{ + Version: version, + Platform: p.Name, + Runner: p.Runner, + BuildOS: p.BuildOS, + Arch: p.Arch, + }) + } + } + } + + return matrix +} + +func outputMatrix(matrix *MatrixOutput) { + data, err := json.Marshal(matrix) + if err != nil { + fmt.Fprintf(os.Stderr, "Error marshaling matrix: %v\n", err) + os.Exit(1) + } + fmt.Println(string(data)) +} + +func createS3Client() (*s3.Client, error) { + cfg, err := config.LoadDefaultConfig(context.Background(), + config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider( + *r2AccessKey, + *r2SecretKey, + "", + )), + config.WithRegion("auto"), + ) + if err != nil { + return nil, err + } + + client := s3.NewFromConfig(cfg, func(o *s3.Options) { + o.BaseEndpoint = aws.String(*r2Endpoint) + }) + + return client, nil +} + +func httpGetWithRetry(url string, maxRetries int) (*http.Response, error) { + var lastErr error + for attempt := 1; attempt <= maxRetries; attempt++ { + resp, err := httpClient.Get(url) + if err != nil { + lastErr = err + if attempt < maxRetries { + time.Sleep(time.Duration(attempt) * 2 * time.Second) + } + continue + } + + if resp.StatusCode >= 500 { + resp.Body.Close() + lastErr = fmt.Errorf("HTTP %d", resp.StatusCode) + if attempt < maxRetries { + time.Sleep(time.Duration(attempt) * 2 * time.Second) + } + continue + } + + return resp, nil + } + return nil, lastErr +} diff --git a/scripts/generate-manifests-from-r2/main.go b/scripts/generate-manifests-from-r2/main.go index 5858e89..86a4ada 100644 --- a/scripts/generate-manifests-from-r2/main.go +++ b/scripts/generate-manifests-from-r2/main.go @@ -32,6 +32,7 @@ type ManifestDownload struct { URL string `json:"url"` SHA256 string `json:"sha256,omitempty"` SHA256Source string `json:"sha256_source,omitempty"` + Source string `json:"source,omitempty"` // "built-from-source" when not from upstream } // Manifest represents the output manifest structure @@ -143,7 +144,7 @@ func generateManifest(client *s3.Client, runtime string) (*Manifest, error) { Versions: make(map[string]map[string]*ManifestDownload), } - // List all .meta.json files for this runtime + // List all files for this runtime, separating meta files from binary keys prefix := runtime + "/" paginator := s3.NewListObjectsV2Paginator(client, &s3.ListObjectsV2Input{ Bucket: r2Bucket, @@ -151,6 +152,7 @@ func generateManifest(client *s3.Client, runtime string) (*Manifest, error) { }) metaFiles := []string{} + allKeys := make(map[string]bool) for paginator.HasMorePages() { page, err := paginator.NextPage(context.Background()) if err != nil { @@ -159,6 +161,7 @@ func generateManifest(client *s3.Client, runtime string) (*Manifest, error) { for _, obj := range page.Contents { key := *obj.Key + allKeys[key] = true if strings.HasSuffix(key, ".meta.json") { metaFiles = append(metaFiles, key) } @@ -193,6 +196,15 @@ func generateManifest(client *s3.Client, runtime string) (*Manifest, error) { // Determine the binary file extension from source URL ext := getExtension(meta.SourceURL) + if !isValidArchiveExt(ext) { + // Source URL doesn't contain a valid extension (e.g., "built-from-source"), + // probe R2 for the actual binary file + ext = findBinaryExtension(allKeys, runtime, version, platform) + if ext == "" { + fmt.Printf(" Warning: no valid binary found for %s/%s/%s, skipping\n", runtime, version, platform) + continue + } + } binaryURL := fmt.Sprintf("%s/%s/%s/%s%s", *baseURL, runtime, version, platform, ext) // Add to manifest @@ -200,11 +212,16 @@ func generateManifest(client *s3.Client, runtime string) (*Manifest, error) { manifest.Versions[version] = make(map[string]*ManifestDownload) } - manifest.Versions[version][platform] = &ManifestDownload{ + download := &ManifestDownload{ URL: binaryURL, SHA256: meta.SHA256, SHA256Source: meta.SHA256Source, } + if meta.SourceURL == "built-from-source" { + download.Source = "built-from-source" + } + + manifest.Versions[version][platform] = download } return manifest, nil @@ -258,6 +275,24 @@ func getExtension(url string) string { return "" } +func isValidArchiveExt(ext string) bool { + switch ext { + case ".tar.gz", ".tar.xz", ".tar.bz2", ".zip", ".7z": + return true + } + return false +} + +func findBinaryExtension(allKeys map[string]bool, runtime, version, platform string) string { + prefix := fmt.Sprintf("%s/%s/%s", runtime, version, platform) + for _, ext := range []string{".tar.gz", ".tar.xz", ".zip", ".7z"} { + if allKeys[prefix+ext] { + return ext + } + } + return "" +} + func writeManifest(manifest *Manifest, path string) error { // Sort versions for consistent output sortedManifest := &Manifest{ diff --git a/scripts/mirror-binaries/main.go b/scripts/mirror-binaries/main.go index c6574a2..2189154 100644 --- a/scripts/mirror-binaries/main.go +++ b/scripts/mirror-binaries/main.go @@ -89,6 +89,7 @@ func main() { // Initialize S3 client for R2 var s3Client *s3.Client var existingKeys map[string]bool + var builtFromSourceKeys map[string]bool if !*dryRun { var err error @@ -106,6 +107,16 @@ func main() { os.Exit(1) } fmt.Printf("Found %d existing files in R2\n", len(existingKeys)) + + fmt.Println("Checking for built-from-source entries...") + builtFromSourceKeys, err = listBuiltFromSourceKeys(s3Client, existingKeys) + if err != nil { + fmt.Fprintf(os.Stderr, "Error checking built-from-source keys: %v\n", err) + os.Exit(1) + } + if len(builtFromSourceKeys) > 0 { + fmt.Printf("Found %d built-from-source entries (will re-mirror from upstream)\n", len(builtFromSourceKeys)) + } } } @@ -146,8 +157,11 @@ func main() { if *syncOnly && existingKeys != nil { var filtered []MirrorJob for _, job := range jobs { - // Check for metadata file existence (indicates successful mirror) if !existingKeys[job.MetaKey] { + // Not yet mirrored + filtered = append(filtered, job) + } else if builtFromSourceKeys[job.MetaKey] { + // Exists but was built from source — re-mirror from upstream filtered = append(filtered, job) } } @@ -218,6 +232,52 @@ func listExistingKeys(client *s3.Client) (map[string]bool, error) { return keys, nil } +// listBuiltFromSourceKeys downloads .meta.json files that exist in R2 and returns +// the set of meta keys where source_url is "built-from-source". Only checks meta +// files that are in the existingKeys set to avoid unnecessary downloads. +func listBuiltFromSourceKeys(client *s3.Client, existingKeys map[string]bool) (map[string]bool, error) { + builtFromSource := make(map[string]bool) + + // Collect all .meta.json keys from existingKeys + var metaKeys []string + for key := range existingKeys { + if strings.HasSuffix(key, ".meta.json") { + metaKeys = append(metaKeys, key) + } + } + + // Download and check each meta file + for _, metaKey := range metaKeys { + resp, err := client.GetObject(context.Background(), &s3.GetObjectInput{ + Bucket: r2Bucket, + Key: aws.String(metaKey), + }) + if err != nil { + fmt.Fprintf(os.Stderr, "Warning: failed to read %s: %v\n", metaKey, err) + continue + } + + data, err := io.ReadAll(resp.Body) + resp.Body.Close() + if err != nil { + fmt.Fprintf(os.Stderr, "Warning: failed to read body of %s: %v\n", metaKey, err) + continue + } + + var meta BinaryMeta + if err := json.Unmarshal(data, &meta); err != nil { + fmt.Fprintf(os.Stderr, "Warning: failed to parse %s: %v\n", metaKey, err) + continue + } + + if meta.SourceURL == "built-from-source" { + builtFromSource[metaKey] = true + } + } + + return builtFromSource, nil +} + func getExtension(url string) string { // Handle common archive extensions if strings.HasSuffix(url, ".tar.gz") {