more efficient CI #1673

Workflow file for this run

	name: CI

	on:
	push:
	branches:
	- main
	pull_request:
	branches:
	- '**'

	env:
	GO_VERSION: 1.24.7

	jobs:
	# Fast checks - no FFI needed, run immediately
	quick-checks:
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4

	- name: Setup Go
	uses: actions/setup-go@v5
	with:
	go-version: ${{ env.GO_VERSION }}
	cache: true
	cache-dependency-path: go.sum

	- name: Run checks in parallel
	run: \|
	# gofmt check
	(
	go fmt ./...
	git diff --quiet \|\| { echo "gofmt needed"; exit 1; }
	) &
	FMT_PID=$!

	# actionlint
	(
	go install github.com/rhysd/actionlint/cmd/actionlint@latest
	actionlint -shellcheck= -pyflakes=
	) &
	LINT_PID=$!

	# mod tidy check
	(
	go mod tidy -v
	git diff --quiet go.mod go.sum \|\| { echo "go mod tidy needed"; exit 1; }
	) &
	TIDY_PID=$!

	wait $FMT_PID \|\| exit 1
	wait $LINT_PID \|\| exit 1
	wait $TIDY_PID \|\| exit 1

	# Lint - runs immediately, no dependencies
	lint:
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4
	with:
	submodules: recursive

	- name: Setup build environment
	uses: ./.github/actions/setup-build-env
	with:
	go-version: ${{ env.GO_VERSION }}

	- name: Install and run golangci-lint
	run: \|
	curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh \| sh -s -- -b $(go env GOPATH)/bin v2.4.0
	golangci-lint run -v --timeout 15m --concurrency 4

	# Build variants - run in parallel with matrix
	build:
	runs-on: ubuntu-latest
	strategy:
	fail-fast: false
	matrix:
	variant: [debug, build, calibnet, 2k]
	steps:
	- uses: actions/checkout@v4
	with:
	submodules: recursive

	- name: Setup build environment
	uses: ./.github/actions/setup-build-env
	with:
	go-version: ${{ env.GO_VERSION }}

	- name: Build ${{ matrix.variant }}
	run: make ${{ matrix.variant }}

	# Unit tests - no database needed
	test-unit:
	runs-on: [self-hosted, docker]
	steps:
	- uses: actions/checkout@v4
	with:
	submodules: recursive

	- name: Setup build environment
	uses: ./.github/actions/setup-build-env
	with:
	go-version: ${{ env.GO_VERSION }}

	- name: Run unit tests
	run: go test -v --tags=debug -timeout 30m $(go list ./... \| grep -v curio/itests)

	# Integration tests - curio_test.go (heavy, runs separately)
	test-itest-curio:
	runs-on: [self-hosted, docker]
	env:
	YB_CONTAINER: yugabyte-curio-${{ github.run_id }}
	steps:
	- name: Check local proof parameters
	id: local-params
	run: \|
	PARAMS_DIR="/var/tmp/filecoin-proof-parameters"
	if [ -d "$PARAMS_DIR" ] && [ "$(ls -A $PARAMS_DIR 2>/dev/null)" ]; then
	echo "exists=true" >> $GITHUB_OUTPUT
	else
	echo "exists=false" >> $GITHUB_OUTPUT
	fi

	- name: Restore proof parameters from cache
	if: steps.local-params.outputs.exists != 'true'
	id: cache-params
	uses: actions/cache/restore@v4
	with:
	path: /var/tmp/filecoin-proof-parameters
	key: proof-params-2k-v1
	restore-keys: proof-params-

	- name: Fetch proof parameters
	if: steps.local-params.outputs.exists != 'true' && steps.cache-params.outputs.cache-hit != 'true'
	run: \|
	echo "Fetching proof parameters (cache miss)..."
	lotus fetch-params 8388608

	- name: Save proof parameters to cache
	if: steps.local-params.outputs.exists != 'true' && steps.cache-params.outputs.cache-hit != 'true' && github.ref == 'refs/heads/main'
	uses: actions/cache/save@v4
	with:
	path: /var/tmp/filecoin-proof-parameters
	key: proof-params-2k-v1

	- uses: actions/checkout@v4
	with:
	submodules: recursive

	- name: Start YugabyteDB
	id: start-yb
	run: \|
	docker stop $YB_CONTAINER 2>/dev/null \|\| true
	docker rm $YB_CONTAINER 2>/dev/null \|\| true
	docker run --rm --name $YB_CONTAINER -d yugabytedb/yugabyte:2024.1.2.0-b77 bin/yugabyted start --daemon=false
	for i in {1..60}; do
	if docker exec $YB_CONTAINER bin/yugabyted status 2>/dev/null \| grep -q Running; then
	echo "YugabyteDB is ready"
	break
	fi
	sleep 1
	done
	YB_IP=$(docker inspect $YB_CONTAINER --format '{{ .NetworkSettings.Networks.bridge.IPAddress }}')
	echo "yb_ip=$YB_IP" >> $GITHUB_OUTPUT

	- name: Setup build environment
	uses: ./.github/actions/setup-build-env
	with:
	go-version: ${{ env.GO_VERSION }}

	- name: Run curio integration test
	env:
	CURIO_HARMONYDB_HOSTS: ${{ steps.start-yb.outputs.yb_ip }}
	LOTUS_HARMONYDB_HOSTS: ${{ steps.start-yb.outputs.yb_ip }}
	run: go test -v --tags=debug -timeout 60m ./itests/curio_test.go

	- name: Stop YugabyteDB
	if: always()
	run: docker stop $YB_CONTAINER 2>/dev/null \|\| true

	# Integration tests - other parallel tests
	test-itest-other:
	runs-on: [self-hosted, docker]
	env:
	YB_CONTAINER: yugabyte-other-${{ github.run_id }}
	steps:
	- name: Check local proof parameters
	id: local-params
	run: \|
	PARAMS_DIR="/var/tmp/filecoin-proof-parameters"
	if [ -d "$PARAMS_DIR" ] && [ "$(ls -A $PARAMS_DIR 2>/dev/null)" ]; then
	echo "exists=true" >> $GITHUB_OUTPUT
	else
	echo "exists=false" >> $GITHUB_OUTPUT
	fi

	- name: Restore proof parameters from cache
	if: steps.local-params.outputs.exists != 'true'
	id: cache-params
	uses: actions/cache/restore@v4
	with:
	path: /var/tmp/filecoin-proof-parameters
	key: proof-params-2k-v1
	restore-keys: proof-params-

	- name: Fetch proof parameters
	if: steps.local-params.outputs.exists != 'true' && steps.cache-params.outputs.cache-hit != 'true'
	run: \|
	echo "Fetching proof parameters (cache miss)..."
	lotus fetch-params 8388608

	- name: Save proof parameters to cache
	if: steps.local-params.outputs.exists != 'true' && steps.cache-params.outputs.cache-hit != 'true' && github.ref == 'refs/heads/main'
	uses: actions/cache/save@v4
	with:
	path: /var/tmp/filecoin-proof-parameters
	key: proof-params-2k-v1

	- uses: actions/checkout@v4
	with:
	submodules: recursive

	- name: Start YugabyteDB
	id: start-yb
	run: \|
	docker stop $YB_CONTAINER 2>/dev/null \|\| true
	docker rm $YB_CONTAINER 2>/dev/null \|\| true
	docker run --rm --name $YB_CONTAINER -d yugabytedb/yugabyte:2024.1.2.0-b77 bin/yugabyted start --daemon=false
	for i in {1..60}; do
	if docker exec $YB_CONTAINER bin/yugabyted status 2>/dev/null \| grep -q Running; then
	echo "YugabyteDB is ready"
	break
	fi
	sleep 1
	done
	YB_IP=$(docker inspect $YB_CONTAINER --format '{{ .NetworkSettings.Networks.bridge.IPAddress }}')
	echo "yb_ip=$YB_IP" >> $GITHUB_OUTPUT

	- name: Setup build environment
	uses: ./.github/actions/setup-build-env
	with:
	go-version: ${{ env.GO_VERSION }}

	- name: Run other integration tests
	env:
	CURIO_HARMONYDB_HOSTS: ${{ steps.start-yb.outputs.yb_ip }}
	LOTUS_HARMONYDB_HOSTS: ${{ steps.start-yb.outputs.yb_ip }}
	run: \|
	go test -v --tags=debug -timeout 30m -parallel 4 \
	./itests/harmonydb_test.go \
	./itests/pdp_prove_test.go \
	./itests/indexstore_test.go \
	./itests/move_shared_test.go \
	./itests/local_test.go

	- name: Stop YugabyteDB
	if: always()
	run: docker stop $YB_CONTAINER 2>/dev/null \|\| true

	# Integration tests - serial tests (modify global state)
	test-itest-serial:
	runs-on: [self-hosted, docker]
	env:
	YB_CONTAINER: yugabyte-serial-${{ github.run_id }}
	steps:
	- name: Check local proof parameters
	id: local-params
	run: \|
	PARAMS_DIR="/var/tmp/filecoin-proof-parameters"
	if [ -d "$PARAMS_DIR" ] && [ "$(ls -A $PARAMS_DIR 2>/dev/null)" ]; then
	echo "exists=true" >> $GITHUB_OUTPUT
	else
	echo "exists=false" >> $GITHUB_OUTPUT
	fi

	- name: Restore proof parameters from cache
	if: steps.local-params.outputs.exists != 'true'
	id: cache-params
	uses: actions/cache/restore@v4
	with:
	path: /var/tmp/filecoin-proof-parameters
	key: proof-params-2k-v1
	restore-keys: proof-params-

	- name: Fetch proof parameters
	if: steps.local-params.outputs.exists != 'true' && steps.cache-params.outputs.cache-hit != 'true'
	run: \|
	echo "Fetching proof parameters (cache miss)..."
	lotus fetch-params 8388608

	- name: Save proof parameters to cache
	if: steps.local-params.outputs.exists != 'true' && steps.cache-params.outputs.cache-hit != 'true' && github.ref == 'refs/heads/main'
	uses: actions/cache/save@v4
	with:
	path: /var/tmp/filecoin-proof-parameters
	key: proof-params-2k-v1

	- uses: actions/checkout@v4
	with:
	submodules: recursive

	- name: Start YugabyteDB
	id: start-yb
	run: \|
	docker stop $YB_CONTAINER 2>/dev/null \|\| true
	docker rm $YB_CONTAINER 2>/dev/null \|\| true
	docker run --rm --name $YB_CONTAINER -d yugabytedb/yugabyte:2024.1.2.0-b77 bin/yugabyted start --daemon=false
	for i in {1..60}; do
	if docker exec $YB_CONTAINER bin/yugabyted status 2>/dev/null \| grep -q Running; then
	echo "YugabyteDB is ready"
	break
	fi
	sleep 1
	done
	YB_IP=$(docker inspect $YB_CONTAINER --format '{{ .NetworkSettings.Networks.bridge.IPAddress }}')
	echo "yb_ip=$YB_IP" >> $GITHUB_OUTPUT

	- name: Setup build environment
	uses: ./.github/actions/setup-build-env
	with:
	go-version: ${{ env.GO_VERSION }}

	- name: Run serial integration tests
	env:
	CURIO_HARMONYDB_HOSTS: ${{ steps.start-yb.outputs.yb_ip }}
	LOTUS_HARMONYDB_HOSTS: ${{ steps.start-yb.outputs.yb_ip }}
	run: go test -v --tags=debug,serial -timeout 30m ./itests/serial/...

	- name: Stop YugabyteDB
	if: always()
	run: docker stop $YB_CONTAINER 2>/dev/null \|\| true

	# Gen check - verify generated code is up to date
	# Optimized: run independent gen steps in parallel, then build once
	gen-check:
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4
	with:
	submodules: recursive

	- name: Setup build environment
	uses: ./.github/actions/setup-build-env
	with:
	go-version: ${{ env.GO_VERSION }}

	- name: Install Go tools in parallel
	run: \|
	go install golang.org/x/tools/cmd/goimports@latest &
	go install github.com/hannahhoward/cbor-gen-for@latest &
	go install github.com/swaggo/swag/cmd/swag@latest &
	wait

	- name: Run code generation (parallel where possible)
	run: \|
	# These can run in parallel - they don't depend on each other
	make api-gen &
	API_PID=$!

	make cfgdoc-gen &
	CFG_PID=$!

	make marketgen &
	MKT_PID=$!

	wait $API_PID \|\| exit 1
	wait $CFG_PID \|\| exit 1
	wait $MKT_PID \|\| exit 1

	# go-generate depends on api-gen completing
	make go-generate

	- name: Generate docs (requires binaries)
	run: \|
	# docsgen builds docgen-md and docgen-openrpc binaries
	make docsgen
	# docsgen-cli builds curio + sptool, then generates CLI docs
	make docsgen-cli

	- name: Fix imports and tidy
	run: \|
	go run ./scripts/fiximports
	go mod tidy

	- name: Check for changes
	run: git diff --quiet \|\| { git diff; exit 1; }

	# Supraseal build (kept separate - long running, different runner)
	build-supraseal-ubuntu24:
	runs-on: ubuntu-24.04
	steps:
	- uses: actions/checkout@v4
	with:
	submodules: recursive

	- name: Free up disk space
	run: \|
	sudo apt-get clean
	sudo rm -rf /usr/share/dotnet /opt/ghc "/usr/local/share/boost" "$AGENT_TOOLSDIRECTORY"

	- name: Install system dependencies
	run: \|
	sudo apt-get update
	sudo apt-get install -y \
	build-essential gcc-12 g++-12 nasm pkg-config \
	autoconf automake libtool libssl-dev libnuma-dev \
	uuid-dev libaio-dev libfuse3-dev libarchive-dev \
	libkeyutils-dev libncurses-dev libgmp-dev libconfig++-dev \
	python3 python3-pip python3-dev curl wget git xxd

	- name: Set up GCC 12
	run: \|
	sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 100
	sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-12 100

	- name: Cache CUDA installation
	id: cache-cuda
	uses: actions/cache@v4
	with:
	path: /usr/local/cuda
	key: cuda-toolkit-ubuntu-24.04-v1

	- name: Install CUDA Toolkit
	if: steps.cache-cuda.outputs.cache-hit != 'true'
	run: \|
	wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64/cuda-keyring_1.1-1_all.deb
	sudo dpkg -i cuda-keyring_1.1-1_all.deb
	sudo apt-get update && sudo apt-get -y install cuda-toolkit

	- name: Set up CUDA environment
	run: \|
	echo "/usr/local/cuda/bin" >> $GITHUB_PATH
	echo "CUDA_HOME=/usr/local/cuda" >> $GITHUB_ENV

	- name: Cache Python venv and SPDK
	uses: actions/cache@v4
	with:
	path: \|
	extern/supraseal/.venv
	extern/supraseal/deps/spdk-v24.05
	key: supraseal-deps-ubuntu24-${{ hashFiles('extern/supraseal/build.sh') }}

	- name: Build Supraseal
	working-directory: extern/supraseal
	run: \|
	export CC=gcc-12 CXX=g++-12 CUDA=/usr/local/cuda
	export PATH=/usr/local/cuda/bin:$PATH
	./build.sh

	- name: Verify binaries
	run: \|
	for bin in seal pc2 tree_r tree_r_cpu tree_d_cpu; do
	test -f extern/supraseal/bin/$bin \|\| exit 1
	done
	echo "✅ All Supraseal binaries built"

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Uh oh!

more efficient CI #1673

Workflow file

more efficient CI #1673

Uh oh!

Jobs

Run details

Workflow file for this run