iceberg: write spec-compliant Avro manifests #27
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build Docker Images | |
| on: | |
| push: | |
| branches: | |
| - master | |
| - develop | |
| paths: | |
| - '.docker/**' | |
| - 'scripts/build_*.sh' | |
| - 'third_party/arrow' | |
| - 'third_party/orc' | |
| - 'third_party/lance-ffi' | |
| - 'third_party/lance' | |
| - 'cmake/gen_dists.py' | |
| - 'CMakeLists.txt' | |
| - 'src/**' | |
| - 'include/**' | |
| - '.github/workflows/docker-images.yml' | |
| workflow_dispatch: | |
| inputs: | |
| image: | |
| description: 'Which image to build' | |
| required: true | |
| type: choice | |
| options: | |
| - all | |
| - base | |
| - orc | |
| - lance | |
| - all-formats | |
| env: | |
| REGISTRY: ghcr.io | |
| IMAGE_PREFIX: ghcr.io/${{ github.repository_owner }}/tpch-cpp | |
| jobs: | |
| check-changes: | |
| name: Check Changed Files | |
| runs-on: ubuntu-22.04 | |
| if: github.event_name == 'push' | |
| outputs: | |
| base: ${{ steps.filter.outputs.base }} | |
| orc: ${{ steps.filter.outputs.orc }} | |
| lance: ${{ steps.filter.outputs.lance }} | |
| all-formats: ${{ steps.filter.outputs.all-formats }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Check changed files | |
| uses: dorny/paths-filter@v3 | |
| id: filter | |
| with: | |
| filters: | | |
| base: | |
| - '.docker/Dockerfile.base' | |
| - 'scripts/build_arrow_from_source.sh' | |
| - 'third_party/arrow/**' | |
| - 'third_party/xsimd/**' | |
| - '.github/workflows/docker-images.yml' | |
| orc: | |
| - '.docker/Dockerfile.orc' | |
| - '.docker/Dockerfile.base' | |
| - 'scripts/build_orc_from_source.sh' | |
| - 'third_party/orc/**' | |
| - '.github/workflows/docker-images.yml' | |
| lance: | |
| - '.docker/Dockerfile.lance' | |
| - '.docker/Dockerfile.base' | |
| - 'third_party/lance-ffi/**' | |
| - 'third_party/lance/**' | |
| - '.github/workflows/docker-images.yml' | |
| all-formats: | |
| - '.docker/Dockerfile.all' | |
| - 'third_party/arrow/**' | |
| - 'third_party/orc/**' | |
| - 'third_party/lance-ffi/**' | |
| - 'third_party/lance/**' | |
| - 'cmake/gen_dists.py' | |
| - 'CMakeLists.txt' | |
| - 'src/**' | |
| - 'include/**' | |
| - '.github/workflows/docker-images.yml' | |
| build-base: | |
| name: Build Base Image | |
| runs-on: ubuntu-22.04 | |
| needs: [check-changes] | |
| if: | | |
| always() && | |
| !cancelled() && | |
| (needs.check-changes.result == 'skipped' || needs.check-changes.outputs.base == 'true' || | |
| github.event.inputs.image == 'all' || | |
| github.event.inputs.image == 'base') | |
| permissions: | |
| contents: read | |
| packages: write | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| fetch-depth: 1 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Log in to GitHub Container Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ${{ env.REGISTRY }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Extract metadata | |
| id: meta | |
| uses: docker/metadata-action@v5 | |
| with: | |
| images: ${{ env.IMAGE_PREFIX }}-base | |
| tags: | | |
| type=raw,value=latest,enable={{is_default_branch}} | |
| type=ref,event=branch | |
| type=sha,prefix={{branch}}- | |
| - name: Build and push base image | |
| uses: docker/build-push-action@v5 | |
| with: | |
| context: . | |
| file: .docker/Dockerfile.base | |
| push: true | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| cache-from: | | |
| type=registry,ref=${{ env.IMAGE_PREFIX }}-base:buildcache | |
| type=gha | |
| cache-to: | | |
| type=registry,ref=${{ env.IMAGE_PREFIX }}-base:buildcache,mode=max | |
| type=gha,mode=max | |
| build-orc: | |
| name: Build ORC Image | |
| runs-on: ubuntu-22.04 | |
| needs: [check-changes, build-base] | |
| if: | | |
| always() && | |
| !cancelled() && | |
| (needs.check-changes.result == 'skipped' || needs.check-changes.outputs.orc == 'true' || | |
| github.event.inputs.image == 'all' || | |
| github.event.inputs.image == 'orc') && | |
| (needs.build-base.result == 'success' || needs.build-base.result == 'skipped') | |
| permissions: | |
| contents: read | |
| packages: write | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| fetch-depth: 1 | |
| - name: Resolve base image | |
| id: base-image | |
| run: | | |
| BRANCH_TAG=$(echo "${GITHUB_REF_NAME}" | tr '/' '-' | tr '[:upper:]' '[:lower:]') | |
| if [ "${{ needs.build-base.result }}" = "success" ]; then | |
| echo "image=${{ env.IMAGE_PREFIX }}-base:${BRANCH_TAG}" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "image=${{ env.IMAGE_PREFIX }}-base:latest" >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Log in to GitHub Container Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ${{ env.REGISTRY }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Extract metadata | |
| id: meta | |
| uses: docker/metadata-action@v5 | |
| with: | |
| images: ${{ env.IMAGE_PREFIX }}-orc | |
| tags: | | |
| type=raw,value=latest,enable={{is_default_branch}} | |
| type=ref,event=branch | |
| type=sha,prefix={{branch}}- | |
| - name: Build and push ORC image | |
| uses: docker/build-push-action@v5 | |
| with: | |
| context: . | |
| file: .docker/Dockerfile.orc | |
| push: true | |
| build-args: | | |
| BASE_IMAGE=${{ steps.base-image.outputs.image }} | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| cache-from: | | |
| type=registry,ref=${{ env.IMAGE_PREFIX }}-orc:buildcache | |
| type=registry,ref=${{ env.IMAGE_PREFIX }}-base:buildcache | |
| type=gha | |
| cache-to: | | |
| type=registry,ref=${{ env.IMAGE_PREFIX }}-orc:buildcache,mode=max | |
| type=gha,mode=max | |
| build-lance: | |
| name: Build Lance Image | |
| runs-on: ubuntu-22.04 | |
| needs: [check-changes, build-base] | |
| if: | | |
| always() && | |
| !cancelled() && | |
| (needs.check-changes.result == 'skipped' || needs.check-changes.outputs.lance == 'true' || | |
| github.event.inputs.image == 'all' || | |
| github.event.inputs.image == 'lance') && | |
| (needs.build-base.result == 'success' || needs.build-base.result == 'skipped') | |
| permissions: | |
| contents: read | |
| packages: write | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| fetch-depth: 1 | |
| - name: Resolve base image | |
| id: base-image | |
| run: | | |
| BRANCH_TAG=$(echo "${GITHUB_REF_NAME}" | tr '/' '-' | tr '[:upper:]' '[:lower:]') | |
| if [ "${{ needs.build-base.result }}" = "success" ]; then | |
| echo "image=${{ env.IMAGE_PREFIX }}-base:${BRANCH_TAG}" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "image=${{ env.IMAGE_PREFIX }}-base:latest" >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Log in to GitHub Container Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ${{ env.REGISTRY }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Extract metadata | |
| id: meta | |
| uses: docker/metadata-action@v5 | |
| with: | |
| images: ${{ env.IMAGE_PREFIX }}-lance | |
| tags: | | |
| type=raw,value=latest,enable={{is_default_branch}} | |
| type=ref,event=branch | |
| type=sha,prefix={{branch}}- | |
| - name: Build and push Lance image | |
| uses: docker/build-push-action@v5 | |
| with: | |
| context: . | |
| file: .docker/Dockerfile.lance | |
| push: true | |
| build-args: | | |
| BASE_IMAGE=${{ steps.base-image.outputs.image }} | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| cache-from: | | |
| type=registry,ref=${{ env.IMAGE_PREFIX }}-lance:buildcache | |
| type=registry,ref=${{ env.IMAGE_PREFIX }}-base:buildcache | |
| type=gha | |
| cache-to: | | |
| type=registry,ref=${{ env.IMAGE_PREFIX }}-lance:buildcache,mode=max | |
| type=gha,mode=max | |
| build-all-formats: | |
| name: Build All-Formats Image | |
| runs-on: ubuntu-22.04 | |
| needs: [check-changes] | |
| if: | | |
| always() && | |
| !cancelled() && | |
| (needs.check-changes.result == 'skipped' || | |
| needs.check-changes.outputs.all-formats == 'true' || | |
| github.event.inputs.image == 'all' || | |
| github.event.inputs.image == 'all-formats') | |
| permissions: | |
| contents: read | |
| packages: write | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| fetch-depth: 1 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Log in to GitHub Container Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ${{ env.REGISTRY }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Extract metadata | |
| id: meta | |
| uses: docker/metadata-action@v5 | |
| with: | |
| images: ${{ env.IMAGE_PREFIX }}-all | |
| tags: | | |
| type=raw,value=latest,enable={{is_default_branch}} | |
| type=ref,event=branch | |
| type=sha,prefix={{branch}}- | |
| - name: Build and push all-formats image | |
| uses: docker/build-push-action@v5 | |
| with: | |
| context: . | |
| file: .docker/Dockerfile.all | |
| push: true | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| # Each named build stage gets its own cache entry so that | |
| # Arrow, ORC, and Rust can be cached independently. | |
| cache-from: | | |
| type=registry,ref=${{ env.IMAGE_PREFIX }}-all:buildcache-deps | |
| type=registry,ref=${{ env.IMAGE_PREFIX }}-all:buildcache-orc | |
| type=registry,ref=${{ env.IMAGE_PREFIX }}-all:buildcache-rust | |
| type=registry,ref=${{ env.IMAGE_PREFIX }}-all:buildcache | |
| type=gha | |
| cache-to: | | |
| type=registry,ref=${{ env.IMAGE_PREFIX }}-all:buildcache,mode=max | |
| type=gha,mode=max |