[LLVM] add wasm32 wheels #389
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | |
| # See https://llvm.org/LICENSE.txt for license information. | |
| # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | |
| # Copyright (c) 2024. | |
| name: "Build, test, release mlir-python-bindings" | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| force_debug_with_tmate: | |
| type: boolean | |
| description: 'Run the build with tmate session' | |
| required: false | |
| default: false | |
| debug_with_tmate: | |
| type: boolean | |
| description: 'Run the build with a tmate session ONLY in case of failure' | |
| required: false | |
| default: false | |
| workflow_call: | |
| inputs: | |
| wheel_version: | |
| description: 'wheel version' | |
| type: string | |
| required: false | |
| default: '' | |
| workflow_call: | |
| description: 'To distinguish workflow_call from regular push' | |
| type: boolean | |
| required: false | |
| default: true | |
| workflow_caller_run_id: | |
| description: '' | |
| type: string | |
| required: false | |
| default: '' | |
| pull_request: | |
| branches: | |
| - main | |
| paths: | |
| - ".github/workflows/build_mlir_python_bindings_wheel.yml" | |
| - "projects/mlir-python-bindings/**" | |
| push: | |
| branches: | |
| - main | |
| paths: | |
| - ".github/workflows/build_mlir_python_bindings_wheel.yml" | |
| - "projects/mlir-python-bindings/**" | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.number || github.sha }}-build_mlir_python_bindings_wheel | |
| cancel-in-progress: true | |
| jobs: | |
| build-mlir-python-bindings: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - name: "ubuntu_x86_64" | |
| runs-on: "ubuntu-22.04" | |
| os: "ubuntu" | |
| host-arch: "x86_64" | |
| - name: "ubuntu_aarch64" | |
| runs-on: "ubuntu-22.04-arm" | |
| os: "ubuntu" | |
| host-arch: "aarch64" | |
| - name: "windows_amd64" | |
| runs-on: "windows-2022" | |
| os: "windows" | |
| host-arch: "amd64" | |
| - name: "macos_arm64" | |
| runs-on: "macos-14" | |
| os: "macos" | |
| host-arch: "arm64" | |
| runs-on: ${{ matrix.runs-on }} | |
| name: "Build mlir-python-bindings ${{ matrix.name }}" | |
| defaults: | |
| run: | |
| shell: bash | |
| permissions: | |
| id-token: write | |
| contents: write | |
| env: | |
| # either the PR number or `branch-N` where N always increments | |
| cache-key: mlir_python_bindings_${{ matrix.name }}_clang_${{ format('{0}-{1}', github.ref_name, github.run_number) }} | |
| steps: | |
| - name: "Check out repository" | |
| uses: actions/[email protected] | |
| - name: "Setup base" | |
| uses: ./.github/actions/setup_base | |
| id: setup_base | |
| with: | |
| cache-key: ${{ env.cache-key }} | |
| restore-key: "mlir_python_bindings_${{ matrix.name }}_clang" | |
| os: ${{ matrix.os }} | |
| host-arch: ${{ matrix.host-arch }} | |
| target-arch: ${{ matrix.target-arch }} | |
| - name: "Download MLIR from releases" | |
| if: ${{ !inputs.workflow_call }} | |
| run: | | |
| pip download mlir-wheel -f https://llvm.github.io/eudsl | |
| unzip mlir_wheel*.whl -d llvm-install | |
| WHEEL_NAME=$(ls mlir_wheel*) | |
| pip install pkginfo | |
| WHEEL_VERSION=$($python3_command -c "from pkginfo import Wheel; print(Wheel('$WHEEL_NAME').version)") | |
| echo "WHEEL_VERSION=$WHEEL_VERSION" >> $GITHUB_ENV | |
| - name: Download LLVM distro artifacts | |
| if: ${{ inputs.workflow_call }} | |
| uses: dawidd6/action-download-artifact@v11 | |
| with: | |
| name: mlir_${{ matrix.os == 'ubuntu' && 'manylinux' || matrix.os }}_${{ matrix.host-arch }}_artifact | |
| path: "./" | |
| run_id: ${{ inputs.workflow_caller_run_id }} | |
| - name: "Set WHEEL_VERSION env var" | |
| if: ${{ inputs.workflow_call }} | |
| run: | | |
| echo "WHEEL_VERSION=${{ inputs.wheel_version }}" >> $GITHUB_ENV | |
| - name: "Configure build" | |
| run: | | |
| if [[ "${{ matrix.os }}" == "ubuntu" ]]; then | |
| echo "LLVM_DIR=/host/$PWD/llvm-install/lib/cmake/llvm" >> $GITHUB_ENV | |
| echo "MLIR_DIR=/host/$PWD/llvm-install/lib/cmake/mlir" >> $GITHUB_ENV | |
| echo "Clang_DIR=/host/$PWD/llvm-install/lib/cmake/clang" >> $GITHUB_ENV | |
| echo "PIP_FIND_LINKS=/host/$PWD/wheelhouse" >> $GITHUB_ENV | |
| echo "CMAKE_PREFIX_PATH=/host/$PWD/llvm-install" >> $GITHUB_ENV | |
| else | |
| echo "LLVM_DIR=$PWD/llvm-install/lib/cmake/llvm" >> $GITHUB_ENV | |
| echo "MLIR_DIR=$PWD/llvm-install/lib/cmake/mlir" >> $GITHUB_ENV | |
| echo "Clang_DIR=$PWD/llvm-install/lib/cmake/clang" >> $GITHUB_ENV | |
| echo "PIP_FIND_LINKS=$PWD/wheelhouse" >> $GITHUB_ENV | |
| echo "CMAKE_PREFIX_PATH=$PWD/llvm-install" >> $GITHUB_ENV | |
| fi | |
| $python3_command -m pip install cibuildwheel | |
| if [[ "${{ matrix.os }}" == "ubuntu" ]] && [[ "${{ matrix.host-arch }}" == "aarch64" ]]; then | |
| sudo apt-get install libarchive-dev antlr libxml2-dev libxslt-dev libcurl4-openssl-dev | |
| fi | |
| if [[ "${{ matrix.os }}" == "macos" ]]; then | |
| brew install antlr boost libarchive | |
| fi | |
| - name: "Build mlir-python-bindings" | |
| run: | | |
| $python3_command -m cibuildwheel "$PWD/projects/mlir-python-bindings" --output-dir wheelhouse | |
| - name: Upload artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: mlir_python_bindings_${{ matrix.name }}_artifact | |
| path: wheelhouse/*.whl | |
| if-no-files-found: warn | |
| - name: "Setup tmate session" | |
| if: (failure() && inputs.debug_with_tmate) || inputs.force_debug_with_tmate | |
| uses: mxschmitt/[email protected] | |
| with: | |
| limit-access-to-actor: true | |
| install-dependencies: ${{ startsWith(matrix.os, 'macos') || startsWith(matrix.os, 'windows') || startsWith(matrix.os, 'ubuntu') }} | |
| test-mlir-python-bindings: | |
| if: github.event_name == 'pull_request' | |
| needs: [build-mlir-python-bindings] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| runs-on: [ | |
| "ubuntu-22.04", | |
| "ubuntu-22.04-arm", | |
| "macos-14", | |
| "windows-2022" | |
| ] | |
| python-version: [ | |
| "3.10", "3.11", "3.12", | |
| "3.13", "3.14", "3.14t" | |
| ] | |
| include: [ | |
| {runs-on: "ubuntu-22.04", name: "ubuntu_x86_64", os: "ubuntu"}, | |
| {runs-on: "ubuntu-22.04-arm", name: "ubuntu_aarch64", os: "ubuntu"}, | |
| {runs-on: "windows-2022", name: "windows_amd64", os: "windows"}, | |
| {runs-on: "macos-14", name: "macos_arm64", os: "macos"}, | |
| ] | |
| exclude: | |
| # <frozen importlib._bootstrap>:491: Warning: Numpy built with MINGW-W64 on Windows 64 bits is experimental, and only available for testing. You are advised not to use it for production. | |
| - runs-on: windows-2022 | |
| python-version: "3.14" | |
| - runs-on: windows-2022 | |
| python-version: "3.14t" | |
| - runs-on: macos-14 | |
| python-version: "3.10" | |
| - runs-on: macos-14 | |
| python-version: "3.11" | |
| - runs-on: macos-14 | |
| python-version: "3.12" | |
| - runs-on: macos-14 | |
| python-version: "3.13" | |
| - runs-on: macos-14 | |
| python-version: "3.14" | |
| runs-on: ${{ matrix.runs-on }} | |
| name: "Test mlir-python-bindings ${{ matrix.name }} ${{ matrix.python-version }}" | |
| defaults: | |
| run: | |
| shell: bash | |
| steps: | |
| - name: "Check out repository" | |
| uses: actions/[email protected] | |
| with: | |
| submodules: false | |
| - name: "Get submodules" | |
| run: git submodule update --init --depth=1 | |
| - name: "Install Python" | |
| uses: actions/[email protected] | |
| with: | |
| python-version: "${{ matrix.python-version }}" | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: mlir_python_bindings_${{ matrix.name }}_artifact | |
| path: wheelhouse | |
| - name: "Install mlir-python-bindings" | |
| run: python -m pip install mlir-python-bindings -f wheelhouse | |
| - name: "Test mlir-python-bindings" | |
| run: | | |
| pip install mlir-python-bindings numpy -v -f wheelhouse | |
| cd third_party/llvm-project/mlir/test/python/dialects | |
| for f in *.py; do | |
| if [[ "$f" == "python_test.py" ]]; then | |
| continue | |
| fi | |
| python $f | |
| done | |
| release-mlir-python-bindings: | |
| if: (github.event_name == 'push' && github.ref_name == 'main') || github.event_name == 'workflow_dispatch' | |
| needs: [build-mlir-python-bindings] | |
| runs-on: "ubuntu-22.04" | |
| permissions: | |
| id-token: write | |
| contents: write | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| name: [ | |
| "ubuntu_x86_64", | |
| "ubuntu_aarch64", | |
| "macos_arm64", | |
| # "macos_x86_64", | |
| "windows_amd64" | |
| ] | |
| name: "Release mlir-python-bindings ${{ matrix.name }}" | |
| steps: | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: mlir_python_bindings_${{ matrix.name }}_artifact | |
| path: wheelhouse | |
| - name: Release current commit | |
| uses: ncipollo/[email protected] | |
| with: | |
| artifacts: "wheelhouse/mlir_python_bindings*.whl" | |
| token: "${{ secrets.GITHUB_TOKEN }}" | |
| tag: mlir-python-bindings | |
| name: mlir-python-bindings | |
| removeArtifacts: false | |
| allowUpdates: true | |
| replacesArtifacts: true | |
| makeLatest: true | |
| omitBody: true | |
| build-mlir-python-bindings-wasm: | |
| runs-on: ubuntu-22.04 | |
| name: "Build mlir-python-bindings wasm" | |
| defaults: | |
| run: | |
| shell: bash | |
| permissions: | |
| id-token: write | |
| contents: write | |
| env: | |
| cache-key: mlir_python_bindings_wasm_emscripten_${{ format('{0}-{1}', github.ref_name, github.run_number) }} | |
| steps: | |
| - name: "Check out repository" | |
| uses: actions/[email protected] | |
| with: | |
| submodules: true | |
| - name: "Setup base" | |
| uses: ./.github/actions/setup_base | |
| id: setup_base | |
| with: | |
| cache-key: ${{ env.cache-key }} | |
| restore-key: "mlir_python_bindings_wasm_emscripten" | |
| os: "ubuntu" | |
| host-arch: "x86_64" | |
| target-arch: "wasm32" | |
| - name: "Pip download MLIR from releases" | |
| if: ${{ !inputs.workflow_call }} | |
| run: | | |
| pip download mlir-wheel --plat pyodide_2024_0_wasm32 --no-deps --python-version 3.12 -f https://llvm.github.io/eudsl | |
| unzip mlir_wheel*.whl -d llvm-install | |
| echo "CMAKE_PREFIX_PATH=$PWD" >> $GITHUB_ENV | |
| WHEEL_NAME=$(ls mlir_wheel*) | |
| pip install pkginfo | |
| WHEEL_VERSION=$($python3_command -c "from pkginfo import Wheel; print(Wheel('$WHEEL_NAME').version)") | |
| echo "WHEEL_VERSION=$WHEEL_VERSION" >> $GITHUB_ENV | |
| - name: Download LLVM distro artifacts | |
| if: ${{ inputs.workflow_call }} | |
| uses: dawidd6/action-download-artifact@v11 | |
| with: | |
| name: mlir_wheel_wasm_artifact | |
| path: "./" | |
| run_id: ${{ inputs.workflow_caller_run_id }} | |
| - name: "Set WHEEL_VERSION env var" | |
| if: ${{ inputs.workflow_call }} | |
| run: | | |
| echo "WHEEL_VERSION=${{ inputs.wheel_version }}" >> $GITHUB_ENV | |
| - name: Download mlir-native-tools artifacts | |
| if: ${{ inputs.workflow_call }} | |
| uses: dawidd6/action-download-artifact@v11 | |
| with: | |
| name: mlir_native_tools_manylinux_x86_64_artifact | |
| path: "./" | |
| run_id: ${{ inputs.workflow_caller_run_id }} | |
| - name: Pip download mlir-native-tools | |
| if: ${{ !inputs.workflow_call }} | |
| run: | | |
| pip download mlir_native_tools -f https://llvm.github.io/eudsl | |
| - name: "Configure build" | |
| run: | | |
| echo "LLVM_DIR=$PWD/llvm-install/lib/cmake/llvm" >> $GITHUB_ENV | |
| echo "MLIR_DIR=$PWD/llvm-install/lib/cmake/mlir" >> $GITHUB_ENV | |
| echo "Clang_DIR=$PWD/llvm-install/lib/cmake/clang" >> $GITHUB_ENV | |
| echo "CMAKE_PREFIX_PATH=$PWD/llvm-install" >> $GITHUB_ENV | |
| unzip -o -j mlir_native_tools-*whl -d mlir_native_tools | |
| rm -rf mlir_native_tools-*whl | |
| export LLVM_NATIVE_TOOL_DIR="$PWD/mlir_native_tools" | |
| echo "LLVM_NATIVE_TOOL_DIR=$LLVM_NATIVE_TOOL_DIR" >> $GITHUB_ENV | |
| echo "LLVM_TABLEGEN=$LLVM_NATIVE_TOOL_DIR/llvm-tblgen" >> $GITHUB_ENV | |
| echo "MLIR_TABLEGEN=$LLVM_NATIVE_TOOL_DIR/mlir-tblgen" >> $GITHUB_ENV | |
| echo "MLIR_LINALG_ODS_YAML_GEN=$LLVM_NATIVE_TOOL_DIR/mlir-linalg-ods-yaml-gen" >> $GITHUB_ENV | |
| - name: build wasm python wheel | |
| run: | | |
| source "$EMSDK/emsdk_env.sh" | |
| echo "PATH=$EMSDK/ccache/git-emscripten_64bit/bin:$PATH" >> $GITHUB_ENV | |
| export CCACHE_COMPILERCHECK="string:$($EMSDK/upstream/bin/clang --version | head -n 1)" | |
| echo "CCACHE_COMPILERCHECK=$CCACHE_COMPILERCHECK" >> $GITHUB_ENV | |
| export CCACHE="$EMSDK/ccache/git-emscripten_64bit/bin/ccache" | |
| export CMAKE_C_COMPILER_LAUNCHER="$CCACHE" | |
| export CMAKE_CXX_COMPILER_LAUNCHER="$CCACHE" | |
| $CCACHE -z | |
| pyodide build projects/mlir-python-bindings-wasm -o wheelhouse --compression-level 10 | |
| $CCACHE -s | |
| - name: Upload ccache log | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: llvm_ccache_log_mlir_python_bindings_wasm32 | |
| path: '/tmp/ccache.log' | |
| - name: "Save cache" | |
| uses: actions/cache/save@v3 | |
| if: (!cancelled() && github.event_name == 'push' && github.ref_name == 'main') | |
| with: | |
| path: ${{ steps.setup_base.outputs.cache-dir }} | |
| key: ${{ env.cache-key }} | |
| - name: Install pyodide and test | |
| shell: bash | |
| run: | | |
| pyodide venv venv | |
| . venv/bin/activate | |
| pip install mlir-python-bindings -f wheelhouse | |
| pushd third_party/llvm-project/mlir/test/python/dialects | |
| for f in *.py; do | |
| if [[ "$f" == "python_test.py" ]]; then | |
| continue | |
| fi | |
| python $f | |
| done | |
| popd | |
| python projects/mlir-python-bindings-wasm/test_wasm_execution_engine.py | |
| - name: Upload wasm32 wheels | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| path: wheelhouse/*.whl | |
| name: build_artifact_python_bindings-ubuntu-wasm | |
| - name: Release current commit | |
| if: (github.event_name == 'push' && github.ref_name == 'main') || github.event_name == 'workflow_dispatch' | |
| uses: ncipollo/[email protected] | |
| with: | |
| artifacts: "wheelhouse/mlir_python_bindings*.whl" | |
| token: "${{ secrets.GITHUB_TOKEN }}" | |
| tag: mlir-python-bindings | |
| name: mlir-python-bindings | |
| removeArtifacts: false | |
| allowUpdates: true | |
| replacesArtifacts: true | |
| makeLatest: true | |
| omitBody: true | |
| call-build-eudsl-python-extras: | |
| needs: [build-mlir-python-bindings] | |
| permissions: | |
| contents: write | |
| id-token: write | |
| pages: write | |
| uses: ./.github/workflows/build_test_release_eudsl_python_extras.yml | |
| secrets: inherit # pass all secrets | |
| with: | |
| workflow_call: true | |
| workflow_caller_run_id: ${{ github.run_id }} | |
| call-deploy-pip-page: | |
| if: (github.event_name == 'push' && github.ref_name == 'main') || github.event_name == 'workflow_dispatch' | |
| needs: [release-mlir-python-bindings] | |
| permissions: | |
| contents: read | |
| id-token: write | |
| pages: write | |
| uses: ./.github/workflows/deploy_pip_page.yml | |
| secrets: inherit # pass all secrets |