Add downstream performance workflow #22
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: 'Downstream Performance' | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| suites: | |
| description: 'Downstream suites to run' | |
| required: true | |
| default: 'all' | |
| type: choice | |
| options: | |
| - all | |
| - kevm | |
| - kontrol | |
| pr_number: | |
| description: 'PR number to comment on when running manually' | |
| required: false | |
| type: string | |
| pull_request: | |
| types: | |
| - opened | |
| - reopened | |
| - synchronize | |
| - labeled | |
| - unlabeled | |
| - ready_for_review | |
| permissions: | |
| contents: read | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name || github.run_id }} | |
| cancel-in-progress: true | |
| jobs: | |
| select: | |
| name: 'Select downstream perf suites' | |
| runs-on: ubuntu-22.04 | |
| outputs: | |
| should_run: ${{ steps.decide.outputs.should_run }} | |
| run_kevm: ${{ steps.decide.outputs.run_kevm }} | |
| run_kontrol: ${{ steps.decide.outputs.run_kontrol }} | |
| reason: ${{ steps.decide.outputs.reason }} | |
| target_pr: ${{ steps.decide.outputs.target_pr }} | |
| steps: | |
| - name: 'Check out code' | |
| uses: actions/checkout@v5 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha || github.sha }} | |
| fetch-depth: 0 | |
| persist-credentials: false | |
| - name: 'Detect relevant changes' | |
| if: ${{ github.event_name == 'pull_request' }} | |
| uses: dorny/paths-filter@v3 | |
| id: changes | |
| with: | |
| filters: | | |
| downstream_perf: | |
| - 'booster/**' | |
| - 'kore/**' | |
| - 'kore-rpc-types/**' | |
| - 'flake.nix' | |
| - 'deps/**' | |
| - 'scripts/performance-tests-kevm.sh' | |
| - 'scripts/performance-tests-kontrol.sh' | |
| - 'scripts/compare.py' | |
| - 'scripts/collect-downstream-perf-results.sh' | |
| - 'scripts/downstream-perf-lib.sh' | |
| - '.github/actions/downstream-perf-suite/**' | |
| - '.github/actionlint.yaml' | |
| - '.github/workflows/downstream-perf.yml' | |
| - name: 'Decide suites' | |
| id: decide | |
| env: | |
| EVENT_NAME: ${{ github.event_name }} | |
| REQUESTED_SUITES: ${{ inputs.suites || 'all' }} | |
| REQUESTED_PR: ${{ inputs.pr_number || '' }} | |
| PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number || '' }} | |
| HAS_PERF_LABEL: ${{ github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'perf') }} | |
| RELEVANT_CHANGES: ${{ steps.changes.outputs.downstream_perf || 'false' }} | |
| run: | | |
| should_run=false | |
| run_kevm=false | |
| run_kontrol=false | |
| reason='not-requested' | |
| target_pr="$PULL_REQUEST_NUMBER" | |
| if [ -z "$target_pr" ] && [ -n "$REQUESTED_PR" ]; then | |
| target_pr="$REQUESTED_PR" | |
| fi | |
| if [ "$EVENT_NAME" = "workflow_dispatch" ]; then | |
| should_run=true | |
| reason='manual-dispatch' | |
| elif [ "$HAS_PERF_LABEL" = "true" ]; then | |
| should_run=true | |
| reason='perf-label' | |
| elif [ "$RELEVANT_CHANGES" = "true" ]; then | |
| should_run=true | |
| reason='relevant-path-change' | |
| fi | |
| if [ "$should_run" = "true" ]; then | |
| case "$REQUESTED_SUITES" in | |
| all) | |
| run_kevm=true | |
| run_kontrol=true | |
| ;; | |
| kevm) | |
| run_kevm=true | |
| ;; | |
| kontrol) | |
| run_kontrol=true | |
| ;; | |
| *) | |
| echo "Unknown suites selection: $REQUESTED_SUITES" >&2 | |
| exit 1 | |
| ;; | |
| esac | |
| fi | |
| { | |
| echo "should_run=$should_run" | |
| echo "run_kevm=$run_kevm" | |
| echo "run_kontrol=$run_kontrol" | |
| echo "reason=$reason" | |
| echo "target_pr=$target_pr" | |
| } >> "$GITHUB_OUTPUT" | |
| { | |
| echo "## Downstream perf selection" | |
| echo | |
| echo "- Event: $EVENT_NAME" | |
| echo "- Reason: $reason" | |
| echo "- KEVM: $run_kevm" | |
| echo "- Kontrol: $run_kontrol" | |
| echo "- Target PR: ${target_pr:-none}" | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| kevm: | |
| name: 'KEVM downstream performance' | |
| needs: select | |
| if: ${{ needs.select.outputs.should_run == 'true' && needs.select.outputs.run_kevm == 'true' }} | |
| runs-on: [self-hosted, linux, normal] | |
| timeout-minutes: 180 | |
| steps: | |
| - name: 'Check out code' | |
| uses: actions/checkout@v5 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha || github.sha }} | |
| fetch-depth: 0 | |
| submodules: recursive | |
| persist-credentials: false | |
| - name: 'Run KEVM downstream suite' | |
| uses: ./.github/actions/downstream-perf-suite | |
| with: | |
| suite: kevm | |
| script: scripts/performance-tests-kevm.sh | |
| artifact-name: kevm-downstream-perf-${{ github.run_id }} | |
| feature-branch-name: ${{ github.event.pull_request.head.ref || github.ref_name }} | |
| reason: ${{ needs.select.outputs.reason }} | |
| kontrol: | |
| name: 'Kontrol downstream performance' | |
| needs: select | |
| if: ${{ needs.select.outputs.should_run == 'true' && needs.select.outputs.run_kontrol == 'true' }} | |
| runs-on: [self-hosted, linux, normal] | |
| timeout-minutes: 180 | |
| steps: | |
| - name: 'Check out code' | |
| uses: actions/checkout@v5 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha || github.sha }} | |
| fetch-depth: 0 | |
| submodules: recursive | |
| persist-credentials: false | |
| - name: 'Run Kontrol downstream suite' | |
| uses: ./.github/actions/downstream-perf-suite | |
| with: | |
| suite: kontrol | |
| script: scripts/performance-tests-kontrol.sh | |
| artifact-name: kontrol-downstream-perf-${{ github.run_id }} | |
| feature-branch-name: ${{ github.event.pull_request.head.ref || github.ref_name }} | |
| reason: ${{ needs.select.outputs.reason }} | |
| report: | |
| name: 'Publish downstream perf PR comment' | |
| needs: | |
| - select | |
| - kevm | |
| - kontrol | |
| if: ${{ always() && needs.select.outputs.should_run == 'true' && needs.select.outputs.target_pr != '' && (needs.select.outputs.run_kevm != 'true' || needs.kevm.result != 'cancelled') && (needs.select.outputs.run_kontrol != 'true' || needs.kontrol.result != 'cancelled') }} | |
| runs-on: ubuntu-22.04 | |
| permissions: | |
| contents: read | |
| issues: write | |
| pull-requests: write | |
| steps: | |
| - name: 'Download KEVM results' | |
| if: ${{ needs.select.outputs.run_kevm == 'true' && needs.kevm.result != 'cancelled' && needs.kevm.result != 'skipped' }} | |
| uses: actions/download-artifact@v4 | |
| continue-on-error: true | |
| with: | |
| name: kevm-downstream-perf-${{ github.run_id }} | |
| path: downstream-perf/kevm | |
| - name: 'Download Kontrol results' | |
| if: ${{ needs.select.outputs.run_kontrol == 'true' && needs.kontrol.result != 'cancelled' && needs.kontrol.result != 'skipped' }} | |
| uses: actions/download-artifact@v4 | |
| continue-on-error: true | |
| with: | |
| name: kontrol-downstream-perf-${{ github.run_id }} | |
| path: downstream-perf/kontrol | |
| - name: 'Build PR comment body' | |
| env: | |
| RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| RUN_KEVM: ${{ needs.select.outputs.run_kevm }} | |
| RUN_KONTROL: ${{ needs.select.outputs.run_kontrol }} | |
| KEVM_RESULT: ${{ needs.kevm.result }} | |
| KONTROL_RESULT: ${{ needs.kontrol.result }} | |
| TRIGGER_REASON: ${{ needs.select.outputs.reason }} | |
| TARGET_PR: ${{ needs.select.outputs.target_pr }} | |
| run: | | |
| set -euo pipefail | |
| mkdir -p downstream-perf/report | |
| comment_file="downstream-perf/report/comment.md" | |
| { | |
| echo '<!-- downstream-perf-report -->' | |
| echo '## Downstream Performance' | |
| echo | |
| echo "- Trigger: $TRIGGER_REASON" | |
| echo "- Target PR: #$TARGET_PR" | |
| echo "- Workflow run: $RUN_URL" | |
| echo | |
| if [ "$RUN_KEVM" = "true" ]; then | |
| echo "### KEVM" | |
| echo | |
| echo "- Job result: $KEVM_RESULT" | |
| echo | |
| if [ -f downstream-perf/kevm/summary.md ]; then | |
| sed '1,/^$/d' downstream-perf/kevm/summary.md | |
| else | |
| echo 'No KEVM summary artifact was produced.' | |
| fi | |
| echo | |
| fi | |
| if [ "$RUN_KONTROL" = "true" ]; then | |
| echo "### Kontrol" | |
| echo | |
| echo "- Job result: $KONTROL_RESULT" | |
| echo | |
| if [ -f downstream-perf/kontrol/summary.md ]; then | |
| sed '1,/^$/d' downstream-perf/kontrol/summary.md | |
| else | |
| echo 'No Kontrol summary artifact was produced.' | |
| fi | |
| echo | |
| fi | |
| } > "$comment_file" | |
| - name: 'Upsert PR comment' | |
| uses: actions/github-script@v7 | |
| env: | |
| COMMENT_PATH: downstream-perf/report/comment.md | |
| TARGET_PR: ${{ needs.select.outputs.target_pr }} | |
| with: | |
| script: | | |
| const fs = require('fs'); | |
| const marker = '<!-- downstream-perf-report -->'; | |
| const body = fs.readFileSync(process.env.COMMENT_PATH, 'utf8'); | |
| const issue_number = Number(process.env.TARGET_PR); | |
| const owner = context.repo.owner; | |
| const repo = context.repo.repo; | |
| const comments = await github.paginate(github.rest.issues.listComments, { | |
| owner, | |
| repo, | |
| issue_number, | |
| per_page: 100, | |
| }); | |
| const existing = comments.find((comment) => comment.body && comment.body.includes(marker)); | |
| if (existing) { | |
| await github.rest.issues.updateComment({ | |
| owner, | |
| repo, | |
| comment_id: existing.id, | |
| body, | |
| }); | |
| } else { | |
| await github.rest.issues.createComment({ | |
| owner, | |
| repo, | |
| issue_number, | |
| body, | |
| }); | |
| } |