From f043d7938813fe3f7e1eb5db4bbbac980c2e752d Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Sun, 25 Oct 2026 18:30:40 -0400 Subject: [PATCH] sonarcloud: build the instrumented archive once and shard tests from it --- .github/workflows/sonarcloud.yml | 134 ++++++++++++++++++++++--------- README.md | 12 +-- tests/test_sonarcloud.py | 23 +++++- 3 files changed, 123 insertions(+), 46 deletions(-) diff --git a/.github/workflows/sonarcloud.yml b/.github/workflows/sonarcloud.yml index 6629f68..ace818c 100644 --- a/.github/workflows/sonarcloud.yml +++ b/.github/workflows/sonarcloud.yml @@ -20,8 +20,8 @@ on: default: 90 shards: description: >- - Split the Rust coverage run into this many parallel cargo-nextest partitions - (1 keeps the single `cargo llvm-cov` job; default 4). Doctests are not run when sharded. + Run the Rust tests as this many parallel partitions of one prebuilt instrumented + archive (1 keeps the single `cargo llvm-cov` job; default 4). Doctests are not run when sharded. type: number required: false default: 4 @@ -61,12 +61,57 @@ jobs: echo 'matrix={"shard":[]}' >>"$GITHUB_OUTPUT" fi - coverage: - name: Coverage ${{ matrix.shard }}/${{ inputs.shards }} + build: + name: Build instrumented test archive needs: plan if: needs.plan.outputs.matrix != '{"shard":[]}' runs-on: ubuntu-latest timeout-minutes: ${{ inputs.timeout-minutes }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Free disk space + run: sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL + - name: Install Linux build dependencies + uses: storytold/craft-common-actions/actions/install-apt-packages@main + with: + packages: >- + pkg-config libxkbcommon-dev libwayland-dev libx11-dev libxcursor-dev libxrandr-dev + libxi-dev libgtk-3-dev libfontconfig1-dev libfreetype-dev libasound2-dev + libgl1-mesa-dev mesa-vulkan-drivers + - name: Install Rust and cargo-llvm-cov + run: | + rustup toolchain install stable --profile minimal --component llvm-tools-preview + rustup default stable + - uses: taiki-e/install-action@8cca0918f8471c33515202b73f56ab27f2bed283 # v2.87.29 + with: + tool: cargo-llvm-cov,cargo-nextest + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 + with: + save-if: ${{ github.ref == 'refs/heads/main' }} + - name: Build instrumented test archive + env: + CARGO_TERM_COLOR: always + # Coverage comes from instrumentation, not DWARF; debuginfo is most of target/. + CARGO_PROFILE_DEV_DEBUG: "0" + run: | + # show-env sets the instrumentation flags the same way the unsharded run gets them. + # shellcheck disable=SC1090 # generated by cargo-llvm-cov at runtime + source <(cargo llvm-cov show-env --sh) + cargo nextest archive --workspace --archive-file nextest-archive.tar.zst + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: nextest-archive + path: nextest-archive.tar.zst + retention-days: 1 + + coverage: + name: Tests ${{ matrix.shard }}/${{ inputs.shards }} + needs: [plan, build] + if: needs.plan.outputs.matrix != '{"shard":[]}' + runs-on: ubuntu-latest + timeout-minutes: ${{ inputs.timeout-minutes }} strategy: fail-fast: false matrix: ${{ fromJSON(needs.plan.outputs.matrix) }} @@ -78,8 +123,6 @@ jobs: uses: storytold/craft-common-actions/actions/checkout-craft-fonts@main with: ref: ${{ inputs.fonts-ref }} - - name: Free disk space - run: sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL - name: Install Linux build dependencies uses: storytold/craft-common-actions/actions/install-apt-packages@main with: @@ -87,41 +130,36 @@ jobs: pkg-config libxkbcommon-dev libwayland-dev libx11-dev libxcursor-dev libxrandr-dev libxi-dev libgtk-3-dev libfontconfig1-dev libfreetype-dev libasound2-dev libgl1-mesa-dev mesa-vulkan-drivers - - name: Install Rust - run: | - rustup toolchain install stable --profile minimal --component llvm-tools-preview - rustup default stable - uses: taiki-e/install-action@8cca0918f8471c33515202b73f56ab27f2bed283 # v2.87.29 with: - tool: cargo-llvm-cov,cargo-nextest - - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 + tool: cargo-nextest + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: - # Every shard builds the same instrumented targets, so they share one cache entry. - shared-key: sonar-coverage - save-if: ${{ github.ref == 'refs/heads/main' && matrix.shard == 1 }} - - name: Generate Rust coverage for this shard + name: nextest-archive + path: archive + - name: Run this shard's tests # A red test must not hide the static analysis, so keep going and scan what was measured. continue-on-error: true env: - CARGO_TERM_COLOR: always CRAFT_FONTS_DIR: ${{ github.workspace }}/craft-fonts - CARGO_PROFILE_DEV_DEBUG: "0" - run: > - cargo llvm-cov nextest --workspace --partition count:${{ matrix.shard }}/${{ inputs.shards }} - --no-fail-fast --ignore-run-fail --lcov --output-path lcov-${{ matrix.shard }}.info - --ignore-filename-regex "$COVERAGE_IGNORE" + # Each instrumented binary writes its profile here; the scan job merges them. + LLVM_PROFILE_FILE: ${{ github.workspace }}/profraw/shard-${{ matrix.shard }}-%p-%m.profraw + run: | + mkdir -p profraw + cargo nextest run --archive-file archive/nextest-archive.tar.zst \ + --partition count:${{ matrix.shard }}/${{ inputs.shards }} --no-fail-fast - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: - name: lcov-${{ matrix.shard }} - path: lcov-${{ matrix.shard }}.info + name: profraw-${{ matrix.shard }} + path: profraw/ if-no-files-found: ignore retention-days: 1 sonarcloud: name: SonarCloud Scan - needs: coverage - # Scan even if a shard failed outright; the plan/coverage jobs are skipped when shards is 1. - if: ${{ !cancelled() }} + needs: [plan, build, coverage] + # Scan unless a prerequisite failed outright; a red test shard still lets the scan measure what ran. + if: ${{ !cancelled() && contains(fromJSON('["success","skipped"]'), needs.plan.result) && contains(fromJSON('["success","skipped"]'), needs.build.result) && contains(fromJSON('["success","skipped"]'), needs.coverage.result) }} runs-on: ubuntu-latest timeout-minutes: ${{ inputs.timeout-minutes }} steps: @@ -155,13 +193,13 @@ jobs: fi printf 'key=%s\n' "$PROJECT_KEY" >>"$GITHUB_OUTPUT" - name: Check out craft-fonts - if: steps.token.outputs.available == 'true' + if: steps.token.outputs.available == 'true' && inputs.shards <= 1 uses: storytold/craft-common-actions/actions/checkout-craft-fonts@main with: ref: ${{ inputs.fonts-ref }} - name: Free disk space # The instrumented workspace build overflows the ~14 GB free on a stock runner. - if: steps.token.outputs.available == 'true' && hashFiles('Cargo.toml') != '' + if: steps.token.outputs.available == 'true' && hashFiles('Cargo.toml') != '' && inputs.shards <= 1 run: sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL - name: Install Linux build dependencies if: steps.token.outputs.available == 'true' && hashFiles('Cargo.toml') != '' && inputs.shards <= 1 @@ -197,21 +235,41 @@ jobs: run: > cargo llvm-cov --workspace --ignore-run-fail --lcov --output-path lcov.info --ignore-filename-regex "$COVERAGE_IGNORE" - - name: Merge shard coverage + - name: Install LLVM tools + if: steps.token.outputs.available == 'true' && inputs.shards > 1 + run: rustup component add llvm-tools-preview + - name: Download shard profiles and archive if: steps.token.outputs.available == 'true' && inputs.shards > 1 uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: - pattern: lcov-* - path: lcov-shards + pattern: profraw-* + path: profraw merge-multiple: true - - name: Combine shard reports + - name: Download test archive if: steps.token.outputs.available == 'true' && inputs.shards > 1 + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: nextest-archive + path: archive + - name: Combine shard coverage + if: steps.token.outputs.available == 'true' && inputs.shards > 1 + shell: bash run: | - # lcov sums hit counts for the same line across shards; a shard that never ran adds nothing. - sudo apt-get install -y --no-install-recommends lcov - set -- lcov-shards/lcov-*.info - [ -e "$1" ] || exit 0 - lcov "${@/#/-a}" -o lcov.info 2>&1 | tail -3 + set -euo pipefail + llvm_bin="$(rustc --print sysroot)/lib/rustlib/$(rustc -vV | sed -n 's/^host: //p')/bin" + mkdir -p coverage-archive + tar --zstd -xf archive/nextest-archive.tar.zst -C coverage-archive + objects=() + while IFS= read -r -d '' file; do objects+=("$file"); done < <(find coverage-archive -type f -executable -print0) + if [ "${#objects[@]}" -eq 0 ]; then + echo "::error::No instrumented test binaries in the archive" + exit 1 + fi + "$llvm_bin/llvm-profdata" merge -sparse profraw/*.profraw -o coverage.profdata + extra=() + for object in "${objects[@]:1}"; do extra+=(-object "$object"); done + "$llvm_bin/llvm-cov" export -format=lcov -instr-profile=coverage.profdata \ + "${objects[0]}" "${extra[@]}" -ignore-filename-regex="$COVERAGE_IGNORE" >lcov.info - name: SonarCloud Scan if: steps.token.outputs.available == 'true' uses: SonarSource/sonarqube-scan-action@d209202bc7d53ff1cc128f7f907dac145c9d6ae9 # v8.3.0 diff --git a/README.md b/README.md index d7238cb..114c22d 100644 --- a/README.md +++ b/README.md @@ -494,12 +494,12 @@ changes the job's time budget without dropping tests or coverage. Pass `fonts-ref` at the app's tested craft-fonts commit when its tests need those fonts. The scan still needs to finish and pass its quality gate. -Workspaces split the coverage run across four parallel `cargo nextest` -partitions by default. Pass `shards: 1` to keep the single `cargo llvm-cov` -job, or another count to change the split. Each shard uploads its own lcov -report and the scan job merges them with `lcov`. Sharded runs skip doctests, -because nextest does not run them, and every shard still compiles the whole -workspace, so more shards stop helping once compile time dominates. +Workspaces run their tests as four parallel `cargo nextest` partitions by +default. One job builds the instrumented test archive; each partition runs from +that archive and uploads its profile data, and the scan job merges the profiles +with the rustup `llvm-profdata` and `llvm-cov` tools. Pass `shards: 1` to keep +the single `cargo llvm-cov` job. Sharded runs skip doctests, because nextest +does not run them. The new limits leave headroom over sampled Craft app job timings; the existing FreeBSD and ARM64 limits are retained. Treat these as initial limits, review diff --git a/tests/test_sonarcloud.py b/tests/test_sonarcloud.py index 713defa..96cd6c7 100644 --- a/tests/test_sonarcloud.py +++ b/tests/test_sonarcloud.py @@ -84,7 +84,7 @@ class SonarCloudTests(unittest.TestCase): self.assertIn('--output-path lcov.info', coverage['run']) self.assertEqual(coverage['continue-on-error'], 'true') - def test_sharding_is_opt_in_and_keeps_the_unsharded_path(self): + def test_sharding_defaults_to_four_and_keeps_the_unsharded_path(self): workflow = yaml.load( (ROOT / '.github/workflows/sonarcloud.yml').read_text(), Loader=yaml.BaseLoader) shards = workflow['on']['workflow_call']['inputs']['shards'] @@ -93,7 +93,26 @@ class SonarCloudTests(unittest.TestCase): self.assertIn('inputs.shards <= 1', coverage['if']) self.assertEqual(workflow['jobs']['plan']['if'], 'inputs.shards > 1') self.assertEqual(workflow['jobs']['coverage']['strategy']['fail-fast'], 'false') - self.assertIn('--partition count:', workflow['jobs']['coverage']['steps'][-2]['run']) + + def test_shards_run_partitions_of_one_prebuilt_archive(self): + workflow = yaml.load( + (ROOT / '.github/workflows/sonarcloud.yml').read_text(), Loader=yaml.BaseLoader) + build = workflow['jobs']['build']['steps'] + self.assertTrue(any('nextest archive --workspace' in step.get('run', '') for step in build)) + tests = next(step for step in workflow['jobs']['coverage']['steps'] + if step.get('name') == "Run this shard's tests") + self.assertIn('--partition count:${{ matrix.shard }}/${{ inputs.shards }}', tests['run']) + self.assertIn('--archive-file archive/nextest-archive.tar.zst', tests['run']) + self.assertEqual(workflow['jobs']['coverage']['needs'], ['plan', 'build']) + + def test_scan_fails_when_a_prerequisite_job_failed(self): + workflow = yaml.load( + (ROOT / '.github/workflows/sonarcloud.yml').read_text(), Loader=yaml.BaseLoader) + condition = workflow['jobs']['sonarcloud']['if'] + for job in ('plan', 'build', 'coverage'): + self.assertIn(f'needs.{job}.result', condition) + self.assertIn('"success","skipped"', condition) + self.assertIn('!cancelled()', condition) def test_token_availability_still_controls_the_scan(self): token_step = next(step for step in self.steps if step.get('id') == 'token')