sonarcloud: build the instrumented archive once and shard tests from it

2026-11-02 18:06:16 +00:00 · 2026-10-25 18:30:40 -04:00
parent b22dd9c0d5
commit f043d79388
3 changed files with 123 additions and 46 deletions
@@ -20,8 +20,8 @@ on:
        default: 90
      shards:
        description: >-
-          Split the Rust coverage run into this many parallel cargo-nextest partitions
-          (1 keeps the single `cargo llvm-cov` job; default 4). Doctests are not run when sharded.
+          Run the Rust tests as this many parallel partitions of one prebuilt instrumented
+          archive (1 keeps the single `cargo llvm-cov` job; default 4). Doctests are not run when sharded.
        type: number
        required: false
        default: 4
@@ -61,12 +61,57 @@ jobs:
            echo 'matrix={"shard":[]}' >>"$GITHUB_OUTPUT"
          fi

-  coverage:
-    name: Coverage ${{ matrix.shard }}/${{ inputs.shards }}
+  build:
+    name: Build instrumented test archive
    needs: plan
    if: needs.plan.outputs.matrix != '{"shard":[]}'
    runs-on: ubuntu-latest
    timeout-minutes: ${{ inputs.timeout-minutes }}
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - name: Free disk space
+        run: sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
+      - name: Install Linux build dependencies
+        uses: storytold/craft-common-actions/actions/install-apt-packages@main
+        with:
+          packages: >-
+            pkg-config libxkbcommon-dev libwayland-dev libx11-dev libxcursor-dev libxrandr-dev
+            libxi-dev libgtk-3-dev libfontconfig1-dev libfreetype-dev libasound2-dev
+            libgl1-mesa-dev mesa-vulkan-drivers
+      - name: Install Rust and cargo-llvm-cov
+        run: |
+          rustup toolchain install stable --profile minimal --component llvm-tools-preview
+          rustup default stable
+      - uses: taiki-e/install-action@8cca0918f8471c33515202b73f56ab27f2bed283 # v2.87.29
+        with:
+          tool: cargo-llvm-cov,cargo-nextest
+      - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3
+        with:
+          save-if: ${{ github.ref == 'refs/heads/main' }}
+      - name: Build instrumented test archive
+        env:
+          CARGO_TERM_COLOR: always
+          # Coverage comes from instrumentation, not DWARF; debuginfo is most of target/.
+          CARGO_PROFILE_DEV_DEBUG: "0"
+        run: |
+          # show-env sets the instrumentation flags the same way the unsharded run gets them.
+          # shellcheck disable=SC1090 # generated by cargo-llvm-cov at runtime
+          source <(cargo llvm-cov show-env --sh)
+          cargo nextest archive --workspace --archive-file nextest-archive.tar.zst
+      - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        with:
+          name: nextest-archive
+          path: nextest-archive.tar.zst
+          retention-days: 1
+
+  coverage:
+    name: Tests ${{ matrix.shard }}/${{ inputs.shards }}
+    needs: [plan, build]
+    if: needs.plan.outputs.matrix != '{"shard":[]}'
+    runs-on: ubuntu-latest
+    timeout-minutes: ${{ inputs.timeout-minutes }}
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.plan.outputs.matrix) }}
@@ -78,8 +123,6 @@ jobs:
        uses: storytold/craft-common-actions/actions/checkout-craft-fonts@main
        with:
          ref: ${{ inputs.fonts-ref }}
-      - name: Free disk space
-        run: sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
      - name: Install Linux build dependencies
        uses: storytold/craft-common-actions/actions/install-apt-packages@main
        with:
@@ -87,41 +130,36 @@ jobs:
            pkg-config libxkbcommon-dev libwayland-dev libx11-dev libxcursor-dev libxrandr-dev
            libxi-dev libgtk-3-dev libfontconfig1-dev libfreetype-dev libasound2-dev
            libgl1-mesa-dev mesa-vulkan-drivers
-      - name: Install Rust
-        run: |
-          rustup toolchain install stable --profile minimal --component llvm-tools-preview
-          rustup default stable
      - uses: taiki-e/install-action@8cca0918f8471c33515202b73f56ab27f2bed283 # v2.87.29
        with:
-          tool: cargo-llvm-cov,cargo-nextest
-      - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3
+          tool: cargo-nextest
+      - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
        with:
-          # Every shard builds the same instrumented targets, so they share one cache entry.
-          shared-key: sonar-coverage
-          save-if: ${{ github.ref == 'refs/heads/main' && matrix.shard == 1 }}
-      - name: Generate Rust coverage for this shard
+          name: nextest-archive
+          path: archive
+      - name: Run this shard's tests
        # A red test must not hide the static analysis, so keep going and scan what was measured.
        continue-on-error: true
        env:
-          CARGO_TERM_COLOR: always
          CRAFT_FONTS_DIR: ${{ github.workspace }}/craft-fonts
-          CARGO_PROFILE_DEV_DEBUG: "0"
-        run: >
-          cargo llvm-cov nextest --workspace --partition count:${{ matrix.shard }}/${{ inputs.shards }}
-          --no-fail-fast --ignore-run-fail --lcov --output-path lcov-${{ matrix.shard }}.info
-          --ignore-filename-regex "$COVERAGE_IGNORE"
+          # Each instrumented binary writes its profile here; the scan job merges them.
+          LLVM_PROFILE_FILE: ${{ github.workspace }}/profraw/shard-${{ matrix.shard }}-%p-%m.profraw
+        run: |
+          mkdir -p profraw
+          cargo nextest run --archive-file archive/nextest-archive.tar.zst \
+            --partition count:${{ matrix.shard }}/${{ inputs.shards }} --no-fail-fast
      - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
        with:
-          name: lcov-${{ matrix.shard }}
-          path: lcov-${{ matrix.shard }}.info
+          name: profraw-${{ matrix.shard }}
+          path: profraw/
          if-no-files-found: ignore
          retention-days: 1

  sonarcloud:
    name: SonarCloud Scan
-    needs: coverage
-    # Scan even if a shard failed outright; the plan/coverage jobs are skipped when shards is 1.
-    if: ${{ !cancelled() }}
+    needs: [plan, build, coverage]
+    # Scan unless a prerequisite failed outright; a red test shard still lets the scan measure what ran.
+    if: ${{ !cancelled() && contains(fromJSON('["success","skipped"]'), needs.plan.result) && contains(fromJSON('["success","skipped"]'), needs.build.result) && contains(fromJSON('["success","skipped"]'), needs.coverage.result) }}
    runs-on: ubuntu-latest
    timeout-minutes: ${{ inputs.timeout-minutes }}
    steps:
@@ -155,13 +193,13 @@ jobs:
          fi
          printf 'key=%s\n' "$PROJECT_KEY" >>"$GITHUB_OUTPUT"
      - name: Check out craft-fonts
-        if: steps.token.outputs.available == 'true'
+        if: steps.token.outputs.available == 'true' && inputs.shards <= 1
        uses: storytold/craft-common-actions/actions/checkout-craft-fonts@main
        with:
          ref: ${{ inputs.fonts-ref }}
      - name: Free disk space
        # The instrumented workspace build overflows the ~14 GB free on a stock runner.
-        if: steps.token.outputs.available == 'true' && hashFiles('Cargo.toml') != ''
+        if: steps.token.outputs.available == 'true' && hashFiles('Cargo.toml') != '' && inputs.shards <= 1
        run: sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
      - name: Install Linux build dependencies
        if: steps.token.outputs.available == 'true' && hashFiles('Cargo.toml') != '' && inputs.shards <= 1
@@ -197,21 +235,41 @@ jobs:
        run: >
          cargo llvm-cov --workspace --ignore-run-fail --lcov --output-path lcov.info
          --ignore-filename-regex "$COVERAGE_IGNORE"
-      - name: Merge shard coverage
+      - name: Install LLVM tools
+        if: steps.token.outputs.available == 'true' && inputs.shards > 1
+        run: rustup component add llvm-tools-preview
+      - name: Download shard profiles and archive
        if: steps.token.outputs.available == 'true' && inputs.shards > 1
        uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
        with:
-          pattern: lcov-*
-          path: lcov-shards
+          pattern: profraw-*
+          path: profraw
          merge-multiple: true
-      - name: Combine shard reports
+      - name: Download test archive
        if: steps.token.outputs.available == 'true' && inputs.shards > 1
+        uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
+        with:
+          name: nextest-archive
+          path: archive
+      - name: Combine shard coverage
+        if: steps.token.outputs.available == 'true' && inputs.shards > 1
+        shell: bash
        run: |
-          # lcov sums hit counts for the same line across shards; a shard that never ran adds nothing.
-          sudo apt-get install -y --no-install-recommends lcov
-          set -- lcov-shards/lcov-*.info
-          [ -e "$1" ] || exit 0
-          lcov "${@/#/-a}" -o lcov.info 2>&1 | tail -3
+          set -euo pipefail
+          llvm_bin="$(rustc --print sysroot)/lib/rustlib/$(rustc -vV | sed -n 's/^host: //p')/bin"
+          mkdir -p coverage-archive
+          tar --zstd -xf archive/nextest-archive.tar.zst -C coverage-archive
+          objects=()
+          while IFS= read -r -d '' file; do objects+=("$file"); done < <(find coverage-archive -type f -executable -print0)
+          if [ "${#objects[@]}" -eq 0 ]; then
+            echo "::error::No instrumented test binaries in the archive"
+            exit 1
+          fi
+          "$llvm_bin/llvm-profdata" merge -sparse profraw/*.profraw -o coverage.profdata
+          extra=()
+          for object in "${objects[@]:1}"; do extra+=(-object "$object"); done
+          "$llvm_bin/llvm-cov" export -format=lcov -instr-profile=coverage.profdata \
+            "${objects[0]}" "${extra[@]}" -ignore-filename-regex="$COVERAGE_IGNORE" >lcov.info
      - name: SonarCloud Scan
        if: steps.token.outputs.available == 'true'
        uses: SonarSource/sonarqube-scan-action@d209202bc7d53ff1cc128f7f907dac145c9d6ae9 # v8.3.0
@@ -494,12 +494,12 @@ changes the job's time budget without dropping tests or coverage. Pass
 `fonts-ref` at the app's tested craft-fonts commit when its tests need those
 fonts. The scan still needs to finish and pass its quality gate.

-Workspaces split the coverage run across four parallel `cargo nextest`
-partitions by default. Pass `shards: 1` to keep the single `cargo llvm-cov`
-job, or another count to change the split. Each shard uploads its own lcov
-report and the scan job merges them with `lcov`. Sharded runs skip doctests,
-because nextest does not run them, and every shard still compiles the whole
-workspace, so more shards stop helping once compile time dominates.
+Workspaces run their tests as four parallel `cargo nextest` partitions by
+default. One job builds the instrumented test archive; each partition runs from
+that archive and uploads its profile data, and the scan job merges the profiles
+with the rustup `llvm-profdata` and `llvm-cov` tools. Pass `shards: 1` to keep
+the single `cargo llvm-cov` job. Sharded runs skip doctests, because nextest
+does not run them.

 The new limits leave headroom over sampled Craft app job timings; the existing
 FreeBSD and ARM64 limits are retained. Treat these as initial limits, review
@@ -84,7 +84,7 @@ class SonarCloudTests(unittest.TestCase):
        self.assertIn('--output-path lcov.info', coverage['run'])
        self.assertEqual(coverage['continue-on-error'], 'true')

-    def test_sharding_is_opt_in_and_keeps_the_unsharded_path(self):
+    def test_sharding_defaults_to_four_and_keeps_the_unsharded_path(self):
        workflow = yaml.load(
            (ROOT / '.github/workflows/sonarcloud.yml').read_text(), Loader=yaml.BaseLoader)
        shards = workflow['on']['workflow_call']['inputs']['shards']
@@ -93,7 +93,26 @@ class SonarCloudTests(unittest.TestCase):
        self.assertIn('inputs.shards <= 1', coverage['if'])
        self.assertEqual(workflow['jobs']['plan']['if'], 'inputs.shards > 1')
        self.assertEqual(workflow['jobs']['coverage']['strategy']['fail-fast'], 'false')
-        self.assertIn('--partition count:', workflow['jobs']['coverage']['steps'][-2]['run'])
+
+    def test_shards_run_partitions_of_one_prebuilt_archive(self):
+        workflow = yaml.load(
+            (ROOT / '.github/workflows/sonarcloud.yml').read_text(), Loader=yaml.BaseLoader)
+        build = workflow['jobs']['build']['steps']
+        self.assertTrue(any('nextest archive --workspace' in step.get('run', '') for step in build))
+        tests = next(step for step in workflow['jobs']['coverage']['steps']
+                     if step.get('name') == "Run this shard's tests")
+        self.assertIn('--partition count:${{ matrix.shard }}/${{ inputs.shards }}', tests['run'])
+        self.assertIn('--archive-file archive/nextest-archive.tar.zst', tests['run'])
+        self.assertEqual(workflow['jobs']['coverage']['needs'], ['plan', 'build'])
+
+    def test_scan_fails_when_a_prerequisite_job_failed(self):
+        workflow = yaml.load(
+            (ROOT / '.github/workflows/sonarcloud.yml').read_text(), Loader=yaml.BaseLoader)
+        condition = workflow['jobs']['sonarcloud']['if']
+        for job in ('plan', 'build', 'coverage'):
+            self.assertIn(f'needs.{job}.result', condition)
+        self.assertIn('"success","skipped"', condition)
+        self.assertIn('!cancelled()', condition)

    def test_token_availability_still_controls_the_scan(self):
        token_step = next(step for step in self.steps if step.get('id') == 'token')