From 346c93cb62428af28a9e96be1d6cc6e5d7424ed8 Mon Sep 17 00:00:00 2001 From: Tyrie Vella Date: Wed, 28 Oct 2026 15:32:02 -0700 Subject: [PATCH 1/5] Probe NativeAOT coverage on one functional shard Build an isolated instrumented installer for x64 shard 0 and collect coverage before Windows services start. Keep ordinary installers and test selections unchanged for the other workers. Assisted-by: GPT-6.1 Sol Signed-off-by: Tyrie Vella --- .github/workflows/build.yaml | 49 ++++++++++++++- .github/workflows/functional-tests.yaml | 80 ++++++++++++++++++++++++- Directory.Build.targets | 9 +++ Directory.Packages.props | 1 + scripts/AotCoverage.config | 11 ++++ scripts/Assert-AotCoverage.Tests.ps1 | 78 ++++++++++++++++++++++++ scripts/Assert-AotCoverage.ps1 | 61 +++++++++++++++++++ 7 files changed, 286 insertions(+), 3 deletions(-) create mode 100644 scripts/AotCoverage.config create mode 100644 scripts/Assert-AotCoverage.Tests.ps1 create mode 100644 scripts/Assert-AotCoverage.ps1 diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index e7de3e1fb9..439057d70b 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -327,12 +327,59 @@ jobs: name: GVFS_${{ matrix.configuration }}_${{ matrix.architecture }} path: artifacts\GVFS.Installers + coverage_build: + runs-on: windows-2025 + name: Build AOT Coverage Probe + needs: validate + timeout-minutes: 30 + + env: + AotMsCodeCoverageInstrumentation: 'true' + MsCodeCoverageSessionId: gvfs-ci-${{ github.run_id }} + + steps: + - name: Checkout source + if: needs.validate.outputs.skip == '' + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: src + persist-credentials: false + + - name: Install .NET SDK + if: needs.validate.outputs.skip == '' + uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0 + with: + global-json-file: src/global.json + dotnet-version: 8.0.x + + - name: Add MSBuild to PATH + if: needs.validate.outputs.skip == '' + uses: microsoft/setup-msbuild@30375c66a4eea26614e0d39710365f22f8b0af57 # v3.0.0 + + - name: Build instrumented x64 installer + if: needs.validate.outputs.skip == '' + shell: cmd + run: src\scripts\Build.bat Release 0.2.173.2 minimal x64 + + - name: Create build artifacts + if: needs.validate.outputs.skip == '' + shell: cmd + run: src\scripts\CreateBuildArtifacts.bat Release artifacts x64 + + - name: Upload coverage-probe installer + if: needs.validate.outputs.skip == '' + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: GVFS_Coverage_Release_x64 + path: artifacts\GVFS.Installers + functional_tests: name: Functional Tests - needs: [validate, build] + needs: [validate, build, coverage_build] uses: ./.github/workflows/functional-tests.yaml with: skip: ${{ needs.validate.outputs.skip }} + aot_coverage_probe: true upgrade_tests: name: Upgrade Tests diff --git a/.github/workflows/functional-tests.yaml b/.github/workflows/functional-tests.yaml index 5b05a0f1a0..85a5ed2858 100644 --- a/.github/workflows/functional-tests.yaml +++ b/.github/workflows/functional-tests.yaml @@ -38,6 +38,11 @@ on: required: false type: string default: '' + aot_coverage_probe: + description: 'Use the instrumented installer and collect coverage on x64 shard 0' + required: false + type: boolean + default: false secrets: vfs_token: description: 'Token for downloading VFSForGit artifacts (required for cross-repository downloads; defaults to GITHUB_TOKEN)' @@ -58,6 +63,8 @@ jobs: env: ARTIFACT_PREFIX: ${{ inputs.output_prefix && format('{0}_', inputs.output_prefix) || '' }} FT_MATRIX_NAME: ${{ format('{0}_{1}-{2}', matrix.configuration, matrix.architecture, matrix.nr) }} + AOT_COVERAGE_PROBE: ${{ inputs.aot_coverage_probe && matrix.architecture == 'x64' && matrix.nr == 0 }} + COVERAGE_SESSION_ID: gvfs-ci-${{ inputs.vfs_run_id || github.run_id }} strategy: matrix: @@ -104,7 +111,7 @@ jobs: continue-on-error: true uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: - name: GVFS_${{ matrix.configuration }}_${{ matrix.architecture }} + name: ${{ env.AOT_COVERAGE_PROBE == 'true' && 'GVFS_Coverage_Release_x64' || format('GVFS_{0}_{1}', matrix.configuration, matrix.architecture) }} path: gvfs repository: ${{ inputs.vfs_repository || github.repository }} run-id: ${{ inputs.vfs_run_id || github.run_id }} @@ -114,7 +121,7 @@ jobs: if: steps.skip.outputs.result != 'true' && steps.download-gvfs.outcome == 'failure' uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: - name: GVFS_${{ matrix.configuration }}_${{ matrix.architecture }} + name: ${{ env.AOT_COVERAGE_PROBE == 'true' && 'GVFS_Coverage_Release_x64' || format('GVFS_{0}_{1}', matrix.configuration, matrix.architecture) }} path: gvfs repository: ${{ inputs.vfs_repository || github.repository }} run-id: ${{ inputs.vfs_run_id || github.run_id }} @@ -153,6 +160,40 @@ jobs: run-id: ${{ inputs.vfs_run_id || github.run_id }} github-token: ${{ secrets.vfs_token || github.token }} + - name: Checkout coverage configuration and source + if: steps.skip.outputs.result != 'true' && env.AOT_COVERAGE_PROBE == 'true' + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: src + persist-credentials: false + + - name: Install coverage runtime and SDK + if: steps.skip.outputs.result != 'true' && env.AOT_COVERAGE_PROBE == 'true' + uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0 + with: + global-json-file: src/global.json + dotnet-version: 8.0.x + + - name: Install coverage tools + if: steps.skip.outputs.result != 'true' && env.AOT_COVERAGE_PROBE == 'true' + shell: pwsh + run: | + dotnet tool install dotnet-coverage --version 18.8.0 --tool-path coverage-tools --configfile src\nuget.config + if ($LASTEXITCODE -ne 0) { throw 'Coverage collector installation failed.' } + dotnet tool install dotnet-reportgenerator-globaltool --version 5.5.11 --tool-path coverage-tools --configfile src\nuget.config + if ($LASTEXITCODE -ne 0) { throw 'Coverage renderer installation failed.' } + New-Item -ItemType Directory -Path coverage -Force | Out-Null + + - name: Start fixed-session coverage before service installation + id: coverage_start + if: steps.skip.outputs.result != 'true' && env.AOT_COVERAGE_PROBE == 'true' + shell: pwsh + run: | + .\coverage-tools\dotnet-coverage.exe collect --server-mode --background --session-id $env:COVERAGE_SESSION_ID --settings src\scripts\AotCoverage.config --output coverage\functional.cobertura.xml --output-format cobertura --log-file coverage\collector.log --log-level Verbose + if ($LASTEXITCODE -ne 0) { throw 'Coverage collector startup failed.' } + .\coverage-tools\dotnet-coverage.exe snapshot $env:COVERAGE_SESSION_ID --output coverage\before-install.coverage + if ($LASTEXITCODE -ne 0) { throw 'Coverage collector readiness check failed.' } + - name: ProjFS details (pre-install) if: steps.skip.outputs.result != 'true' shell: cmd @@ -197,6 +238,14 @@ jobs: git\logs gvfs\logs + - name: Record installed service identity + if: steps.skip.outputs.result != 'true' && env.AOT_COVERAGE_PROBE == 'true' + shell: pwsh + run: | + Get-CimInstance Win32_Service -Filter "Name = 'GVFS.Service'" | + Select-Object Name,StartName,State,ProcessId,PathName | + ConvertTo-Json | Set-Content coverage\installed-service.json + - name: Run functional tests if: steps.skip.outputs.result != 'true' shell: cmd @@ -207,6 +256,33 @@ jobs: SET GVFS_TEST_DIAGNOSTICS_DIR=C:\temp\gvfs-ft-diagnostics ft\GVFS.FunctionalTests.exe /result:TestResult.xml --ci --slice=${{ matrix.nr }},12 + - name: Stop coverage and verify SCM service capture + id: coverage_stop + if: always() && env.AOT_COVERAGE_PROBE == 'true' && steps.coverage_start.outcome == 'success' + shell: pwsh + run: | + .\coverage-tools\dotnet-coverage.exe shutdown $env:COVERAGE_SESSION_ID + if ($LASTEXITCODE -ne 0) { throw 'Coverage collector shutdown failed.' } + src\scripts\Assert-AotCoverage.ps1 -CoveragePath coverage\functional.cobertura.xml | + Tee-Object -FilePath coverage\probe.json + + - name: Render single-shard coverage report + if: always() && env.AOT_COVERAGE_PROBE == 'true' && steps.coverage_stop.outcome == 'success' + shell: pwsh + run: | + .\coverage-tools\reportgenerator.exe -reports:coverage\functional.cobertura.xml -targetdir:coverage\report "-reporttypes:Html;MarkdownSummaryGithub;Cobertura;JsonSummary" "-filefilters:+$env:GITHUB_WORKSPACE\src\GVFS\*" "-title:Managed NativeAOT product - x64 shard 0 of 12" + if ($LASTEXITCODE -ne 0) { throw 'Coverage rendering failed.' } + 'One-shard probe only. Native C++ and the other shards are not measured.' >> $env:GITHUB_STEP_SUMMARY + Get-Content coverage\report\SummaryGithub.md >> $env:GITHUB_STEP_SUMMARY + + - name: Upload coverage-probe evidence + if: always() && steps.skip.outputs.result != 'true' && env.AOT_COVERAGE_PROBE == 'true' + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: ${{ env.ARTIFACT_PREFIX }}AotCoverage_${{ env.FT_MATRIX_NAME }} + path: coverage + if-no-files-found: error + - name: Upload failure diagnostics (mount dumps + logs) if: always() && steps.skip.outputs.result != 'true' uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 diff --git a/Directory.Build.targets b/Directory.Build.targets index 59c793d2d4..7aa29b1094 100644 --- a/Directory.Build.targets +++ b/Directory.Build.targets @@ -12,6 +12,15 @@ + + + true + + + + + + + diff --git a/scripts/AotCoverage.config b/scripts/AotCoverage.config new file mode 100644 index 0000000000..7b3c03e238 --- /dev/null +++ b/scripts/AotCoverage.config @@ -0,0 +1,11 @@ + + + False + + False + False + False + False + True + + diff --git a/scripts/Assert-AotCoverage.Tests.ps1 b/scripts/Assert-AotCoverage.Tests.ps1 new file mode 100644 index 0000000000..537fb088d7 --- /dev/null +++ b/scripts/Assert-AotCoverage.Tests.ps1 @@ -0,0 +1,78 @@ +BeforeAll { + $assertion = Join-Path $PSScriptRoot 'Assert-AotCoverage.ps1' + + function New-CoverageReport { + param( + [int]$OnStartHits = 1, + [int]$ConsoleHits = 0 + ) + + $modules = @('GVFS', 'GVFS.Hooks', 'GVFS.Mount', 'GVFS.Service', 'GVFS.Common', 'GVFS.Platform.Windows', 'GVFS.Virtualization') + $packages = ($modules | ForEach-Object { + "" + }) -join '' + [xml]$document = "$packages" + $service = $document.SelectSingleNode("//package[@name='GVFS.Service']/classes") + $service.InnerXml = @" + + + + + + + + +"@ + return ,$document + } + + function Save-CoverageReport { + param([xml]$Document) + $path = Join-Path $TestDrive 'coverage.xml' + $Document.Save($path) + return $path + } +} + +Describe 'NativeAOT SCM coverage evidence' { + It 'Accepts product coverage with SCM startup and no console-service entry' { + $path = Save-CoverageReport (New-CoverageReport) + $result = & $assertion -CoveragePath $path | ConvertFrom-Json + $result.ScmServiceStartupCovered | Should -BeTrue + $result.CompleteSuiteCoverage | Should -BeFalse + } + + It 'Rejects a report without SCM startup execution' { + $path = Save-CoverageReport (New-CoverageReport -OnStartHits 0) + { & $assertion -CoveragePath $path } | Should -Throw '*no executed GVFSService.OnStart*' + } + + It 'Rejects console-service execution even when SCM startup also executes' { + $path = Save-CoverageReport (New-CoverageReport -ConsoleHits 1) + { & $assertion -CoveragePath $path } | Should -Throw '*Console-service execution contaminates*' + } + + It 'Rejects missing product modules' { + $document = New-CoverageReport + $node = $document.SelectSingleNode("//package[@name='GVFS.Mount']") + [void]$node.ParentNode.RemoveChild($node) + $path = Save-CoverageReport $document + { & $assertion -CoveragePath $path } | Should -Throw '*Expected one coverage module for GVFS.Mount*' + } + + It 'Rejects product modules with metadata but no executed lines' { + $document = New-CoverageReport + $document.SelectSingleNode("//package[@name='GVFS.Hooks']//line").SetAttribute('hits', '0') + $path = Save-CoverageReport $document + { & $assertion -CoveragePath $path } | Should -Throw '*No executed lines in required product module GVFS.Hooks*' + } + + It 'Rejects unit-copy coverage in a product report' { + $document = New-CoverageReport + $node = $document.CreateElement('package') + $node.SetAttribute('name', 'GVFS.UnitTests') + [void]$document.coverage.packages.AppendChild($node) + $path = Save-CoverageReport $document + { & $assertion -CoveragePath $path } | Should -Throw '*Test assemblies contaminate*' + } +} diff --git a/scripts/Assert-AotCoverage.ps1 b/scripts/Assert-AotCoverage.ps1 new file mode 100644 index 0000000000..86ef5a1d56 --- /dev/null +++ b/scripts/Assert-AotCoverage.ps1 @@ -0,0 +1,61 @@ +param( + [Parameter(Mandatory)] + [string]$CoveragePath +) + +$ErrorActionPreference = 'Stop' + +[xml]$coverage = Get-Content -LiteralPath $CoveragePath -Raw +$packages = @($coverage.coverage.packages.package) +$requiredModules = @( + 'GVFS', + 'GVFS.Hooks', + 'GVFS.Mount', + 'GVFS.Service', + 'GVFS.Common', + 'GVFS.Platform.Windows', + 'GVFS.Virtualization' +) + +foreach ($name in $requiredModules) { + $module = @($packages | Where-Object { $_.name -eq $name }) + if ($module.Count -ne 1) { + throw "Expected one coverage module for $name; found $($module.Count)." + } + + $hitLines = @($module[0].classes.class.lines.line | Where-Object { [int]$_.hits -gt 0 }) + if ($hitLines.Count -eq 0) { + throw "No executed lines in required product module $name." + } +} + +$testModules = @($packages | Where-Object { $_.name -match '(UnitTests|FunctionalTests|^GVFS.Tests$)' }) +if ($testModules.Count -ne 0) { + throw "Test assemblies contaminate the product report: $($testModules.name -join ', ')." +} + +$service = $packages | Where-Object { $_.name -eq 'GVFS.Service' } +$serviceClass = $service.classes.class | Where-Object { $_.name -eq 'GVFS.Service.GVFSService' } +$onStart = @($serviceClass.methods.method | Where-Object { $_.name -eq 'OnStart' }) +$startupHits = @($onStart.lines.line | Where-Object { [int]$_.hits -gt 0 }) +if ($startupHits.Count -eq 0) { + throw 'The report contains no executed GVFSService.OnStart lines.' +} + +$program = $service.classes.class | Where-Object { $_.name -eq 'GVFS.Service.Program' } +$consoleEntry = @($program.methods.method | Where-Object { $_.name -eq 'RunAsConsoleApp' }) +$consoleHits = @($consoleEntry.lines.line | Where-Object { [int]$_.hits -gt 0 }) +if ($consoleHits.Count -ne 0) { + throw 'Console-service execution contaminates the SCM-service probe.' +} + +[ordered]@{ + Scope = 'Managed NativeAOT product; x64 shard 0 of 12' + Modules = $packages.name + ScmServiceStartupCovered = $true + ConsoleServiceEntryCovered = $false + NativeCppCovered = $false + CompleteSuiteCoverage = $false + CoveredLines = [int]$coverage.coverage.'lines-covered' + MeasuredLines = [int]$coverage.coverage.'lines-valid' +} | ConvertTo-Json -Depth 3 From 8cbad7cf00b85a75fd7e8d5fe3ac9416e49efa9a Mon Sep 17 00:00:00 2001 From: Tyrie Vella Date: Thu, 5 Nov 2026 10:40:16 -0800 Subject: [PATCH 2/5] Use service-capable collection for the AOT probe Grant LocalSystem access through the Visual Studio collector. Verify service startup coverage before the selected functional shard runs. Assisted-by: GPT-6.1 Sol Signed-off-by: Tyrie Vella --- .github/workflows/functional-tests.yaml | 24 +++++++++++++++++++----- scripts/AotCoverage.config | 3 +++ scripts/Assert-AotCoverage.Tests.ps1 | 16 ++++++++++++++++ scripts/Assert-AotCoverage.ps1 | 8 ++++++-- 4 files changed, 44 insertions(+), 7 deletions(-) diff --git a/.github/workflows/functional-tests.yaml b/.github/workflows/functional-tests.yaml index 85a5ed2858..7c3bf2d7f0 100644 --- a/.github/workflows/functional-tests.yaml +++ b/.github/workflows/functional-tests.yaml @@ -178,8 +178,11 @@ jobs: if: steps.skip.outputs.result != 'true' && env.AOT_COVERAGE_PROBE == 'true' shell: pwsh run: | - dotnet tool install dotnet-coverage --version 18.8.0 --tool-path coverage-tools --configfile src\nuget.config - if ($LASTEXITCODE -ne 0) { throw 'Coverage collector installation failed.' } + $collectors = @(& "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.exe" -products * -latest -find Common7\IDE\Extensions\Microsoft\CodeCoverage.Console\Microsoft.CodeCoverage.Console.exe) + if ($LASTEXITCODE -ne 0 -or $collectors.Count -ne 1) { throw 'Expected one Visual Studio coverage collector.' } + Split-Path -Parent $collectors[0] >> $env:GITHUB_PATH + & $collectors[0] --version + if ($LASTEXITCODE -ne 0) { throw 'Visual Studio coverage collector cannot start.' } dotnet tool install dotnet-reportgenerator-globaltool --version 5.5.11 --tool-path coverage-tools --configfile src\nuget.config if ($LASTEXITCODE -ne 0) { throw 'Coverage renderer installation failed.' } New-Item -ItemType Directory -Path coverage -Force | Out-Null @@ -189,9 +192,9 @@ jobs: if: steps.skip.outputs.result != 'true' && env.AOT_COVERAGE_PROBE == 'true' shell: pwsh run: | - .\coverage-tools\dotnet-coverage.exe collect --server-mode --background --session-id $env:COVERAGE_SESSION_ID --settings src\scripts\AotCoverage.config --output coverage\functional.cobertura.xml --output-format cobertura --log-file coverage\collector.log --log-level Verbose + Microsoft.CodeCoverage.Console.exe collect --server-mode --background --session-id $env:COVERAGE_SESSION_ID --settings src\scripts\AotCoverage.config --output coverage\functional.cobertura.xml --output-format cobertura --log-file coverage\collector.log --log-level Verbose if ($LASTEXITCODE -ne 0) { throw 'Coverage collector startup failed.' } - .\coverage-tools\dotnet-coverage.exe snapshot $env:COVERAGE_SESSION_ID --output coverage\before-install.coverage + Microsoft.CodeCoverage.Console.exe snapshot $env:COVERAGE_SESSION_ID --output coverage\before-install.coverage if ($LASTEXITCODE -ne 0) { throw 'Coverage collector readiness check failed.' } - name: ProjFS details (pre-install) @@ -246,6 +249,17 @@ jobs: Select-Object Name,StartName,State,ProcessId,PathName | ConvertTo-Json | Set-Content coverage\installed-service.json + - name: Verify service coverage before functional tests + if: steps.skip.outputs.result != 'true' && env.AOT_COVERAGE_PROBE == 'true' + shell: pwsh + run: | + Microsoft.CodeCoverage.Console.exe snapshot $env:COVERAGE_SESSION_ID --output coverage\after-install.coverage + if ($LASTEXITCODE -ne 0) { throw 'Post-installation coverage snapshot failed.' } + Microsoft.CodeCoverage.Console.exe merge coverage\after-install.coverage --output coverage\after-install.cobertura.xml --output-format cobertura + if ($LASTEXITCODE -ne 0) { throw 'Post-installation coverage conversion failed.' } + src\scripts\Assert-AotCoverage.ps1 -CoveragePath coverage\after-install.cobertura.xml -ServiceOnly | + Tee-Object -FilePath coverage\service-startup-probe.json + - name: Run functional tests if: steps.skip.outputs.result != 'true' shell: cmd @@ -261,7 +275,7 @@ jobs: if: always() && env.AOT_COVERAGE_PROBE == 'true' && steps.coverage_start.outcome == 'success' shell: pwsh run: | - .\coverage-tools\dotnet-coverage.exe shutdown $env:COVERAGE_SESSION_ID + Microsoft.CodeCoverage.Console.exe shutdown $env:COVERAGE_SESSION_ID if ($LASTEXITCODE -ne 0) { throw 'Coverage collector shutdown failed.' } src\scripts\Assert-AotCoverage.ps1 -CoveragePath coverage\functional.cobertura.xml | Tee-Object -FilePath coverage\probe.json diff --git a/scripts/AotCoverage.config b/scripts/AotCoverage.config index 7b3c03e238..b528f8a923 100644 --- a/scripts/AotCoverage.config +++ b/scripts/AotCoverage.config @@ -2,6 +2,9 @@ False + + NT AUTHORITY\SYSTEM + False False False diff --git a/scripts/Assert-AotCoverage.Tests.ps1 b/scripts/Assert-AotCoverage.Tests.ps1 index 537fb088d7..282201c07c 100644 --- a/scripts/Assert-AotCoverage.Tests.ps1 +++ b/scripts/Assert-AotCoverage.Tests.ps1 @@ -47,6 +47,22 @@ Describe 'NativeAOT SCM coverage evidence' { { & $assertion -CoveragePath $path } | Should -Throw '*no executed GVFSService.OnStart*' } + It 'Accepts a service-only startup snapshot before other product processes run' { + $document = New-CoverageReport + foreach ($node in @($document.SelectNodes("//package[@name!='GVFS.Service']"))) { + [void]$node.ParentNode.RemoveChild($node) + } + $path = Save-CoverageReport $document + $result = & $assertion -CoveragePath $path -ServiceOnly | ConvertFrom-Json + $result.ScmServiceStartupCovered | Should -BeTrue + $result.Scope | Should -Be 'SCM service startup before tests' + } + + It 'Rejects metadata-only service coverage in the startup snapshot' { + $path = Save-CoverageReport (New-CoverageReport -OnStartHits 0) + { & $assertion -CoveragePath $path -ServiceOnly } | Should -Throw '*no executed GVFSService.OnStart*' + } + It 'Rejects console-service execution even when SCM startup also executes' { $path = Save-CoverageReport (New-CoverageReport -ConsoleHits 1) { & $assertion -CoveragePath $path } | Should -Throw '*Console-service execution contaminates*' diff --git a/scripts/Assert-AotCoverage.ps1 b/scripts/Assert-AotCoverage.ps1 index 86ef5a1d56..ba7c9a9253 100644 --- a/scripts/Assert-AotCoverage.ps1 +++ b/scripts/Assert-AotCoverage.ps1 @@ -1,6 +1,7 @@ param( [Parameter(Mandatory)] - [string]$CoveragePath + [string]$CoveragePath, + [switch]$ServiceOnly ) $ErrorActionPreference = 'Stop' @@ -16,6 +17,9 @@ $requiredModules = @( 'GVFS.Platform.Windows', 'GVFS.Virtualization' ) +if ($ServiceOnly) { + $requiredModules = @('GVFS.Service') +} foreach ($name in $requiredModules) { $module = @($packages | Where-Object { $_.name -eq $name }) @@ -50,7 +54,7 @@ if ($consoleHits.Count -ne 0) { } [ordered]@{ - Scope = 'Managed NativeAOT product; x64 shard 0 of 12' + Scope = if ($ServiceOnly) { 'SCM service startup before tests' } else { 'Managed NativeAOT product; x64 shard 0 of 12' } Modules = $packages.name ScmServiceStartupCovered = $true ConsoleServiceEntryCovered = $false From 1094500c603036f13e86963aef89daca4d0d7da6 Mon Sep 17 00:00:00 2001 From: Tyrie Vella Date: Thu, 5 Nov 2026 12:21:20 -0800 Subject: [PATCH 3/5] Collect managed coverage across existing CI workers Instrument non-release payloads in the existing builds. Collect unit, functional, and upgrade evidence without a second test suite, and preserve source identity and separate unit-copy attribution. Assisted-by: GPT-6.1 Sol Signed-off-by: Tyrie Vella --- .github/actions/setup-coverage/action.yml | 12 + .github/workflows/build.yaml | 144 +++-- .github/workflows/functional-tests.yaml | 99 ++-- .github/workflows/upgrade-tests.yaml | 50 ++ Directory.Build.targets | 2 +- scripts/Assert-AotCoverage.Tests.ps1 | 2 +- scripts/Assert-AotCoverage.ps1 | 2 +- scripts/Invoke-Coverage.Tests.ps1 | 59 ++ scripts/Invoke-Coverage.ps1 | 84 +++ scripts/Merge-Coverage.Tests.ps1 | 350 ++++++++++++ scripts/Merge-Coverage.ps1 | 632 ++++++++++++++++++++++ scripts/New-CoverageManifest.ps1 | 26 + scripts/UnitCoverage.config | 24 + 13 files changed, 1380 insertions(+), 106 deletions(-) create mode 100644 .github/actions/setup-coverage/action.yml create mode 100644 scripts/Invoke-Coverage.Tests.ps1 create mode 100644 scripts/Invoke-Coverage.ps1 create mode 100644 scripts/Merge-Coverage.Tests.ps1 create mode 100644 scripts/Merge-Coverage.ps1 create mode 100644 scripts/New-CoverageManifest.ps1 create mode 100644 scripts/UnitCoverage.config diff --git a/.github/actions/setup-coverage/action.yml b/.github/actions/setup-coverage/action.yml new file mode 100644 index 0000000000..304019ea61 --- /dev/null +++ b/.github/actions/setup-coverage/action.yml @@ -0,0 +1,12 @@ +name: Set up coverage collector +description: Locate the Visual Studio collector with cross-account service support. +runs: + using: composite + steps: + - shell: pwsh + run: | + $collectors = @(& "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.exe" -products * -latest -find Common7\IDE\Extensions\Microsoft\CodeCoverage.Console\Microsoft.CodeCoverage.Console.exe) + if ($LASTEXITCODE -ne 0 -or $collectors.Count -ne 1) { throw 'Expected one Visual Studio coverage collector.' } + Split-Path -Parent $collectors[0] >> $env:GITHUB_PATH + & $collectors[0] --version + if ($LASTEXITCODE -ne 0) { throw 'Visual Studio coverage collector cannot start.' } diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 439057d70b..de47789faa 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -258,6 +258,9 @@ jobs: runs-on: ${{ matrix.architecture == 'arm64' && 'windows-11-arm' || 'windows-2025' }} name: Build and Unit Test needs: validate + env: + AotMsCodeCoverageInstrumentation: 'true' + MsCodeCoverageSessionId: gvfs-ci-${{ github.run_id }} strategy: matrix: @@ -280,12 +283,14 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: path: src + persist-credentials: false - name: Install .NET SDK if: steps.skip.outputs.result != 'true' uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0 with: global-json-file: src/global.json + dotnet-version: 8.0.x - name: Add MSBuild to PATH if: steps.skip.outputs.result != 'true' @@ -296,16 +301,54 @@ jobs: shell: cmd run: src\scripts\Build.bat ${{ matrix.configuration }} 0.2.173.2 minimal ${{ matrix.architecture }} + - name: Locate coverage collector + if: steps.skip.outputs.result != 'true' + uses: ./src/.github/actions/setup-coverage + + - name: Record compiled coverage manifest + if: steps.skip.outputs.result != 'true' + shell: pwsh + env: + COVERAGE_ARCH: ${{ matrix.architecture }} + run: src\scripts\New-CoverageManifest.ps1 -Path coverage-build.json -Repository $env:GITHUB_REPOSITORY -SessionId $env:MsCodeCoverageSessionId -Architecture $env:COVERAGE_ARCH + + - name: Start unit coverage + id: unit_coverage + if: steps.skip.outputs.result != 'true' + shell: pwsh + run: src\scripts\Invoke-Coverage.ps1 -Action Start -Directory unit-coverage -BuildManifest coverage-build.json -Suite Unit + - name: Run unit tests + id: unit_tests if: steps.skip.outputs.result != 'true' - shell: cmd - run: src\scripts\RunUnitTests.bat ${{ matrix.configuration }} ${{ matrix.architecture }} + shell: pwsh + run: | + Microsoft.CodeCoverage.Console.exe connect $env:MsCodeCoverageSessionId cmd.exe /d /c src\scripts\RunUnitTests.bat ${{ matrix.configuration }} ${{ matrix.architecture }} + exit $LASTEXITCODE + + - name: Stop unit coverage + if: always() && steps.unit_coverage.outcome == 'success' + shell: pwsh + run: src\scripts\Invoke-Coverage.ps1 -Action Stop -Directory unit-coverage -TestOutcome '${{ steps.unit_tests.outcome }}' + + - name: Upload unit coverage + if: always() && steps.unit_coverage.outcome == 'success' + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: Coverage_Unit_${{ matrix.architecture }} + path: unit-coverage + if-no-files-found: error - name: Create build artifacts if: steps.skip.outputs.result != 'true' shell: cmd run: src\scripts\CreateBuildArtifacts.bat ${{ matrix.configuration }} artifacts ${{ matrix.architecture }} + - name: Include coverage manifest with installer + if: steps.skip.outputs.result != 'true' + shell: pwsh + run: Copy-Item coverage-build.json artifacts\GVFS.Installers\coverage-build.json + - name: Upload functional tests drop if: steps.skip.outputs.result != 'true' uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 @@ -327,66 +370,75 @@ jobs: name: GVFS_${{ matrix.configuration }}_${{ matrix.architecture }} path: artifacts\GVFS.Installers - coverage_build: - runs-on: windows-2025 - name: Build AOT Coverage Probe - needs: validate - timeout-minutes: 30 + functional_tests: + name: Functional Tests + needs: [validate, build] + uses: ./.github/workflows/functional-tests.yaml + with: + skip: ${{ needs.validate.outputs.skip }} - env: - AotMsCodeCoverageInstrumentation: 'true' - MsCodeCoverageSessionId: gvfs-ci-${{ github.run_id }} + upgrade_tests: + name: Upgrade Tests + needs: [validate, build] + uses: ./.github/workflows/upgrade-tests.yaml + with: + skip: ${{ needs.validate.outputs.skip }} + coverage_summary: + name: Managed Code Coverage + runs-on: windows-2025 + needs: [validate, build, functional_tests, upgrade_tests] + if: always() && needs.validate.outputs.skip == '' steps: - - name: Checkout source - if: needs.validate.outputs.skip == '' + - name: Checkout tested source uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: path: src + fetch-depth: 0 persist-credentials: false - - name: Install .NET SDK - if: needs.validate.outputs.skip == '' + - name: Install report runtime uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0 with: global-json-file: src/global.json dotnet-version: 8.0.x - - name: Add MSBuild to PATH - if: needs.validate.outputs.skip == '' - uses: microsoft/setup-msbuild@30375c66a4eea26614e0d39710365f22f8b0af57 # v3.0.0 - - - name: Build instrumented x64 installer - if: needs.validate.outputs.skip == '' - shell: cmd - run: src\scripts\Build.bat Release 0.2.173.2 minimal x64 - - - name: Create build artifacts - if: needs.validate.outputs.skip == '' - shell: cmd - run: src\scripts\CreateBuildArtifacts.bat Release artifacts x64 + - name: Download all coverage workers + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + pattern: Coverage_* + path: coverage-inputs - - name: Upload coverage-probe installer - if: needs.validate.outputs.skip == '' + - name: Verify workers and changed-line evidence + shell: pwsh + env: + BASE_COMMIT: ${{ github.event.pull_request.base.sha || github.event.before }} + run: | + $sourceCommit = git -C src rev-parse HEAD + if ($LASTEXITCODE -ne 0) { throw 'Cannot identify tested source.' } + $base = $env:BASE_COMMIT + if ($base -match '^0+$') { $base = '' } + src\scripts\Merge-Coverage.ps1 -InputDirectory coverage-inputs -OutputDirectory coverage-report -SourceDirectory src -SourceCommit $sourceCommit -Repository $env:GITHUB_REPOSITORY -BaseCommit $base + Get-Content coverage-report\Summary.md >> $env:GITHUB_STEP_SUMMARY + Get-Content coverage-report\ChangedLines.md >> $env:GITHUB_STEP_SUMMARY + + - name: Render full managed reports + shell: pwsh + run: | + dotnet tool install dotnet-reportgenerator-globaltool --version 5.5.11 --tool-path coverage-tools --configfile src\nuget.config + if ($LASTEXITCODE -ne 0) { throw 'Coverage renderer installation failed.' } + foreach ($group in Get-ChildItem coverage-report\normalized -Directory) { + .\coverage-tools\reportgenerator.exe "-reports:$($group.FullName)\*.cobertura.xml" "-targetdir:coverage-report\$($group.Name)" "-reporttypes:Html;MarkdownSummaryGithub;Cobertura;JsonSummary" "-title:Managed source coverage - $($group.Name)" + if ($LASTEXITCODE -ne 0) { throw "Coverage rendering failed for $($group.Name)." } + } + + - name: Upload complete coverage reports + if: always() uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: - name: GVFS_Coverage_Release_x64 - path: artifacts\GVFS.Installers - - functional_tests: - name: Functional Tests - needs: [validate, build, coverage_build] - uses: ./.github/workflows/functional-tests.yaml - with: - skip: ${{ needs.validate.outputs.skip }} - aot_coverage_probe: true - - upgrade_tests: - name: Upgrade Tests - needs: [validate, build] - uses: ./.github/workflows/upgrade-tests.yaml - with: - skip: ${{ needs.validate.outputs.skip }} + name: ManagedCoverage + path: coverage-report + if-no-files-found: error result: runs-on: ubuntu-latest diff --git a/.github/workflows/functional-tests.yaml b/.github/workflows/functional-tests.yaml index 7c3bf2d7f0..e5bbc9a200 100644 --- a/.github/workflows/functional-tests.yaml +++ b/.github/workflows/functional-tests.yaml @@ -38,11 +38,6 @@ on: required: false type: string default: '' - aot_coverage_probe: - description: 'Use the instrumented installer and collect coverage on x64 shard 0' - required: false - type: boolean - default: false secrets: vfs_token: description: 'Token for downloading VFSForGit artifacts (required for cross-repository downloads; defaults to GITHUB_TOKEN)' @@ -63,8 +58,6 @@ jobs: env: ARTIFACT_PREFIX: ${{ inputs.output_prefix && format('{0}_', inputs.output_prefix) || '' }} FT_MATRIX_NAME: ${{ format('{0}_{1}-{2}', matrix.configuration, matrix.architecture, matrix.nr) }} - AOT_COVERAGE_PROBE: ${{ inputs.aot_coverage_probe && matrix.architecture == 'x64' && matrix.nr == 0 }} - COVERAGE_SESSION_ID: gvfs-ci-${{ inputs.vfs_run_id || github.run_id }} strategy: matrix: @@ -111,7 +104,7 @@ jobs: continue-on-error: true uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: - name: ${{ env.AOT_COVERAGE_PROBE == 'true' && 'GVFS_Coverage_Release_x64' || format('GVFS_{0}_{1}', matrix.configuration, matrix.architecture) }} + name: GVFS_${{ matrix.configuration }}_${{ matrix.architecture }} path: gvfs repository: ${{ inputs.vfs_repository || github.repository }} run-id: ${{ inputs.vfs_run_id || github.run_id }} @@ -121,7 +114,7 @@ jobs: if: steps.skip.outputs.result != 'true' && steps.download-gvfs.outcome == 'failure' uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: - name: ${{ env.AOT_COVERAGE_PROBE == 'true' && 'GVFS_Coverage_Release_x64' || format('GVFS_{0}_{1}', matrix.configuration, matrix.architecture) }} + name: GVFS_${{ matrix.configuration }}_${{ matrix.architecture }} path: gvfs repository: ${{ inputs.vfs_repository || github.repository }} run-id: ${{ inputs.vfs_run_id || github.run_id }} @@ -160,42 +153,45 @@ jobs: run-id: ${{ inputs.vfs_run_id || github.run_id }} github-token: ${{ secrets.vfs_token || github.token }} - - name: Checkout coverage configuration and source - if: steps.skip.outputs.result != 'true' && env.AOT_COVERAGE_PROBE == 'true' + - name: Read installer coverage manifest + id: coverage_manifest + if: steps.skip.outputs.result != 'true' + shell: pwsh + env: + EXPECTED_ARCH: ${{ matrix.architecture }} + run: | + if (Test-Path gvfs\coverage-build.json) { + $manifest = Get-Content gvfs\coverage-build.json -Raw | ConvertFrom-Json + if ($manifest.SchemaVersion -ne 1 -or $manifest.SourceCommit -notmatch '^[a-f0-9]{40}$' -or $manifest.Repository -notmatch '^[\w.-]+/[\w.-]+$' -or $manifest.Architecture -ne $env:EXPECTED_ARCH -or $manifest.SessionId -notmatch '^[\w.-]+$') { + throw 'Invalid installer coverage manifest.' + } + "enabled=true" >> $env:GITHUB_OUTPUT + "source_commit=$($manifest.SourceCommit)" >> $env:GITHUB_OUTPUT + "repository=$($manifest.Repository)" >> $env:GITHUB_OUTPUT + "session_id=$($manifest.SessionId)" >> $env:GITHUB_OUTPUT + } else { + "enabled=false" >> $env:GITHUB_OUTPUT + } + + - name: Checkout compiled coverage source + if: steps.skip.outputs.result != 'true' && steps.coverage_manifest.outputs.enabled == 'true' uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: path: src + repository: ${{ steps.coverage_manifest.outputs.repository }} + ref: ${{ steps.coverage_manifest.outputs.source_commit }} + token: ${{ secrets.vfs_token || github.token }} persist-credentials: false - - name: Install coverage runtime and SDK - if: steps.skip.outputs.result != 'true' && env.AOT_COVERAGE_PROBE == 'true' - uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0 - with: - global-json-file: src/global.json - dotnet-version: 8.0.x - - - name: Install coverage tools - if: steps.skip.outputs.result != 'true' && env.AOT_COVERAGE_PROBE == 'true' - shell: pwsh - run: | - $collectors = @(& "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.exe" -products * -latest -find Common7\IDE\Extensions\Microsoft\CodeCoverage.Console\Microsoft.CodeCoverage.Console.exe) - if ($LASTEXITCODE -ne 0 -or $collectors.Count -ne 1) { throw 'Expected one Visual Studio coverage collector.' } - Split-Path -Parent $collectors[0] >> $env:GITHUB_PATH - & $collectors[0] --version - if ($LASTEXITCODE -ne 0) { throw 'Visual Studio coverage collector cannot start.' } - dotnet tool install dotnet-reportgenerator-globaltool --version 5.5.11 --tool-path coverage-tools --configfile src\nuget.config - if ($LASTEXITCODE -ne 0) { throw 'Coverage renderer installation failed.' } - New-Item -ItemType Directory -Path coverage -Force | Out-Null + - name: Locate coverage collector + if: steps.skip.outputs.result != 'true' && steps.coverage_manifest.outputs.enabled == 'true' + uses: ./src/.github/actions/setup-coverage - name: Start fixed-session coverage before service installation id: coverage_start - if: steps.skip.outputs.result != 'true' && env.AOT_COVERAGE_PROBE == 'true' + if: steps.skip.outputs.result != 'true' && steps.coverage_manifest.outputs.enabled == 'true' shell: pwsh - run: | - Microsoft.CodeCoverage.Console.exe collect --server-mode --background --session-id $env:COVERAGE_SESSION_ID --settings src\scripts\AotCoverage.config --output coverage\functional.cobertura.xml --output-format cobertura --log-file coverage\collector.log --log-level Verbose - if ($LASTEXITCODE -ne 0) { throw 'Coverage collector startup failed.' } - Microsoft.CodeCoverage.Console.exe snapshot $env:COVERAGE_SESSION_ID --output coverage\before-install.coverage - if ($LASTEXITCODE -ne 0) { throw 'Coverage collector readiness check failed.' } + run: src\scripts\Invoke-Coverage.ps1 -Action Start -Directory coverage -BuildManifest gvfs\coverage-build.json -Suite Functional -Shard ${{ matrix.nr }} - name: ProjFS details (pre-install) if: steps.skip.outputs.result != 'true' @@ -242,7 +238,7 @@ jobs: gvfs\logs - name: Record installed service identity - if: steps.skip.outputs.result != 'true' && env.AOT_COVERAGE_PROBE == 'true' + if: steps.skip.outputs.result != 'true' && steps.coverage_manifest.outputs.enabled == 'true' shell: pwsh run: | Get-CimInstance Win32_Service -Filter "Name = 'GVFS.Service'" | @@ -250,8 +246,10 @@ jobs: ConvertTo-Json | Set-Content coverage\installed-service.json - name: Verify service coverage before functional tests - if: steps.skip.outputs.result != 'true' && env.AOT_COVERAGE_PROBE == 'true' + if: steps.skip.outputs.result != 'true' && steps.coverage_manifest.outputs.enabled == 'true' shell: pwsh + env: + COVERAGE_SESSION_ID: ${{ steps.coverage_manifest.outputs.session_id }} run: | Microsoft.CodeCoverage.Console.exe snapshot $env:COVERAGE_SESSION_ID --output coverage\after-install.coverage if ($LASTEXITCODE -ne 0) { throw 'Post-installation coverage snapshot failed.' } @@ -261,6 +259,7 @@ jobs: Tee-Object -FilePath coverage\service-startup-probe.json - name: Run functional tests + id: functional_tests if: steps.skip.outputs.result != 'true' shell: cmd timeout-minutes: 60 @@ -271,29 +270,15 @@ jobs: ft\GVFS.FunctionalTests.exe /result:TestResult.xml --ci --slice=${{ matrix.nr }},12 - name: Stop coverage and verify SCM service capture - id: coverage_stop - if: always() && env.AOT_COVERAGE_PROBE == 'true' && steps.coverage_start.outcome == 'success' - shell: pwsh - run: | - Microsoft.CodeCoverage.Console.exe shutdown $env:COVERAGE_SESSION_ID - if ($LASTEXITCODE -ne 0) { throw 'Coverage collector shutdown failed.' } - src\scripts\Assert-AotCoverage.ps1 -CoveragePath coverage\functional.cobertura.xml | - Tee-Object -FilePath coverage\probe.json - - - name: Render single-shard coverage report - if: always() && env.AOT_COVERAGE_PROBE == 'true' && steps.coverage_stop.outcome == 'success' + if: always() && steps.coverage_start.outcome == 'success' shell: pwsh - run: | - .\coverage-tools\reportgenerator.exe -reports:coverage\functional.cobertura.xml -targetdir:coverage\report "-reporttypes:Html;MarkdownSummaryGithub;Cobertura;JsonSummary" "-filefilters:+$env:GITHUB_WORKSPACE\src\GVFS\*" "-title:Managed NativeAOT product - x64 shard 0 of 12" - if ($LASTEXITCODE -ne 0) { throw 'Coverage rendering failed.' } - 'One-shard probe only. Native C++ and the other shards are not measured.' >> $env:GITHUB_STEP_SUMMARY - Get-Content coverage\report\SummaryGithub.md >> $env:GITHUB_STEP_SUMMARY + run: src\scripts\Invoke-Coverage.ps1 -Action Stop -Directory coverage -TestOutcome '${{ steps.functional_tests.outcome }}' - - name: Upload coverage-probe evidence - if: always() && steps.skip.outputs.result != 'true' && env.AOT_COVERAGE_PROBE == 'true' + - name: Upload worker coverage + if: always() && steps.coverage_start.outcome == 'success' uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: - name: ${{ env.ARTIFACT_PREFIX }}AotCoverage_${{ env.FT_MATRIX_NAME }} + name: ${{ env.ARTIFACT_PREFIX }}Coverage_Functional_${{ matrix.architecture }}_${{ matrix.nr }} path: coverage if-no-files-found: error diff --git a/.github/workflows/upgrade-tests.yaml b/.github/workflows/upgrade-tests.yaml index 75fdbfd3be..94b55497ce 100644 --- a/.github/workflows/upgrade-tests.yaml +++ b/.github/workflows/upgrade-tests.yaml @@ -94,6 +94,42 @@ jobs: name: GVFS_${{ matrix.configuration }}_x64 path: gvfs-new + - name: Read installer coverage manifest + id: coverage_manifest + if: steps.skip.outputs.result != 'true' + shell: pwsh + run: | + if (Test-Path gvfs-new\coverage-build.json) { + $manifest = Get-Content gvfs-new\coverage-build.json -Raw | ConvertFrom-Json + if ($manifest.SchemaVersion -ne 1 -or $manifest.SourceCommit -notmatch '^[a-f0-9]{40}$' -or $manifest.Repository -notmatch '^[\w.-]+/[\w.-]+$' -or $manifest.Architecture -ne 'x64' -or $manifest.SessionId -notmatch '^[\w.-]+$') { + throw 'Invalid installer coverage manifest.' + } + "enabled=true" >> $env:GITHUB_OUTPUT + "source_commit=$($manifest.SourceCommit)" >> $env:GITHUB_OUTPUT + "repository=$($manifest.Repository)" >> $env:GITHUB_OUTPUT + } else { + "enabled=false" >> $env:GITHUB_OUTPUT + } + + - name: Checkout compiled coverage source + if: steps.skip.outputs.result != 'true' && steps.coverage_manifest.outputs.enabled == 'true' + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: src + repository: ${{ steps.coverage_manifest.outputs.repository }} + ref: ${{ steps.coverage_manifest.outputs.source_commit }} + persist-credentials: false + + - name: Locate coverage collector + if: steps.skip.outputs.result != 'true' && steps.coverage_manifest.outputs.enabled == 'true' + uses: ./src/.github/actions/setup-coverage + + - name: Start coverage before upgrade scenario + id: coverage_start + if: steps.skip.outputs.result != 'true' && steps.coverage_manifest.outputs.enabled == 'true' + shell: pwsh + run: src\scripts\Invoke-Coverage.ps1 -Action Start -Directory coverage -BuildManifest gvfs-new\coverage-build.json -Suite Upgrade -Scenario '${{ matrix.scenario }}' + # -- Setup -- - name: Install Git @@ -113,6 +149,7 @@ jobs: # -- Test Execution -- - name: Run upgrade test - ${{ matrix.scenario }} + id: upgrade_test if: steps.skip.outputs.result != 'true' shell: pwsh run: | @@ -385,6 +422,19 @@ jobs: } } + - name: Stop upgrade coverage + if: always() && steps.coverage_start.outcome == 'success' + shell: pwsh + run: src\scripts\Invoke-Coverage.ps1 -Action Stop -Directory coverage -TestOutcome '${{ steps.upgrade_test.outcome }}' + + - name: Upload upgrade coverage + if: always() && steps.coverage_start.outcome == 'success' + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: Coverage_Upgrade_x64_${{ matrix.scenario }} + path: coverage + if-no-files-found: error + - name: Upload service logs if: always() && steps.skip.outputs.result != 'true' uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 diff --git a/Directory.Build.targets b/Directory.Build.targets index 7aa29b1094..04c79999a2 100644 --- a/Directory.Build.targets +++ b/Directory.Build.targets @@ -12,7 +12,7 @@ - + true diff --git a/scripts/Assert-AotCoverage.Tests.ps1 b/scripts/Assert-AotCoverage.Tests.ps1 index 282201c07c..c2a6b62135 100644 --- a/scripts/Assert-AotCoverage.Tests.ps1 +++ b/scripts/Assert-AotCoverage.Tests.ps1 @@ -55,7 +55,7 @@ Describe 'NativeAOT SCM coverage evidence' { $path = Save-CoverageReport $document $result = & $assertion -CoveragePath $path -ServiceOnly | ConvertFrom-Json $result.ScmServiceStartupCovered | Should -BeTrue - $result.Scope | Should -Be 'SCM service startup before tests' + $result.Scope | Should -Be 'SCM service execution' } It 'Rejects metadata-only service coverage in the startup snapshot' { diff --git a/scripts/Assert-AotCoverage.ps1 b/scripts/Assert-AotCoverage.ps1 index ba7c9a9253..7f39df7a0f 100644 --- a/scripts/Assert-AotCoverage.ps1 +++ b/scripts/Assert-AotCoverage.ps1 @@ -54,7 +54,7 @@ if ($consoleHits.Count -ne 0) { } [ordered]@{ - Scope = if ($ServiceOnly) { 'SCM service startup before tests' } else { 'Managed NativeAOT product; x64 shard 0 of 12' } + Scope = if ($ServiceOnly) { 'SCM service execution' } else { 'Managed NativeAOT product' } Modules = $packages.name ScmServiceStartupCovered = $true ConsoleServiceEntryCovered = $false diff --git a/scripts/Invoke-Coverage.Tests.ps1 b/scripts/Invoke-Coverage.Tests.ps1 new file mode 100644 index 0000000000..e8b379b58a --- /dev/null +++ b/scripts/Invoke-Coverage.Tests.ps1 @@ -0,0 +1,59 @@ +BeforeAll { + $lifecycle = Join-Path $PSScriptRoot 'Invoke-Coverage.ps1' + $sourceCommit = git -C (Split-Path -Parent $PSScriptRoot) rev-parse HEAD + if ($LASTEXITCODE -ne 0) { throw 'Cannot identify test source commit.' } + + function New-BuildManifest { + param([hashtable]$Overrides = @{}) + $manifest = @{ + SchemaVersion = 1 + SourceCommit = $sourceCommit + SourceRoot = 'C:\build\src' + Repository = 'microsoft/VFSForGit' + SessionId = 'test-session' + Architecture = 'x64' + Configuration = 'Release' + } + foreach ($key in $Overrides.Keys) { $manifest[$key] = $Overrides[$key] } + $path = Join-Path $TestDrive 'build.json' + $manifest | ConvertTo-Json | Set-Content $path + return $path + } +} + +Describe 'Coverage lifecycle input validation' { + BeforeEach { + Mock Get-Command { [pscustomobject]@{ Source = 'unreachable-collector.exe' } } -ParameterFilter { + $Name -eq 'Microsoft.CodeCoverage.Console.exe' + } + } + + It 'Rejects malformed build identity before a collector starts' { + $path = New-BuildManifest @{ SourceCommit = 'not-a-commit' } + { & $lifecycle -Action Start -Directory $TestDrive -BuildManifest $path -Suite Unit } | + Should -Throw '*Invalid coverage build manifest*' + } + + It 'Rejects a build from another source checkout' { + $path = New-BuildManifest @{ SourceCommit = '0000000000000000000000000000000000000000' } + { & $lifecycle -Action Start -Directory $TestDrive -BuildManifest $path -Suite Unit } | + Should -Throw '*does not match the compiled manifest*' + } + + It 'Rejects functional shard identifiers outside the existing matrix' { + $path = New-BuildManifest + { & $lifecycle -Action Start -Directory $TestDrive -BuildManifest $path -Suite Functional -Shard 12 } | + Should -Throw '*Invalid functional shard*' + } + + It 'Rejects upgrade collection without a scenario identity' { + $path = New-BuildManifest + { & $lifecycle -Action Start -Directory $TestDrive -BuildManifest $path -Suite Upgrade } | + Should -Throw '*requires a scenario*' + } + + It 'Requires a test outcome before completion can be recorded' { + { & $lifecycle -Action Stop -Directory $TestDrive } | + Should -Throw '*Stop requires the test outcome*' + } +} diff --git a/scripts/Invoke-Coverage.ps1 b/scripts/Invoke-Coverage.ps1 new file mode 100644 index 0000000000..397f0c3ac1 --- /dev/null +++ b/scripts/Invoke-Coverage.ps1 @@ -0,0 +1,84 @@ +param( + [Parameter(Mandatory)] + [ValidateSet('Start', 'Stop')] + [string]$Action, + [Parameter(Mandatory)] + [string]$Directory, + [string]$BuildManifest, + [ValidateSet('Unit', 'Functional', 'Upgrade')] + [string]$Suite, + [int]$Shard = -1, + [string]$Scenario, + [ValidateSet('success', 'failure', 'skipped', 'cancelled')] + [string]$TestOutcome +) + +$ErrorActionPreference = 'Stop' +$collector = (Get-Command Microsoft.CodeCoverage.Console.exe -ErrorAction Stop).Source +$metadataPath = Join-Path $Directory 'metadata.json' +$coveragePath = Join-Path $Directory 'coverage.cobertura.xml' + +if ($Action -eq 'Start') { + if (-not $BuildManifest -or -not $Suite) { throw 'Start requires a build manifest and suite.' } + $build = Get-Content -LiteralPath $BuildManifest -Raw | ConvertFrom-Json + if ($build.SchemaVersion -ne 1 -or $build.SourceCommit -notmatch '^[a-f0-9]{40}$' -or + $build.Architecture -notin @('x64', 'arm64') -or $build.SessionId -notmatch '^[\w.-]+$' -or -not $build.SourceRoot -or $build.Configuration -ne 'Release') { + throw 'Invalid coverage build manifest.' + } + $sourceCommit = git -C (Split-Path -Parent $PSScriptRoot) rev-parse HEAD + if ($LASTEXITCODE -ne 0 -or $sourceCommit -ne $build.SourceCommit) { + throw 'Collector source checkout does not match the compiled manifest.' + } + if ($Suite -eq 'Functional' -and ($Shard -lt 0 -or $Shard -ge 12)) { throw 'Invalid functional shard.' } + if ($Suite -eq 'Upgrade' -and -not $Scenario) { throw 'Upgrade coverage requires a scenario.' } + New-Item -ItemType Directory -Path $Directory -Force | Out-Null + $collectorVersion = (& $collector --version | Out-String).Trim() + if ($LASTEXITCODE -ne 0) { throw 'Cannot identify the coverage collector version.' } + $metadata = [ordered]@{ + SchemaVersion = 1 + Repository = $build.Repository + SourceCommit = $build.SourceCommit + SourceRoot = $build.SourceRoot + SessionId = $build.SessionId + Architecture = $build.Architecture + Configuration = $build.Configuration + Suite = $Suite + Shard = $Shard + Scenario = $Scenario + Complete = $false + TestOutcome = $null + Collector = $collector + CollectorVersion = $collectorVersion + StartedUtc = [DateTime]::UtcNow.ToString('o') + CompletedUtc = $null + } + $metadata | ConvertTo-Json | Set-Content -LiteralPath $metadataPath -Encoding utf8 + $configuration = Join-Path $PSScriptRoot $(if ($Suite -eq 'Unit') { 'UnitCoverage.config' } else { 'AotCoverage.config' }) + & $collector collect --server-mode --background --session-id $build.SessionId --settings $configuration --output $coveragePath --output-format cobertura --log-file (Join-Path $Directory 'collector.log') --log-level Info + if ($LASTEXITCODE -ne 0) { throw 'Coverage collector startup failed.' } + try { + & $collector snapshot $build.SessionId --output (Join-Path $Directory 'before-tests.coverage') + if ($LASTEXITCODE -ne 0) { throw 'Coverage collector readiness check failed.' } + } catch { + & $collector shutdown $build.SessionId + throw + } +} else { + if (-not $TestOutcome) { throw 'Stop requires the test outcome.' } + $metadata = Get-Content -LiteralPath $metadataPath -Raw | ConvertFrom-Json + & $collector shutdown $metadata.SessionId + if ($LASTEXITCODE -ne 0) { throw 'Coverage collector shutdown failed.' } + [xml]$report = Get-Content -LiteralPath $coveragePath -Raw + if ([int]$report.coverage.'lines-covered' -le 0) { throw 'Coverage report contains no executed lines.' } + if ($metadata.Suite -ne 'Unit') { + & (Join-Path $PSScriptRoot 'Assert-AotCoverage.ps1') -CoveragePath $coveragePath -ServiceOnly | + Set-Content -LiteralPath (Join-Path $Directory 'service.json') -Encoding utf8 + } + if ($metadata.Suite -in @('Unit', 'Functional')) { + Copy-Item -LiteralPath 'TestResult.xml' -Destination (Join-Path $Directory 'tests.xml') + } + $metadata.TestOutcome = $TestOutcome + $metadata.Complete = $true + $metadata.CompletedUtc = [DateTime]::UtcNow.ToString('o') + $metadata | ConvertTo-Json | Set-Content -LiteralPath $metadataPath -Encoding utf8 +} diff --git a/scripts/Merge-Coverage.Tests.ps1 b/scripts/Merge-Coverage.Tests.ps1 new file mode 100644 index 0000000000..f4ca29cb1c --- /dev/null +++ b/scripts/Merge-Coverage.Tests.ps1 @@ -0,0 +1,350 @@ +BeforeAll { + $mergeScript = Join-Path $PSScriptRoot 'Merge-Coverage.ps1' + $modules = @('GVFS', 'GVFS.Hooks', 'GVFS.Mount', 'GVFS.Service', 'GVFS.Common', 'GVFS.Platform.Windows', 'GVFS.Virtualization') + $scenarios = @('staging-upgrade', 'clean-upgrade', 'double-staging', 'staging-then-clean', 'mount-safety-deferral', 'unmount-all-triggers-upgrade') + $commit = 'a' * 40 + $repository = 'microsoft/VFSForGit' + $runnerRoot = 'D:\a\VFSForGit\VFSForGit\src' + + function New-Worker { + param([string]$Suite, [string]$Architecture, [int]$Shard = -1, [string]$Scenario) + + $id = "$($Suite.ToLowerInvariant())-$Architecture" + if ($Suite -eq 'Functional') { $id += "-shard-$Shard" } + if ($Suite -eq 'Upgrade') { $id += "-$Scenario" } + $directory = Join-Path $inputPath $id + [void][System.IO.Directory]::CreateDirectory($directory) + [ordered]@{ + SchemaVersion = 1; Repository = $repository; SourceCommit = $commit; SourceRoot = $runnerRoot + SessionId = "session-$id"; Architecture = $Architecture; Configuration = 'Release' + Suite = $Suite; Shard = $Shard; Scenario = if ($Suite -eq 'Upgrade') { $Scenario } else { $null } + Complete = $true; TestOutcome = 'success' + } | ConvertTo-Json | Set-Content -LiteralPath (Join-Path $directory 'metadata.json') + $workerModules = if ($Suite -eq 'Unit') { @('GVFS.UnitTests', 'GVFS.Common') } + elseif ($Suite -eq 'Upgrade') { @('GVFS.Service') } else { $modules } + $packages = foreach ($module in $workerModules) { + $secondHits = if ($Suite -eq 'Unit') { 1 } else { 0 } + $firstHits = 1 + if ($Suite -eq 'Unit' -and $module -eq 'GVFS.Common') { + $firstHits = 0 + $secondHits = 0 + } + @" + + + + + +"@ + } + [xml]$xml = "$runnerRoot$($packages -join '')" + $xml.Save((Join-Path $directory 'coverage.cobertura.xml')) + if ($Suite -ne 'Upgrade') { + @' + + + + + +'@ | Set-Content -LiteralPath (Join-Path $directory 'tests.xml') + } + } + + function New-Plan { + foreach ($architecture in @('x64', 'arm64')) { + New-Worker Unit $architecture + foreach ($shard in 0..11) { New-Worker Functional $architecture -Shard $shard } + } + foreach ($scenario in $scenarios) { New-Worker Upgrade x64 -Scenario $scenario } + } + + function Invoke-Merge { + param([string]$Base) + $arguments = @{ + InputDirectory = $inputPath; OutputDirectory = $outputPath; SourceDirectory = $sourcePath + SourceCommit = $commit; Repository = $repository + } + if ($Base) { $arguments.BaseCommit = $Base } + & $mergeScript @arguments + } + + function Set-WorkerMetadata { + param([string]$Id, [string]$Field, $Value) + $path = Join-Path $inputPath "$Id\metadata.json" + $metadata = Get-Content -LiteralPath $path -Raw | ConvertFrom-Json -AsHashtable + $metadata[$Field] = $Value + $metadata | ConvertTo-Json | Set-Content -LiteralPath $path + } + + function Edit-WorkerCoverage { + param([string]$Id, [scriptblock]$Edit) + $path = Join-Path $inputPath "$Id\coverage.cobertura.xml" + [xml]$document = Get-Content -LiteralPath $path -Raw + & $Edit $document + $document.Save($path) + } + + function Invoke-TestGit { + param([string[]]$Arguments) + $result = git -C $sourcePath @Arguments 2>&1 + if ($LASTEXITCODE -ne 0) { throw "Fixture git command failed: $result" } + return $result + } +} + +Describe 'Managed-source coverage aggregation integrity' { + BeforeEach { + $root = Join-Path $TestDrive ([guid]::NewGuid().ToString('N')) + $inputPath = Join-Path $root 'input' + $outputPath = Join-Path $root 'output' + $sourcePath = Join-Path $root 'src' + [void][System.IO.Directory]::CreateDirectory($inputPath) + [void][System.IO.Directory]::CreateDirectory((Join-Path $sourcePath 'GVFS\GVFS.Common')) + "first`nsecond`nthird" | Set-Content -LiteralPath (Join-Path $sourcePath 'GVFS\GVFS.Common\Policy.cs') + New-Plan + } + + It 'Accepts exactly 32 workers, keeps assembly attribution, and counts fixture skips without execution' { + Set-WorkerMetadata unit-x64 CollectorVersion '18.8.0' + Set-WorkerMetadata unit-x64 StartedUtc '2026-11-05T20:00:00Z' + Set-WorkerMetadata unit-x64 CompletedUtc '2026-11-05T20:01:00Z' + Invoke-Merge + $summary = Get-Content -LiteralPath (Join-Path $outputPath 'summary.json') -Raw | ConvertFrom-Json + $summary.Complete | Should -BeTrue + $summary.Workers | Should -Be 32 + $summary.ExpectedWorkers | Should -Be 32 + $summary.Groups.Count | Should -Be 5 + foreach ($group in @($summary.Groups | Where-Object Suite -eq 'Functional')) { + $group.Workers | Should -Be 12 + $group.ExpectedWorkers | Should -Be 12 + $group.Tests.Passed | Should -Be 12 + $group.Tests.Skipped | Should -Be 12 + $group.Tests.Executed | Should -Be 12 + } + $summary.Groups[0].Tests.Passed | Should -Be 1 + $summary.Groups[0].Tests.Skipped | Should -Be 1 + $summary.Groups[4].Tests | Should -BeNullOrEmpty + $summary.Measurements.PRODUCT.Measured | Should -Be 2 + $summary.Measurements.PRODUCT.Hit | Should -Be 1 + $summary.Measurements.UNIT.Measured | Should -Be 2 + $summary.Measurements.UNIT.Hit | Should -Be 2 + ($summary.WorkerReports | Where-Object WorkerId -eq 'unit-x64').Modules[0].Coverage.Hit | Should -Be 0 + ($summary.WorkerReports | Where-Object WorkerId -eq 'unit-x64').CollectorVersion | Should -Be '18.8.0' + $completedUtc = [datetime]($summary.WorkerReports | Where-Object WorkerId -eq 'unit-x64').CompletedUtc + $completedUtc.ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ssZ') | Should -Be '2026-11-05T20:01:00Z' + @(Get-ChildItem -LiteralPath (Join-Path $outputPath 'normalized') -Filter '*.xml' -Recurse).Count | Should -Be 32 + [xml]$unit = Get-Content -LiteralPath (Join-Path $outputPath 'normalized\unit-x64\unit-x64.cobertura.xml') -Raw + @($unit.coverage.packages.package.name) | Should -Contain 'GVFS.UnitTests' + $unit.coverage.'lines-valid' | Should -Be '4' + $unit.SelectSingleNode('//method/lines/line').hits | Should -Be '99' + (Get-Content -LiteralPath (Join-Path $outputPath 'Summary.md') -Raw) | Should -Match '12/12' + $changed = Get-Content -LiteralPath (Join-Path $outputPath 'changed-lines.json') -Raw | ConvertFrom-Json + $changed.Available | Should -BeFalse + (Get-Content -LiteralPath (Join-Path $outputPath 'ChangedLines.md') -Raw) | Should -Match 'unavailable because BaseCommit is absent' + $before = Get-Content -LiteralPath (Join-Path $outputPath 'summary.json') -Raw + Invoke-Merge + (Get-Content -LiteralPath (Join-Path $outputPath 'summary.json') -Raw) | Should -BeExactly $before + } + + It 'Rejects a missing worker and invalidates an old complete report' { + Invoke-Merge + Remove-Item -LiteralPath (Join-Path $inputPath 'functional-arm64-shard-11') -Recurse -Force + { Invoke-Merge } | Should -Throw '*Missing workers (31/32)*functional-arm64-shard-11*' + $summary = Get-Content -LiteralPath (Join-Path $outputPath 'summary.json') -Raw | ConvertFrom-Json + $summary.Complete | Should -BeFalse + Test-Path -LiteralPath (Join-Path $outputPath 'Summary.md') | Should -BeFalse + } + + It 'Rejects duplicate worker identities instead of treating them as another shard' { + Copy-Item -LiteralPath (Join-Path $inputPath 'unit-x64') -Destination (Join-Path $inputPath 'duplicate') -Recurse + { Invoke-Merge } | Should -Throw '*Duplicate worker unit-x64*' + } + + It 'Rejects mixed source commits and foreign repository evidence' { + Set-WorkerMetadata unit-x64 SourceCommit ('b' * 40) + { Invoke-Merge } | Should -Throw '*SourceCommit mismatch*' + Set-WorkerMetadata unit-x64 SourceCommit $commit + Set-WorkerMetadata unit-x64 Repository 'someone/other' + { Invoke-Merge } | Should -Throw '*Foreign repository*' + } + + It 'Rejects incomplete collectors, failed or skipped outcomes, malformed metadata, and invalid identities' { + foreach ($mutation in @( + @{ Field = 'Complete'; Value = $false; Original = $true; Error = '*Incomplete worker*' }, + @{ Field = 'Complete'; Value = 'true'; Original = $true; Error = '*Incomplete worker*' }, + @{ Field = 'TestOutcome'; Value = 'failure'; Original = 'success'; Error = '*TestOutcome*not success*' }, + @{ Field = 'TestOutcome'; Value = 'skipped'; Original = 'success'; Error = '*TestOutcome*not success*' }, + @{ Field = 'SchemaVersion'; Value = 2; Original = 1; Error = '*SchemaVersion*' }, + @{ Field = 'Shard'; Value = 12; Original = 0; Error = '*Invalid Functional shard*' }, + @{ Field = 'Architecture'; Value = 'other'; Original = 'x64'; Error = '*Invalid worker configuration*' } + )) { + Set-WorkerMetadata functional-x64-shard-0 $mutation.Field $mutation.Value + { Invoke-Merge } | Should -Throw $mutation.Error + Set-WorkerMetadata functional-x64-shard-0 $mutation.Field $mutation.Original + } + Set-WorkerMetadata upgrade-x64-clean-upgrade Scenario 'not-a-scenario' + { Invoke-Merge } | Should -Throw '*Invalid Upgrade*' + } + + It 'Remaps another runner with case and slash differences, excludes generated/external documents, and preserves raw XML' { + $rawPath = Join-Path $inputPath 'functional-x64-shard-0\coverage.cobertura.xml' + Edit-WorkerCoverage functional-x64-shard-0 { + param($document) + $classes = $document.SelectSingleNode('//package/classes') + $template = $classes.SelectSingleNode('./class') + $template.SetAttribute('filename', 'd:/A/vfsforgit/VFSFORGIT/SRC/gvfs/gvfs.common/policy.cs') + foreach ($filename in @("$runnerRoot\obj\Generated.g.cs", 'C:\framework\External.cs')) { + $class = $template.CloneNode($true) + $class.SetAttribute('filename', $filename) + [void]$classes.AppendChild($class) + } + } + $raw = Get-Content -LiteralPath $rawPath -Raw + $raw.Replace('', 'Setup failed') | + Set-Content -LiteralPath $path + { Invoke-Merge } | Should -Throw '*Failed NUnit tests or fixture errors*' + '' | + Set-Content -LiteralPath $path + { Invoke-Merge } | Should -Throw '*No executed NUnit tests*' + '' | Set-Content -LiteralPath $path + { Invoke-Merge } | Should -Throw '*Invalid count*' + $original.Replace('passed="1"', 'passed="2"') | Set-Content -LiteralPath $path + { Invoke-Merge } | Should -Throw '*Inconsistent NUnit test counts*' + Remove-Item -LiteralPath $path + { Invoke-Merge } | Should -Throw '*Missing tests.xml*' + } + + It 'Uses the tested git commits, separates unit-only hits, and keeps native, test, non-code, and no-entry changes visible with N/A' { + [void][System.IO.Directory]::CreateDirectory((Join-Path $sourcePath 'GVFS\GVFS.UnitTests')) + [void][System.IO.Directory]::CreateDirectory((Join-Path $sourcePath 'GVFS\GVFS.NativeTests')) + foreach ($relative in @('GVFS\GVFS.Common\No Entry.cs', 'GVFS\GVFS.Common\No é Entry.cs', 'GVFS\GVFS.Common\Native.cpp', 'GVFS\GVFS.UnitTests\Test.cs', 'GVFS\GVFS.NativeTests\Test.cpp', 'Readme.md')) { + 'old' | Set-Content -LiteralPath (Join-Path $sourcePath $relative) + } + [System.IO.File]::WriteAllBytes((Join-Path $sourcePath 'binary.dat'), [byte[]]@(0, 1, 2, 3)) + Invoke-TestGit @('init', '--quiet') | Out-Null + Invoke-TestGit @('config', 'user.email', 'fixture@example.invalid') | Out-Null + Invoke-TestGit @('config', 'user.name', 'Coverage fixture') | Out-Null + Invoke-TestGit @('config', 'core.autocrlf', 'false') | Out-Null + Invoke-TestGit @('config', 'commit.gpgsign', 'false') | Out-Null + Invoke-TestGit @('add', '.') | Out-Null + Invoke-TestGit @('-c', 'core.hooksPath=NUL', 'commit', '--quiet', '-m', 'Base fixture') | Out-Null + $base = [string](Invoke-TestGit @('rev-parse', 'HEAD')) + "first`nmodified`nthird" | Set-Content -LiteralPath (Join-Path $sourcePath 'GVFS\GVFS.Common\Policy.cs') + foreach ($relative in @('GVFS\GVFS.Common\No Entry.cs', 'GVFS\GVFS.Common\No é Entry.cs', 'GVFS\GVFS.Common\Native.cpp', 'GVFS\GVFS.UnitTests\Test.cs', 'GVFS\GVFS.NativeTests\Test.cpp', 'Readme.md')) { + 'new' | Set-Content -LiteralPath (Join-Path $sourcePath $relative) + } + [System.IO.File]::WriteAllBytes((Join-Path $sourcePath 'binary.dat'), [byte[]]@(0, 1, 2, 4)) + Invoke-TestGit @('add', '.') | Out-Null + Invoke-TestGit @('-c', 'core.hooksPath=NUL', 'commit', '--quiet', '-m', 'Tested fixture') | Out-Null + $commit = [string](Invoke-TestGit @('rev-parse', 'HEAD')) + foreach ($path in Get-ChildItem -LiteralPath $inputPath -Filter 'metadata.json' -Recurse) { + Set-WorkerMetadata $path.Directory.Name SourceCommit $commit + } + # A later working-copy edit must not change the commit-based line selection. + "first`nmodified`nlater-uncommitted" | Set-Content -LiteralPath (Join-Path $sourcePath 'GVFS\GVFS.Common\Policy.cs') + Invoke-Merge -Base $base + $changed = Get-Content -LiteralPath (Join-Path $outputPath 'changed-lines.json') -Raw | ConvertFrom-Json + $changed.Available | Should -BeTrue + $changed.SourceCommit | Should -Be $commit + $policy = $changed.Files | Where-Object Path -eq 'GVFS/GVFS.Common/Policy.cs' + @($policy.AddedModifiedLines) | Should -Be @(2) + $policy.PRODUCT.Measured | Should -Be 1 + $policy.PRODUCT.Hit | Should -Be 0 + $policy.PRODUCT.Uncovered | Should -Be 1 + $policy.PRODUCT.Lines[0].State | Should -Be 'uncovered' + $policy.UNIT.Hit | Should -Be 1 + $policy.UNIT.Lines[0].Provenance.Module | Should -Contain 'GVFS.UnitTests' + $policy.PRODUCT.Lines[0].Provenance.Suite | Should -Not -Contain 'Unit' + $absent = $changed.Files | Where-Object Path -eq 'GVFS/GVFS.Common/No Entry.cs' + $absent.PRODUCT.Percentage | Should -BeNullOrEmpty + $absent.PRODUCT.NoEntry | Should -Be 1 + $absent.PRODUCT.Lines[0].State | Should -Be 'no-entry/unmeasured-or-non-executable' + ($changed.Files | Where-Object Path -eq 'GVFS/GVFS.Common/No é Entry.cs').PRODUCT.NoEntry | Should -Be 1 + ($changed.Files | Where-Object Path -like '*Native.cpp').Classification | Should -Be 'unsupported-native' + ($changed.Files | Where-Object Path -like '*UnitTests/Test.cs').Classification | Should -Be 'intentionally-excluded-test-code' + ($changed.Files | Where-Object Path -like '*NativeTests/Test.cpp').Classification | Should -Be 'intentionally-excluded-test-code' + ($changed.Files | Where-Object Path -eq 'Readme.md').Classification | Should -Be 'not-applicable' + ($changed.Files | Where-Object Path -eq 'binary.dat').Classification | Should -Be 'not-applicable' + $changed.Files.Count | Should -Be 8 + (Get-Content -LiteralPath (Join-Path $outputPath 'ChangedLines.md') -Raw) | Should -Match 'N/A' + { Invoke-Merge -Base 'not-a-sha' } | Should -Throw '*BaseCommit must be a 40-hex*' + (Get-Content -LiteralPath (Join-Path $outputPath 'summary.json') -Raw | ConvertFrom-Json).Complete | Should -BeFalse + } +} diff --git a/scripts/Merge-Coverage.ps1 b/scripts/Merge-Coverage.ps1 new file mode 100644 index 0000000000..3ae9b55438 --- /dev/null +++ b/scripts/Merge-Coverage.ps1 @@ -0,0 +1,632 @@ +param( + [Parameter(Mandatory)] + [string]$InputDirectory, + [Parameter(Mandatory)] + [string]$OutputDirectory, + [Parameter(Mandatory)] + [string]$SourceDirectory, + [Parameter(Mandatory)] + [string]$SourceCommit, + [Parameter(Mandatory)] + [string]$Repository, + [string]$BaseCommit +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version Latest + +function Read-ReportXml { + param([string]$Path) + + $settings = [System.Xml.XmlReaderSettings]::new() + $settings.DtdProcessing = [System.Xml.DtdProcessing]::Ignore + $settings.XmlResolver = $null + $reader = [System.Xml.XmlReader]::Create($Path, $settings) + try { + $document = [System.Xml.XmlDocument]::new() + $document.XmlResolver = $null + $document.Load($reader) + return ,$document + } + finally { + $reader.Dispose() + } +} + +function Get-Count { + param([string]$Value, [string]$Context, [switch]$Positive) + + [long]$count = 0 + if ($Value -notmatch '^\d+$' -or + -not [long]::TryParse($Value, [ref]$count) -or ($Positive -and $count -eq 0)) { + throw "Invalid count '$Value' in $Context." + } + return $count +} + +function Get-WindowsPath { + param([string]$Path) + + $pathText = $Path.Replace('/', '\') + if ($pathText -notmatch '^(?:[A-Za-z]:\\|\\\\[^\\]+\\[^\\]+\\)') { + throw "Expected an absolute Windows source path: '$Path'." + } + return [System.IO.Path]::GetFullPath($pathText).TrimEnd('\') +} + +function Get-TestTotals { + param([string]$Path, [string]$WorkerId) + + $document = Read-ReportXml $Path + $run = $document.SelectSingleNode('/test-run') + if ($null -eq $run) { throw "Expected an NUnit test-run in $WorkerId." } + $total = Get-Count $run.GetAttribute('total') "$WorkerId tests" -Positive + $passed = Get-Count $run.GetAttribute('passed') "$WorkerId passed tests" + $failed = Get-Count $run.GetAttribute('failed') "$WorkerId failed tests" + $skipped = Get-Count $run.GetAttribute('skipped') "$WorkerId skipped tests" + $inconclusive = 0L + if ($run.HasAttribute('inconclusive')) { + $inconclusive = Get-Count $run.GetAttribute('inconclusive') "$WorkerId inconclusive tests" + } + $errors = 0L + if ($run.HasAttribute('errors')) { + $errors = Get-Count $run.GetAttribute('errors') "$WorkerId test errors" + } + if ($failed -ne 0 -or $errors -ne 0 -or $run.GetAttribute('result') -cne 'Passed' -or + $document.SelectNodes('//failure | //*[@result="Failed" or @result="Error"]').Count -ne 0) { + throw "Failed NUnit tests or fixture errors in $WorkerId." + } + $cases = @($document.SelectNodes('//test-case')) + $casePassed = @($cases | Where-Object { $_.GetAttribute('result') -ceq 'Passed' }).Count + $caseSkipped = @($cases | Where-Object { $_.GetAttribute('result') -ceq 'Skipped' }).Count + $caseInconclusive = @($cases | Where-Object { $_.GetAttribute('result') -ceq 'Inconclusive' }).Count + if ($passed + $failed + $skipped + $inconclusive -ne $total -or $cases.Count -ne $total -or + $casePassed -ne $passed -or $caseSkipped -ne $skipped -or $caseInconclusive -ne $inconclusive) { + throw "Inconsistent NUnit test counts in $WorkerId." + } + if ($passed + $inconclusive -eq 0) { + throw "No executed NUnit tests in $WorkerId. Skipped cases are not executed tests." + } + return [ordered]@{ + Total = $total + Executed = $passed + $inconclusive + Passed = $passed + Skipped = $skipped + Inconclusive = $inconclusive + Failed = $failed + Errors = $errors + } +} + +function Get-LineStatistics { + param([bool[]]$Lines) + + $measured = @($Lines).Count + $hit = @($Lines | Where-Object { $_ }).Count + return [ordered]@{ + Measured = $measured + Hit = $hit + Uncovered = $measured - $hit + Percentage = if ($measured -eq 0) { $null } else { [Math]::Round(100.0 * $hit / $measured, 2) } + } +} + +function Update-CoverageRates { + param([System.Xml.XmlElement]$Element) + + # Cobertura method lines repeat class lines. Only class lines define the denominator. + $lines = if ($Element.LocalName -eq 'class') { + @($Element.SelectNodes('./lines/line')) + } else { + @($Element.SelectNodes('.//class/lines/line')) + } + $covered = @($lines | Where-Object { [long]$_.GetAttribute('hits') -gt 0 }).Count + $rate = if ($lines.Count -eq 0) { '0' } else { + ($covered / [double]$lines.Count).ToString('R', [System.Globalization.CultureInfo]::InvariantCulture) + } + $Element.SetAttribute('line-rate', $rate) + if ($Element.LocalName -eq 'coverage') { + $Element.SetAttribute('lines-valid', [string]$lines.Count) + $Element.SetAttribute('lines-covered', [string]$covered) + } +} + +function ConvertFrom-GitPath { + param([string]$Path) + + if (-not $Path.StartsWith('"')) { return $Path } + if (-not $Path.EndsWith('"')) { throw "Invalid quoted git path: $Path." } + $bytes = [System.Collections.Generic.List[byte]]::new() + $text = $Path.Substring(1, $Path.Length - 2) + for ($index = 0; $index -lt $text.Length; $index++) { + if ($text[$index] -eq '\') { + $index++ + if ($index -ge $text.Length) { throw "Invalid git path escape: $Path." } + if ($text[$index] -match '[0-7]') { + $octal = [string]$text[$index] + for ($digit = 0; $digit -lt 2 -and $index + 1 -lt $text.Length -and $text[$index + 1] -match '[0-7]'; $digit++) { + $index++ + $octal += $text[$index] + } + $bytes.Add([Convert]::ToByte($octal, 8)) + } else { + $escapes = @{ a = 7; b = 8; t = 9; n = 10; v = 11; f = 12; r = 13; '\' = 92; '"' = 34 } + $key = [string]$text[$index] + if (-not $escapes.ContainsKey($key)) { throw "Invalid git path escape: $Path." } + $bytes.Add([byte]$escapes[$key]) + } + } else { + $length = if ([char]::IsHighSurrogate($text[$index])) { 2 } else { 1 } + $bytes.AddRange([System.Text.Encoding]::UTF8.GetBytes($text.Substring($index, $length))) + $index += $length - 1 + } + } + return [System.Text.Encoding]::UTF8.GetString($bytes.ToArray()) +} + +function Get-ChangedFiles { + $patch = @(git -C $SourceDirectory diff --unified=0 --no-ext-diff --no-textconv $BaseCommit $SourceCommit --) + if ($LASTEXITCODE -ne 0) { throw 'Cannot read the diff between BaseCommit and the tested SourceCommit.' } + $files = @{} + $file = $null + $inHunk = $false + $oldRemaining = 0 + $newRemaining = 0 + $newLine = 0 + foreach ($text in $patch) { + if ($text -match '^diff --git (?"(?:\\.|[^"])*"|a/.*?) (?"(?:\\.|[^"])*"|b/.*)$') { + $path = (ConvertFrom-GitPath $Matches.new).Substring(2) + $file = [ordered]@{ Path = $path; Lines = [System.Collections.Generic.HashSet[int]]::new() } + $files[$path] = $file + $inHunk = $false + } elseif ($text -match '^@@ -\d+(?:,(?\d+))? \+(?\d+)(?:,(?\d+))? @@') { + if ($null -eq $file) { throw 'A diff hunk has no source file.' } + $oldRemaining = if ($Matches.ContainsKey('oldCount')) { [int]$Matches.oldCount } else { 1 } + $newRemaining = if ($Matches.ContainsKey('newCount')) { [int]$Matches.newCount } else { 1 } + $newLine = [int]$Matches.start + $inHunk = $true + } elseif ($inHunk -and ($oldRemaining -gt 0 -or $newRemaining -gt 0)) { + if ($text.StartsWith('+')) { + [void]$file.Lines.Add($newLine) + $newLine++ + $newRemaining-- + } elseif ($text.StartsWith('-')) { + $oldRemaining-- + } elseif ($text.StartsWith(' ')) { + $newLine++ + $newRemaining-- + $oldRemaining-- + } elseif (-not $text.StartsWith('\ No newline')) { + throw "Invalid diff hunk for $($file.Path)." + } + } elseif ($text.StartsWith('+++ ')) { + $path = ConvertFrom-GitPath $text.Substring(4).TrimEnd("`t") + if ($path -ne '/dev/null') { + $oldPath = $file.Path + $file.Path = $path.Substring(2) + if ($oldPath -ne $file.Path) { + $files.Remove($oldPath) + $files[$file.Path] = $file + } + } + } + } + return @($files.Values | Sort-Object Path) +} + +function Get-ChangedMeasurement { + param([string]$Path, [int[]]$Numbers, [hashtable]$Index, [string]$Classification) + + $lines = @( + foreach ($number in $Numbers) { + $key = "$Path`0$number" + $entry = $Index[$key] + [ordered]@{ + Line = $number + State = if ($Classification -ne 'managed-source') { $Classification } + elseif ($null -eq $entry) { 'no-entry/unmeasured-or-non-executable' } + elseif ($entry.Hit) { 'hit' } else { 'uncovered' } + Provenance = if ($null -eq $entry -or $Classification -ne 'managed-source') { @() } else { + @($entry.Provenance.Values | Sort-Object WorkerId, Module, SourceDocument) + } + } + } + ) + $measured = @($lines | Where-Object { $_.State -in @('hit', 'uncovered') }) + $hit = @($measured | Where-Object { $_.State -eq 'hit' }).Count + return [ordered]@{ + Measured = $measured.Count + Hit = $hit + Uncovered = $measured.Count - $hit + NoEntry = @($lines | Where-Object { $_.State -eq 'no-entry/unmeasured-or-non-executable' }).Count + Percentage = if ($measured.Count -eq 0) { $null } else { [Math]::Round(100.0 * $hit / $measured.Count, 2) } + Lines = $lines + } +} + +function Format-Percentage { + param($Value) + if ($null -eq $Value) { return 'N/A' } + return $Value.ToString('0.##', [System.Globalization.CultureInfo]::InvariantCulture) + '%' +} + +$SourceDirectory = Get-WindowsPath (Resolve-Path -LiteralPath $SourceDirectory).Path +$InputDirectory = Get-WindowsPath (Resolve-Path -LiteralPath $InputDirectory).Path +$OutputDirectory = Get-WindowsPath $OutputDirectory +if ($OutputDirectory -eq $InputDirectory -or $OutputDirectory -eq $SourceDirectory -or + $InputDirectory.StartsWith("$OutputDirectory\", [StringComparison]::OrdinalIgnoreCase) -or + $SourceDirectory.StartsWith("$OutputDirectory\", [StringComparison]::OrdinalIgnoreCase)) { + throw 'OutputDirectory must not contain InputDirectory or SourceDirectory.' +} +[void][System.IO.Directory]::CreateDirectory($OutputDirectory) +$summaryPath = Join-Path $OutputDirectory 'summary.json' + +# Invalidate old success reports before any input check can fail. +foreach ($name in @('Summary.md', 'changed-lines.json', 'ChangedLines.md')) { + $path = Join-Path $OutputDirectory $name + if (Test-Path -LiteralPath $path) { Remove-Item -LiteralPath $path -Force } +} +[ordered]@{ SchemaVersion = 1; Complete = $false; Repository = $Repository; SourceCommit = $SourceCommit } | + ConvertTo-Json | Set-Content -LiteralPath $summaryPath -Encoding utf8 + +try { + if ($SourceCommit -notmatch '^[0-9a-fA-F]{40}$') { throw 'SourceCommit must be a 40-hex tested commit SHA.' } + if ($BaseCommit -and $BaseCommit -notmatch '^[0-9a-fA-F]{40}$') { throw 'BaseCommit must be a 40-hex commit SHA.' } + $requiredModules = @('GVFS', 'GVFS.Hooks', 'GVFS.Mount', 'GVFS.Service', 'GVFS.Common', 'GVFS.Platform.Windows', 'GVFS.Virtualization') + $scenarios = @('staging-upgrade', 'clean-upgrade', 'double-staging', 'staging-then-clean', 'mount-safety-deferral', 'unmount-all-triggers-upgrade') + $expected = @{} + foreach ($architecture in @('x64', 'arm64')) { + $expected["unit-$architecture"] = $true + foreach ($shard in 0..11) { $expected["functional-$architecture-shard-$shard"] = $true } + } + foreach ($scenario in $scenarios) { $expected["upgrade-x64-$scenario"] = $true } + + $workers = [System.Collections.Generic.List[object]]::new() + $seen = @{} + $functionalHits = @{} + $indexes = @{ PRODUCT = @{}; UNIT = @{} } + $changedFiles = if ($BaseCommit) { @(Get-ChangedFiles) } else { @() } + $changedLineKeys = @{} + foreach ($file in $changedFiles) { + foreach ($number in $file.Lines) { $changedLineKeys["$($file.Path)`0$number"] = $true } + } + $exclusions = @{} + $sourceFiles = @{} + foreach ($source in Get-ChildItem -LiteralPath $SourceDirectory -File -Recurse -Filter '*.cs') { + $relative = $source.FullName.Substring($SourceDirectory.Length + 1).Replace('\', '/') + $sourceFiles[$relative] = $source.FullName + } + $metadataFiles = @(Get-ChildItem -LiteralPath $InputDirectory -Filter 'metadata.json' -File -Recurse | Sort-Object FullName) + foreach ($coverageFile in Get-ChildItem -LiteralPath $InputDirectory -Filter 'coverage.cobertura.xml' -File -Recurse) { + if (-not (Test-Path -LiteralPath (Join-Path $coverageFile.DirectoryName 'metadata.json') -PathType Leaf)) { + throw "Missing metadata.json beside coverage artifact $($coverageFile.FullName)." + } + } + foreach ($metadataFile in $metadataFiles) { + $metadata = Get-Content -LiteralPath $metadataFile.FullName -Raw | ConvertFrom-Json -AsHashtable + if ($metadata -isnot [System.Collections.IDictionary]) { throw "Invalid metadata object in $($metadataFile.FullName)." } + foreach ($field in @('SchemaVersion', 'Repository', 'SourceCommit', 'SourceRoot', 'SessionId', 'Architecture', 'Configuration', 'Suite', 'Shard', 'Scenario', 'Complete', 'TestOutcome')) { + if (-not $metadata.ContainsKey($field)) { throw "Missing metadata field $field in $($metadataFile.FullName)." } + } + if ($metadata.SchemaVersion -isnot [long] -and $metadata.SchemaVersion -isnot [int]) { + throw "Invalid metadata SchemaVersion in $($metadataFile.FullName)." + } + if ($metadata.SchemaVersion -ne 1) { throw "Unsupported metadata SchemaVersion in $($metadataFile.FullName)." } + foreach ($field in @('Repository', 'SourceCommit', 'SourceRoot', 'SessionId', 'Architecture', 'Configuration', 'Suite', 'TestOutcome')) { + if ($metadata[$field] -isnot [string] -or [string]::IsNullOrWhiteSpace($metadata[$field])) { + throw "Invalid metadata $field in $($metadataFile.FullName)." + } + } + if ($metadata.Repository -ne $Repository) { throw "Foreign repository in $($metadataFile.FullName): $($metadata.Repository)." } + if ($metadata.SourceCommit -ne $SourceCommit) { throw "SourceCommit mismatch in $($metadataFile.FullName): $($metadata.SourceCommit)." } + if ($metadata.Configuration -cne 'Release' -or $metadata.Architecture -cnotin @('x64', 'arm64') -or + $metadata.Suite -cnotin @('Unit', 'Functional', 'Upgrade')) { + throw "Invalid worker configuration, architecture, or suite in $($metadataFile.FullName)." + } + if ($metadata.Shard -isnot [long] -and $metadata.Shard -isnot [int]) { throw "Invalid worker Shard in $($metadataFile.FullName)." } + $group = "$($metadata.Suite.ToLowerInvariant())-$($metadata.Architecture)" + if ($metadata.Suite -ceq 'Functional') { + if ($metadata.Shard -lt 0 -or $metadata.Shard -gt 11 -or $null -ne $metadata.Scenario) { + throw "Invalid Functional shard or scenario in $($metadataFile.FullName)." + } + $workerId = "$group-shard-$($metadata.Shard)" + } elseif ($metadata.Suite -ceq 'Upgrade') { + if ($metadata.Architecture -cne 'x64' -or $metadata.Shard -ne -1 -or $metadata.Scenario -cnotin $scenarios) { + throw "Invalid Upgrade architecture, shard, or scenario in $($metadataFile.FullName)." + } + $workerId = "$group-$($metadata.Scenario)" + } else { + if ($metadata.Shard -ne -1 -or $null -ne $metadata.Scenario) { throw "Invalid Unit shard or scenario in $($metadataFile.FullName)." } + $workerId = $group + } + if (-not $expected.ContainsKey($workerId)) { throw "Unexpected worker $workerId." } + if ($seen.ContainsKey($workerId)) { throw "Duplicate worker $workerId." } + $seen[$workerId] = $true + if ($metadata.Complete -isnot [bool] -or -not $metadata.Complete) { throw "Incomplete worker $workerId." } + if ($metadata.TestOutcome -cne 'success') { throw "Worker $workerId has TestOutcome '$($metadata.TestOutcome)', not success." } + $sourceRoot = Get-WindowsPath $metadata.SourceRoot + $coveragePath = Join-Path $metadataFile.DirectoryName 'coverage.cobertura.xml' + if (-not (Test-Path -LiteralPath $coveragePath -PathType Leaf)) { throw "Missing coverage.cobertura.xml for $workerId." } + $tests = $null + if ($metadata.Suite -cne 'Upgrade') { + $testPath = Join-Path $metadataFile.DirectoryName 'tests.xml' + if (-not (Test-Path -LiteralPath $testPath -PathType Leaf)) { throw "Missing tests.xml for $workerId." } + $tests = Get-TestTotals $testPath $workerId + } + $document = Read-ReportXml $coveragePath + $coverage = $document.SelectSingleNode('/coverage') + if ($null -eq $coverage) { throw "Invalid Cobertura document for $workerId." } + $modules = @{} + $workerLines = @{} + foreach ($package in @($coverage.SelectNodes('./packages/package'))) { + $module = $package.GetAttribute('name') + if ([string]::IsNullOrWhiteSpace($module)) { throw "Missing assembly name in $workerId." } + if ($metadata.Suite -cne 'Unit' -and $module -match '(UnitTests|FunctionalTests|^GVFS\.Tests(?:$|\.))') { + throw "Test assemblies contaminate product worker ${workerId}: $module." + } + if (-not $modules.ContainsKey($module)) { $modules[$module] = @{} } + foreach ($class in @($package.SelectNodes('./classes/class'))) { + $rawSource = $class.GetAttribute('filename') + if ([string]::IsNullOrWhiteSpace($rawSource)) { throw "Missing source filename in $workerId ($module)." } + $sourcePath = Get-WindowsPath $rawSource + $inside = $sourcePath.StartsWith("$sourceRoot\", [StringComparison]::OrdinalIgnoreCase) + $documentPath = if ($inside) { '\' + $sourcePath.Substring($sourceRoot.Length + 1) } else { $sourcePath } + $generated = $documentPath -match '(?i)\\(?:obj|bin|out|publish)\\' + if ($documentPath -match '(?i)(?:\.g(?:\.i)?|\.generated|\.AssemblyInfo|\.AssemblyAttributes)\.cs$') { + $generated = $generated -or -not $inside -or + -not $sourceFiles.ContainsKey($sourcePath.Substring($sourceRoot.Length + 1).Replace('\', '/')) + } + $external = $documentPath -match '(?i)\\(?:packages|\.nuget|nuget)\\' + $reason = $null + $relative = $null + if ($generated) { + $reason = 'generated-document' + } elseif ($external) { + $reason = 'external-document' + } elseif (-not $inside) { + $matchesRepository = $sourcePath -match '(?i)\\GVFS\\' + foreach ($candidate in $sourceFiles.Keys) { + if ($sourcePath.EndsWith("\" + $candidate.Replace('/', '\'), [StringComparison]::OrdinalIgnoreCase)) { + $matchesRepository = $true + break + } + } + if ($matchesRepository) { throw "Unmatched repository source in ${workerId}: '$rawSource' does not match SourceRoot '$sourceRoot'." } + $reason = 'external-document' + } else { + $relative = $sourcePath.Substring($sourceRoot.Length + 1).Replace('\', '/') + $mapped = Get-WindowsPath (Join-Path $SourceDirectory $relative.Replace('/', '\')) + if (-not $mapped.StartsWith("$SourceDirectory\", [StringComparison]::OrdinalIgnoreCase)) { + throw "Source path escapes SourceDirectory in ${workerId}: $rawSource." + } + if (-not (Test-Path -LiteralPath $mapped -PathType Leaf)) { throw "Missing repository source in ${workerId}: $relative." } + if ([System.IO.Path]::GetExtension($mapped) -ine '.cs') { + $reason = 'non-managed-document' + } else { + $mapped = $sourceFiles[$relative] + if ($null -eq $mapped) { throw "Missing repository source in ${workerId}: $relative." } + $relative = $mapped.Substring($SourceDirectory.Length + 1).Replace('\', '/') + } + } + if ($null -ne $reason) { + $key = "$workerId`0$sourcePath" + if (-not $exclusions.ContainsKey($key)) { + $exclusions[$key] = [ordered]@{ WorkerId = $workerId; Suite = $metadata.Suite; Architecture = $metadata.Architecture; SourceDocument = $rawSource; Reason = $reason; Modules = @{} } + } + $exclusions[$key].Modules[$module] = $true + [void]$class.ParentNode.RemoveChild($class) + continue + } + $class.SetAttribute('filename', $mapped) + $scope = if ($metadata.Suite -ceq 'Unit') { 'UNIT' } else { 'PRODUCT' } + foreach ($line in @($class.SelectNodes('./lines/line'))) { + $number = Get-Count $line.GetAttribute('number') "$workerId ($relative) line number" -Positive + $hits = Get-Count $line.GetAttribute('hits') "$workerId ($relative) line hits" + $key = "$relative`0$number" + $workerLines[$key] = $workerLines[$key] -eq $true -or $hits -gt 0 + $modules[$module][$key] = $modules[$module][$key] -eq $true -or $hits -gt 0 + if (-not $indexes[$scope].ContainsKey($key)) { + $indexes[$scope][$key] = [ordered]@{ Path = $relative; Line = $number; Hit = $false; Provenance = @{} } + } + $entry = $indexes[$scope][$key] + $entry.Hit = $entry.Hit -or $hits -gt 0 + # Unchanged lines need only hit unions. Detailed provenance stays bounded to the diff. + if ($changedLineKeys.ContainsKey($key)) { + $provenanceKey = "$workerId`0$module`0$sourcePath" + if (-not $entry.Provenance.ContainsKey($provenanceKey)) { + $entry.Provenance[$provenanceKey] = [ordered]@{ + WorkerId = $workerId; Suite = $metadata.Suite; Architecture = $metadata.Architecture + Module = $module; SourceDocument = $rawSource; Hit = $false + } + } + $entry.Provenance[$provenanceKey].Hit = $entry.Provenance[$provenanceKey].Hit -or $hits -gt 0 + } + if ($metadata.Suite -ceq 'Functional' -and $hits -gt 0) { $functionalHits[$module] = $true } + } + Update-CoverageRates $class + } + if ($package.SelectNodes('./classes/class').Count -eq 0) { + [void]$package.ParentNode.RemoveChild($package) + } else { + Update-CoverageRates $package + } + } + $statistics = Get-LineStatistics @($workerLines.Values) + if ($statistics.Hit -eq 0) { throw "No executed repository source coverage in $workerId." } + Update-CoverageRates $coverage + $sources = $coverage.SelectSingleNode('./sources') + if ($null -ne $sources) { [void]$coverage.RemoveChild($sources) } + $sources = $document.CreateElement('sources') + $sourceNode = $document.CreateElement('source') + $sourceNode.InnerText = $SourceDirectory + [void]$sources.AppendChild($sourceNode) + [void]$coverage.PrependChild($sources) + $moduleSummary = @( + foreach ($module in @($modules.Keys | Sort-Object)) { + [ordered]@{ Name = $module; Coverage = Get-LineStatistics @($modules[$module].Values) } + } + ) + $worker = [ordered]@{ + WorkerId = $workerId; Group = $group; Suite = $metadata.Suite; Architecture = $metadata.Architecture + Shard = $metadata.Shard; Scenario = $metadata.Scenario; SessionId = $metadata.SessionId + SourceRoot = $metadata.SourceRoot; TestOutcome = $metadata.TestOutcome; Complete = $true + Tests = $tests; Coverage = $statistics; Modules = $moduleSummary + NormalizedReport = "normalized\$group\$workerId.cobertura.xml"; Document = $document + } + foreach ($field in @('CollectorVersion', 'StartedUtc', 'CompletedUtc')) { + if ($metadata.ContainsKey($field)) { $worker[$field] = $metadata[$field] } + } + $workers.Add($worker) + } + $missing = @($expected.Keys | Where-Object { -not $seen.ContainsKey($_) } | Sort-Object) + if ($missing.Count -ne 0) { throw "Missing workers ($($seen.Count)/32): $($missing -join ', ')." } + foreach ($module in $requiredModules) { + if (-not $functionalHits.ContainsKey($module)) { throw "No executed Functional source coverage in required product module $module." } + } + + $changed = [ordered]@{ + SchemaVersion = 1; Repository = $Repository; SourceCommit = $SourceCommit; BaseCommit = $BaseCommit + Available = [bool]$BaseCommit + Reason = if ($BaseCommit) { $null } else { 'Changed-line analysis is unavailable because BaseCommit is absent.' } + Scopes = [ordered]@{ PRODUCT = 'Functional + Upgrade, managed NativeAOT product execution'; UNIT = 'Unit execution, including source-linked production code' } + Files = @() + } + if ($BaseCommit) { + $changed.Files = @( + foreach ($file in $changedFiles) { + $path = $file.Path + if ($path -match '(?i)(?:^|/)(?:GVFS\.)?(?:UnitTests|FunctionalTests|Tests|NativeTests)(?:/|$)') { + $classification = 'intentionally-excluded-test-code' + } elseif ($path -match '(?i)\.(?:c|cc|cpp|cxx|h|hh|hpp|hxx|inl|inc|m|mm)$') { + $classification = 'unsupported-native' + } elseif ($path -match '(?i)\.cs$') { + $classification = 'managed-source' + } else { + $classification = 'not-applicable' + } + $numbers = @($file.Lines | Sort-Object) + [ordered]@{ + Path = $path; Classification = $classification; AddedModifiedLines = $numbers + PRODUCT = Get-ChangedMeasurement $path $numbers $indexes.PRODUCT $classification + UNIT = Get-ChangedMeasurement $path $numbers $indexes.UNIT $classification + } + } + ) + } + $groups = @( + foreach ($group in @('unit-x64', 'unit-arm64', 'functional-x64', 'functional-arm64', 'upgrade-x64')) { + $members = @($workers | Where-Object { $_.Group -eq $group }) + $expectedCount = if ($group.StartsWith('unit-')) { 1 } elseif ($group.StartsWith('functional-')) { 12 } else { 6 } + $totals = $null + if (-not $group.StartsWith('upgrade-')) { + $totals = [ordered]@{} + foreach ($name in @('Total', 'Executed', 'Passed', 'Skipped', 'Inconclusive', 'Failed', 'Errors')) { + $totals[$name] = [long](($members | ForEach-Object { $_.Tests[$name] } | Measure-Object -Sum).Sum) + } + } + [ordered]@{ + Group = $group; Suite = $members[0].Suite; Architecture = $members[0].Architecture + Workers = $members.Count; ExpectedWorkers = $expectedCount; Tests = $totals + TestEvidence = if ($null -eq $totals) { 'Scenario outcome from metadata. No NUnit test count is inferred.' } else { 'NUnit cases. Skipped cases are not executed tests.' } + } + } + ) + $excludedDocuments = @( + foreach ($entry in @($exclusions.Values | Sort-Object WorkerId, SourceDocument)) { + $entry.Modules = @($entry.Modules.Keys | Sort-Object) + $entry + } + ) + $limitations = @( + 'Native C++ coverage is unsupported.' + 'Branch coverage availability is not assumed. These summaries use class source lines.' + 'Unit-linked source coverage is not deployed product execution.' + 'Lines without coverage entries are unmeasured or non-executable, not inferred covered.' + 'No coverage thresholds apply.' + ) + $summary = [ordered]@{ + SchemaVersion = 1; Complete = $true; Repository = $Repository; SourceCommit = $SourceCommit + Configuration = 'Release'; SourceDirectory = $SourceDirectory + Workers = $workers.Count; ExpectedWorkers = 32; Groups = $groups + Scopes = $changed.Scopes + Measurements = [ordered]@{ + PRODUCT = Get-LineStatistics @($indexes.PRODUCT.Values | ForEach-Object { $_.Hit }) + UNIT = Get-LineStatistics @($indexes.UNIT.Values | ForEach-Object { $_.Hit }) + } + RequiredFunctionalModulesWithHits = $requiredModules + ExcludedDocumentCount = $excludedDocuments.Count; ExcludedDocuments = $excludedDocuments + Limitations = $limitations + WorkerReports = @($workers | Sort-Object WorkerId | ForEach-Object { + $report = [ordered]@{} + foreach ($key in $_.Keys) { if ($key -ne 'Document') { $report[$key] = $_[$key] } } + $report + }) + } + $markdown = [System.Collections.Generic.List[string]]::new() + $markdown.Add('# Managed-source coverage') + $markdown.Add('') + $markdown.Add("Workers: $($workers.Count)/32. Tested commit: ``$SourceCommit``.") + foreach ($scope in @('PRODUCT', 'UNIT')) { + $measurement = $summary.Measurements[$scope] + $markdown.Add("$scope`: $($changed.Scopes[$scope]). $($measurement.Hit)/$($measurement.Measured) measured lines have hits ($(Format-Percentage $measurement.Percentage)).") + } + $markdown.Add('') + $markdown.Add('| Suite / architecture | Workers | Passed | Skipped | Executed |') + $markdown.Add('| --- | --- | --- | --- | --- |') + foreach ($group in $groups) { + $counts = if ($null -eq $group.Tests) { 'N/A | N/A | N/A' } else { + "$($group.Tests.Passed) | $($group.Tests.Skipped) | $($group.Tests.Executed)" + } + $markdown.Add("| $($group.Suite) / $($group.Architecture) | $($group.Workers)/$($group.ExpectedWorkers) | $counts |") + } + $markdown.Add('') + $markdown.Add("Excluded generated/external/non-managed documents: $($excludedDocuments.Count). See summary.json for documents and original module provenance.") + $markdown.Add('Upgrade metadata records scenario outcomes, not NUnit case counts.') + foreach ($limitation in $limitations) { $markdown.Add("- $limitation") } + + $changedMarkdown = [System.Collections.Generic.List[string]]::new() + $changedMarkdown.Add('# Changed source lines') + $changedMarkdown.Add('') + if (-not $BaseCommit) { + $changedMarkdown.Add($changed.Reason) + } else { + $changedMarkdown.Add("Diff: ``$BaseCommit`` to tested commit ``$SourceCommit``.") + $changedMarkdown.Add('PRODUCT uses Functional + Upgrade. UNIT uses unit execution. These measurements remain separate.') + $changedMarkdown.Add('Percentages use measured changed lines only. N/A means that no changed lines have coverage entries.') + $changedMarkdown.Add('No-entry means unmeasured or non-executable. It does not mean covered.') + $changedMarkdown.Add('') + $changedMarkdown.Add('| File | Scope | Measured | Hit | Uncovered | No-entry | Percentage | Classification |') + $changedMarkdown.Add('| --- | --- | --- | --- | --- | --- | --- | --- |') + foreach ($file in $changed.Files) { + foreach ($scope in @('PRODUCT', 'UNIT')) { + $measurement = $file[$scope] + $displayPath = $file.Path.Replace('|', '\|').Replace("`n", ' ').Replace("`r", ' ') + $changedMarkdown.Add("| $displayPath | $scope | $($measurement.Measured) | $($measurement.Hit) | $($measurement.Uncovered) | $($measurement.NoEntry) | $(Format-Percentage $measurement.Percentage) | $($file.Classification) |") + } + } + $changedMarkdown.Add('') + $changedMarkdown.Add('changed-lines.json lists new-side line numbers, per-line states, and source/module/worker provenance.') + } + $normalized = Join-Path $OutputDirectory 'normalized' + if (Test-Path -LiteralPath $normalized) { Remove-Item -LiteralPath $normalized -Recurse -Force } + foreach ($worker in @($workers | Sort-Object WorkerId)) { + $path = Join-Path $OutputDirectory $worker.NormalizedReport + [void][System.IO.Directory]::CreateDirectory((Split-Path -Parent $path)) + $worker.Document.Save($path) + } + $changed | ConvertTo-Json -Depth 20 | Set-Content -LiteralPath (Join-Path $OutputDirectory 'changed-lines.json') -Encoding utf8 + $changedMarkdown | Set-Content -LiteralPath (Join-Path $OutputDirectory 'ChangedLines.md') -Encoding utf8 + $markdown | Set-Content -LiteralPath (Join-Path $OutputDirectory 'Summary.md') -Encoding utf8 + $summary | ConvertTo-Json -Depth 20 | Set-Content -LiteralPath $summaryPath -Encoding utf8 +} +catch { + foreach ($name in @('Summary.md', 'changed-lines.json', 'ChangedLines.md')) { + $path = Join-Path $OutputDirectory $name + if (Test-Path -LiteralPath $path) { Remove-Item -LiteralPath $path -Force } + } + [ordered]@{ + SchemaVersion = 1; Complete = $false; Repository = $Repository; SourceCommit = $SourceCommit + Error = $_.Exception.Message + } | ConvertTo-Json | Set-Content -LiteralPath $summaryPath -Encoding utf8 + throw +} diff --git a/scripts/New-CoverageManifest.ps1 b/scripts/New-CoverageManifest.ps1 new file mode 100644 index 0000000000..c775f97b97 --- /dev/null +++ b/scripts/New-CoverageManifest.ps1 @@ -0,0 +1,26 @@ +param( + [Parameter(Mandatory)] + [string]$Path, + [Parameter(Mandatory)] + [string]$Repository, + [Parameter(Mandatory)] + [string]$SessionId, + [Parameter(Mandatory)] + [ValidateSet('x64', 'arm64')] + [string]$Architecture +) + +$ErrorActionPreference = 'Stop' +$sourceRoot = Split-Path -Parent $PSScriptRoot +$commit = git -C $sourceRoot rev-parse HEAD +if ($LASTEXITCODE -ne 0) { throw 'Cannot identify the compiled source commit.' } + +[ordered]@{ + SchemaVersion = 1 + Repository = $Repository + SourceCommit = $commit + SourceRoot = $sourceRoot + SessionId = $SessionId + Architecture = $Architecture + Configuration = 'Release' +} | ConvertTo-Json | Set-Content -LiteralPath $Path -Encoding utf8 diff --git a/scripts/UnitCoverage.config b/scripts/UnitCoverage.config new file mode 100644 index 0000000000..36af40e4c0 --- /dev/null +++ b/scripts/UnitCoverage.config @@ -0,0 +1,24 @@ + + + True + + + NT AUTHORITY\SYSTEM + + True + False + False + False + True + + + .*[\\/]GVFS[^\\/]*\.(dll|exe)$ + + + + + .*[\\/]GVFS\.(UnitTests|FunctionalTests|Tests)[\\/].* + + + + From ebdda05eb8010377765239c125ca681c5f0f50d8 Mon Sep 17 00:00:00 2001 From: Tyrie Vella Date: Thu, 5 Nov 2026 12:40:15 -0800 Subject: [PATCH 4/5] Accept quoted unit-runner arguments from the collector Strip argument quotes before the batch script compares or assigns configuration and architecture. Preserve the existing defaults and test command. Assisted-by: GPT-6.1 Sol Signed-off-by: Tyrie Vella --- scripts/RunUnitTests.bat | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/RunUnitTests.bat b/scripts/RunUnitTests.bat index dc35bdb2ad..839168ac1d 100644 --- a/scripts/RunUnitTests.bat +++ b/scripts/RunUnitTests.bat @@ -1,8 +1,8 @@ @ECHO OFF CALL %~dp0\InitializeEnvironment.bat || EXIT /b 10 -IF "%1"=="" (SET "CONFIGURATION=Debug") ELSE (SET "CONFIGURATION=%1") -IF "%2"=="" (SET "ARCH=x64") ELSE (SET "ARCH=%2") +IF "%~1"=="" (SET "CONFIGURATION=Debug") ELSE (SET "CONFIGURATION=%~1") +IF "%~2"=="" (SET "ARCH=x64") ELSE (SET "ARCH=%~2") SET RESULT=0 From 3f10523f7c5722fc90c51ea2f8506a3cf93b5b77 Mon Sep 17 00:00:00 2001 From: Tyrie Vella Date: Thu, 5 Nov 2026 13:23:05 -0800 Subject: [PATCH 5/5] Handle real worker metadata in coverage reports Resolve local relative paths without weakening source-path checks. Accept empty optional scenario fields and valid NUnit skipped roots. Compare PR coverage against the tested merge base. Assisted-by: GPT-6.1 Sol Signed-off-by: Tyrie Vella --- .github/workflows/build.yaml | 6 ++++ scripts/Merge-Coverage.Tests.ps1 | 55 ++++++++++++++++++++++++++++++++ scripts/Merge-Coverage.ps1 | 12 ++++--- 3 files changed, 69 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index de47789faa..1686f57bbe 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -413,10 +413,16 @@ jobs: shell: pwsh env: BASE_COMMIT: ${{ github.event.pull_request.base.sha || github.event.before }} + IS_PULL_REQUEST: ${{ github.event_name == 'pull_request' }} run: | $sourceCommit = git -C src rev-parse HEAD if ($LASTEXITCODE -ne 0) { throw 'Cannot identify tested source.' } $base = $env:BASE_COMMIT + if ($env:IS_PULL_REQUEST -eq 'true') { + # The event base can lag the target used by the tested merge. + $base = git -C src rev-parse "$sourceCommit^1" + if ($LASTEXITCODE -ne 0) { throw 'Cannot identify the tested merge base.' } + } if ($base -match '^0+$') { $base = '' } src\scripts\Merge-Coverage.ps1 -InputDirectory coverage-inputs -OutputDirectory coverage-report -SourceDirectory src -SourceCommit $sourceCommit -Repository $env:GITHUB_REPOSITORY -BaseCommit $base Get-Content coverage-report\Summary.md >> $env:GITHUB_STEP_SUMMARY diff --git a/scripts/Merge-Coverage.Tests.ps1 b/scripts/Merge-Coverage.Tests.ps1 index f4ca29cb1c..ace1a4bc57 100644 --- a/scripts/Merge-Coverage.Tests.ps1 +++ b/scripts/Merge-Coverage.Tests.ps1 @@ -146,6 +146,51 @@ Describe 'Managed-source coverage aggregation integrity' { (Get-Content -LiteralPath (Join-Path $outputPath 'summary.json') -Raw) | Should -BeExactly $before } + It 'Resolves relative caller directories from the PowerShell location and creates a nonexistent output directory' { + Push-Location -LiteralPath $root + try { + foreach ($relativeOutput in @('coverage-report', '.\reports\nested\..\coverage-report')) { + Test-Path -LiteralPath $relativeOutput | Should -BeFalse + & $mergeScript -InputDirectory '.\input' -OutputDirectory $relativeOutput -SourceDirectory 'src' ` + -SourceCommit $commit -Repository $repository + $resolvedOutput = $ExecutionContext.SessionState.Path.GetUnresolvedProviderPathFromPSPath($relativeOutput) + $summary = Get-Content -LiteralPath (Join-Path $resolvedOutput 'summary.json') -Raw | ConvertFrom-Json + $summary.Complete | Should -BeTrue + $summary.Workers | Should -Be 32 + $summary.SourceDirectory | Should -Be $sourcePath + @(Get-ChildItem -LiteralPath (Join-Path $resolvedOutput 'normalized') -Filter '*.xml' -Recurse).Count | Should -Be 32 + [xml]$normalized = Get-Content -LiteralPath (Join-Path $resolvedOutput 'normalized\unit-x64\unit-x64.cobertura.xml') -Raw + $normalized.SelectSingleNode('//class').filename | Should -Be (Join-Path $sourcePath 'GVFS\GVFS.Common\Policy.cs') + } + } + finally { + Pop-Location + } + } + + It 'Accepts lifecycle empty scenarios, Int64 shards, and a skipped NUnit root with passed cases' { + foreach ($file in Get-ChildItem -LiteralPath $inputPath -Filter 'metadata.json' -Recurse -File) { + $metadata = Get-Content -LiteralPath $file.FullName -Raw | ConvertFrom-Json -AsHashtable + $metadata.Shard | Should -BeOfType ([long]) + if ($metadata.Suite -ne 'Upgrade') { + $metadata.Scenario = '' + $metadata | ConvertTo-Json | Set-Content -LiteralPath $file.FullName + } + } + $testPath = Join-Path $inputPath 'functional-arm64-shard-0\tests.xml' + (Get-Content -LiteralPath $testPath -Raw).Replace('