Skip to content

Commit b7438b9

Browse files
AriPerkkioAriPerkkio
authored
feat!: --merge-reports to support coverage (#5736)
1 parent a50330e commit b7438b9

File tree

16 files changed

+302
-39
lines changed

16 files changed

+302
-39
lines changed

‎docs/guide/features.md

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -232,10 +232,12 @@ test('my types work properly', () => {
232232
## Sharding
233233

234234
Run tests on different machines using [`--shard`](/guide/cli#shard) and [`--reporter=blob`](/guide/reporters#blob-reporter) flags.
235-
All test results can be merged at the end of your CI pipeline using `--merge-reports` command:
235+
All test and coverage results can be merged at the end of your CI pipeline using `--merge-reports` command:
236236

237237
```bash
238238
vitest --shard=1/2 --reporter=blob
239239
vitest --shard=2/2 --reporter=blob
240-
vitest --merge-reports --reporter=junit
240+
vitest --merge-reports --reporter=junit --coverage.reporter=text
241241
```
242+
243+
See [`Improving Performance | Sharding`](/guide/improving-performance#sharding) for more information.

‎docs/guide/improving-performance.md

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,3 +72,116 @@ export default defineConfig({
7272
})
7373
```
7474
:::
75+
76+
## Sharding
77+
78+
Test sharding means running a small subset of test cases at a time. It can be useful when you have multiple machines that could be used to run tests simultaneously.
79+
80+
To split Vitest tests on multiple different runs, use [`--shard`](/guide/cli#shard) option with [`--reporter=blob`](/guide/reporters#blob-reporter) option:
81+
82+
```sh
83+
vitest run --reporter=blob --shard=1/3 # 1st machine
84+
vitest run --reporter=blob --shard=2/3 # 2nd machine
85+
vitest run --reporter=blob --shard=3/3 # 3rd machine
86+
```
87+
88+
Collect the results stored in `.vitest-reports` directory from each machine and merge them with [`--merge-reports`](/guide/cli#merge-reports) option:
89+
90+
```sh
91+
vitest --merge-reports
92+
```
93+
94+
<details>
95+
<summary>Github action example</summary>
96+
97+
This setup is also used at https://github.com/vitest-tests/test-sharding.
98+
99+
```yaml
100+
# Inspired from https://playwright.dev/docs/test-sharding
101+
name: Tests
102+
on:
103+
push:
104+
branches:
105+
- main
106+
jobs:
107+
tests:
108+
runs-on: ubuntu-latest
109+
strategy:
110+
matrix:
111+
shardIndex: [1, 2, 3, 4]
112+
shardTotal: [4]
113+
steps:
114+
- uses: actions/checkout@v4
115+
- uses: actions/setup-node@v4
116+
with:
117+
node-version: 20
118+
119+
- name: Install pnpm
120+
uses: pnpm/action-setup@v4
121+
122+
- name: Install dependencies
123+
run: pnpm i
124+
125+
- name: Run tests
126+
run: pnpm run test --reporter=blob --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
127+
128+
- name: Upload blob report to GitHub Actions Artifacts
129+
if: ${{ !cancelled() }}
130+
uses: actions/upload-artifact@v4
131+
with:
132+
name: blob-report-${{ matrix.shardIndex }}
133+
path: .vitest-reports/*
134+
retention-days: 1
135+
136+
merge-reports:
137+
if: ${{ !cancelled() }}
138+
needs: [tests]
139+
140+
runs-on: ubuntu-latest
141+
steps:
142+
- uses: actions/checkout@v4
143+
- uses: actions/setup-node@v4
144+
with:
145+
node-version: 20
146+
147+
- name: Install pnpm
148+
uses: pnpm/action-setup@v4
149+
150+
- name: Install dependencies
151+
run: pnpm i
152+
153+
- name: Download blob reports from GitHub Actions Artifacts
154+
uses: actions/download-artifact@v4
155+
with:
156+
path: .vitest-reports
157+
pattern: blob-report-*
158+
merge-multiple: true
159+
160+
- name: Merge reports
161+
run: npx vitest --merge-reports
162+
```
163+
164+
</details>
165+
166+
:::tip
167+
Test sharding can also become useful on high CPU-count machines.
168+
169+
Vitest will run only a single Vite server in its main thread. Rest of the threads are used to run test files.
170+
In a high CPU-count machine the main thread can become a bottleneck as it cannot handle all the requests coming from the threads. For example in 32 CPU machine the main thread is responsible to handle load coming from 31 test threads.
171+
172+
To reduce the load from main thread's Vite server you can use test sharding. The load can be balanced on multiple Vite server.
173+
174+
```sh
175+
# Example for splitting tests on 32 CPU to 4 shards.
176+
# As each process needs 1 main thread, there's 7 threads for test runners (1+7)*4 = 32
177+
# Use VITEST_MAX_THREADS or VITEST_MAX_FORKS depending on the pool:
178+
VITEST_MAX_THREADS=7 vitest run --reporter=blob --shard=1/4 & \
179+
VITEST_MAX_THREADS=7 vitest run --reporter=blob --shard=2/4 & \
180+
VITEST_MAX_THREADS=7 vitest run --reporter=blob --shard=3/4 & \
181+
VITEST_MAX_THREADS=7 vitest run --reporter=blob --shard=4/4 & \
182+
wait # https://man7.org/linux/man-pages/man2/waitpid.2.html
183+
184+
vitest --merge-reports
185+
```
186+
187+
:::

‎packages/coverage-istanbul/src/provider.ts

Lines changed: 31 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -162,7 +162,7 @@ export class IstanbulCoverageProvider extends BaseCoverageProvider implements Co
162162
this.pendingPromises = []
163163
}
164164

165-
async reportCoverage({ allTestsRun }: ReportContext = {}) {
165+
async generateCoverage({ allTestsRun }: ReportContext) {
166166
const coverageMap = libCoverage.createCoverageMap({})
167167
let index = 0
168168
const total = this.pendingPromises.length
@@ -202,6 +202,29 @@ export class IstanbulCoverageProvider extends BaseCoverageProvider implements Co
202202
coverageMap.merge(await transformCoverage(uncoveredCoverage))
203203
}
204204

205+
return coverageMap
206+
}
207+
208+
async reportCoverage(coverageMap: unknown, { allTestsRun }: ReportContext) {
209+
await this.generateReports(
210+
coverageMap as CoverageMap || libCoverage.createCoverageMap({}),
211+
allTestsRun,
212+
)
213+
214+
// In watch mode we need to preserve the previous results if cleanOnRerun is disabled
215+
const keepResults = !this.options.cleanOnRerun && this.ctx.config.watch
216+
217+
if (!keepResults) {
218+
this.coverageFiles = new Map()
219+
await fs.rm(this.coverageFilesDirectory, { recursive: true })
220+
221+
// Remove empty reports directory, e.g. when only text-reporter is used
222+
if (readdirSync(this.options.reportsDirectory).length === 0)
223+
await fs.rm(this.options.reportsDirectory, { recursive: true })
224+
}
225+
}
226+
227+
async generateReports(coverageMap: CoverageMap, allTestsRun: boolean | undefined) {
205228
const context = libReport.createContext({
206229
dir: this.options.reportsDirectory,
207230
coverageMap,
@@ -248,21 +271,18 @@ export class IstanbulCoverageProvider extends BaseCoverageProvider implements Co
248271
})
249272
}
250273
}
274+
}
251275

252-
// In watch mode we need to preserve the previous results if cleanOnRerun is disabled
253-
const keepResults = !this.options.cleanOnRerun && this.ctx.config.watch
276+
async mergeReports(coverageMaps: unknown[]) {
277+
const coverageMap = libCoverage.createCoverageMap({})
254278

255-
if (!keepResults) {
256-
this.coverageFiles = new Map()
257-
await fs.rm(this.coverageFilesDirectory, { recursive: true })
279+
for (const coverage of coverageMaps)
280+
coverageMap.merge(coverage as CoverageMap)
258281

259-
// Remove empty reports directory, e.g. when only text-reporter is used
260-
if (readdirSync(this.options.reportsDirectory).length === 0)
261-
await fs.rm(this.options.reportsDirectory, { recursive: true })
262-
}
282+
await this.generateReports(coverageMap, true)
263283
}
264284

265-
async getCoverageMapForUncoveredFiles(coveredFiles: string[]) {
285+
private async getCoverageMapForUncoveredFiles(coveredFiles: string[]) {
266286
const allFiles = await this.testExclude.glob(this.ctx.config.root)
267287
let includedFiles = allFiles.map(file => resolve(this.ctx.config.root, file))
268288

‎packages/coverage-v8/src/provider.ts

Lines changed: 33 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -148,10 +148,7 @@ export class V8CoverageProvider extends BaseCoverageProvider implements Coverage
148148
this.pendingPromises.push(promise)
149149
}
150150

151-
async reportCoverage({ allTestsRun }: ReportContext = {}) {
152-
if (provider === 'stackblitz')
153-
this.ctx.logger.log(c.blue(' % ') + c.yellow('@vitest/coverage-v8 does not work on Stackblitz. Report will be empty.'))
154-
151+
async generateCoverage({ allTestsRun }: ReportContext) {
155152
const coverageMap = libCoverage.createCoverageMap({})
156153
let index = 0
157154
const total = this.pendingPromises.length
@@ -193,6 +190,32 @@ export class V8CoverageProvider extends BaseCoverageProvider implements Coverage
193190
coverageMap.merge(await transformCoverage(converted))
194191
}
195192

193+
return coverageMap
194+
}
195+
196+
async reportCoverage(coverageMap: unknown, { allTestsRun }: ReportContext) {
197+
if (provider === 'stackblitz')
198+
this.ctx.logger.log(c.blue(' % ') + c.yellow('@vitest/coverage-v8 does not work on Stackblitz. Report will be empty.'))
199+
200+
await this.generateReports(
201+
coverageMap as CoverageMap || libCoverage.createCoverageMap({}),
202+
allTestsRun,
203+
)
204+
205+
// In watch mode we need to preserve the previous results if cleanOnRerun is disabled
206+
const keepResults = !this.options.cleanOnRerun && this.ctx.config.watch
207+
208+
if (!keepResults) {
209+
this.coverageFiles = new Map()
210+
await fs.rm(this.coverageFilesDirectory, { recursive: true })
211+
212+
// Remove empty reports directory, e.g. when only text-reporter is used
213+
if (readdirSync(this.options.reportsDirectory).length === 0)
214+
await fs.rm(this.options.reportsDirectory, { recursive: true })
215+
}
216+
}
217+
218+
async generateReports(coverageMap: CoverageMap, allTestsRun?: boolean) {
196219
const context = libReport.createContext({
197220
dir: this.options.reportsDirectory,
198221
coverageMap,
@@ -239,18 +262,15 @@ export class V8CoverageProvider extends BaseCoverageProvider implements Coverage
239262
})
240263
}
241264
}
265+
}
242266

243-
// In watch mode we need to preserve the previous results if cleanOnRerun is disabled
244-
const keepResults = !this.options.cleanOnRerun && this.ctx.config.watch
267+
async mergeReports(coverageMaps: unknown[]) {
268+
const coverageMap = libCoverage.createCoverageMap({})
245269

246-
if (!keepResults) {
247-
this.coverageFiles = new Map()
248-
await fs.rm(this.coverageFilesDirectory, { recursive: true })
270+
for (const coverage of coverageMaps)
271+
coverageMap.merge(coverage as CoverageMap)
249272

250-
// Remove empty reports directory, e.g. when only text-reporter is used
251-
if (readdirSync(this.options.reportsDirectory).length === 0)
252-
await fs.rm(this.options.reportsDirectory, { recursive: true })
253-
}
273+
await this.generateReports(coverageMap, true)
254274
}
255275

256276
private async getUntestedFiles(testedFiles: string[]): Promise<RawCoverage> {

‎packages/vitest/src/node/core.ts

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -393,7 +393,7 @@ export class Vitest {
393393
if (this.reporters.some(r => r instanceof BlobReporter))
394394
throw new Error('Cannot merge reports when `--reporter=blob` is used. Remove blob reporter from the config first.')
395395

396-
const { files, errors } = await readBlobs(this.config.mergeReports, this.projects)
396+
const { files, errors, coverages } = await readBlobs(this.config.mergeReports, this.projects)
397397

398398
await this.report('onInit', this)
399399
await this.report('onPathsCollected', files.flatMap(f => f.filepath))
@@ -439,6 +439,8 @@ export class Vitest {
439439
process.exitCode = 1
440440

441441
await this.report('onFinished', files, errors)
442+
await this.initCoverageProvider()
443+
await this.coverageProvider?.mergeReports?.(coverages)
442444
}
443445

444446
async start(filters?: string[]) {
@@ -459,7 +461,9 @@ export class Vitest {
459461

460462
// if run with --changed, don't exit if no tests are found
461463
if (!files.length) {
462-
await this.reportCoverage(true)
464+
// Report coverage for uncovered files
465+
const coverage = await this.coverageProvider?.generateCoverage?.({ allTestsRun: true })
466+
await this.reportCoverage(coverage, true)
463467

464468
this.logger.printNoTestFound(filters)
465469

@@ -645,8 +649,10 @@ export class Vitest {
645649
.finally(async () => {
646650
// can be duplicate files if different projects are using the same file
647651
const files = Array.from(new Set(specs.map(([, p]) => p)))
648-
await this.report('onFinished', this.state.getFiles(files), this.state.getUnhandledErrors())
649-
await this.reportCoverage(allTestsRun)
652+
const coverage = await this.coverageProvider?.generateCoverage({ allTestsRun })
653+
654+
await this.report('onFinished', this.state.getFiles(files), this.state.getUnhandledErrors(), coverage)
655+
await this.reportCoverage(coverage, allTestsRun)
650656

651657
this.runningPromise = undefined
652658
this.isFirstRun = false
@@ -946,12 +952,12 @@ export class Vitest {
946952
return Array.from(new Set(files))
947953
}
948954

949-
private async reportCoverage(allTestsRun: boolean) {
955+
private async reportCoverage(coverage: unknown, allTestsRun: boolean) {
950956
if (!this.config.coverage.reportOnFailure && this.state.getCountOfFailedTests() > 0)
951957
return
952958

953959
if (this.coverageProvider) {
954-
await this.coverageProvider.reportCoverage({ allTestsRun })
960+
await this.coverageProvider.reportCoverage(coverage, { allTestsRun })
955961
// notify coverage iframe reload
956962
for (const reporter of this.reporters) {
957963
if (reporter instanceof WebSocketReporter)

‎packages/vitest/src/node/reporters/blob.ts

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ export class BlobReporter implements Reporter {
2626
this.ctx = ctx
2727
}
2828

29-
async onFinished(files: File[] = [], errors: unknown[] = []) {
29+
async onFinished(files: File[] = [], errors: unknown[] = [], coverage: unknown) {
3030
let outputFile = this.options.outputFile ?? getOutputFile(this.ctx.config, 'blob')
3131
if (!outputFile) {
3232
const shard = this.ctx.config.shard
@@ -39,7 +39,7 @@ export class BlobReporter implements Reporter {
3939
return [project.getName(), [...project.server.moduleGraph.idToModuleMap.keys()]]
4040
})
4141

42-
const report = stringify([this.ctx.version, files, errors, moduleKeys] satisfies MergeReport)
42+
const report = stringify([this.ctx.version, files, errors, moduleKeys, coverage] satisfies MergeReport)
4343

4444
const reportFile = resolve(this.ctx.config.root, outputFile)
4545

@@ -62,8 +62,8 @@ export async function readBlobs(blobsDirectory: string, projectsArray: Workspace
6262
const blobsFiles = await readdir(resolvedDir)
6363
const promises = blobsFiles.map(async (file) => {
6464
const content = await readFile(resolve(resolvedDir, file), 'utf-8')
65-
const [version, files, errors, moduleKeys] = parse(content) as MergeReport
66-
return { version, files, errors, moduleKeys }
65+
const [version, files, errors, moduleKeys, coverage] = parse(content) as MergeReport
66+
return { version, files, errors, moduleKeys, coverage }
6767
})
6868
const blobs = await Promise.all(promises)
6969

@@ -108,10 +108,12 @@ export async function readBlobs(blobsDirectory: string, projectsArray: Workspace
108108
return time1 - time2
109109
})
110110
const errors = blobs.flatMap(blob => blob.errors)
111+
const coverages = blobs.map(blob => blob.coverage)
111112

112113
return {
113114
files,
114115
errors,
116+
coverages,
115117
}
116118
}
117119

@@ -120,6 +122,7 @@ type MergeReport = [
120122
files: File[],
121123
errors: unknown[],
122124
moduleKeys: MergeReportModuleKeys[],
125+
coverage: unknown,
123126
]
124127

125128
type MergeReportModuleKeys = [projectName: string, moduleIds: string[]]

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy