From 15f4e3a03977823de7ee1dc0df085ae354cf0137 Mon Sep 17 00:00:00 2001 From: "David A. Ramos" Date: Tue, 4 Jul 2023 13:19:36 -0700 Subject: [PATCH 01/53] [jest] Support Jest 29.6 --- .github/workflows/ci.yaml | 1 + scripts/set-jest-version.ts | 9 ++++++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 30afa6a..0105775 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -306,6 +306,7 @@ jobs: - 18 - 20 jest: + - "29.6" - "29.5" - "29.4" - "29.3" diff --git a/scripts/set-jest-version.ts b/scripts/set-jest-version.ts index 5b3e58a..56a614d 100644 --- a/scripts/set-jest-version.ts +++ b/scripts/set-jest-version.ts @@ -95,6 +95,7 @@ const PACKAGE_VERSION_MAP = { "27.2 - 27.3": "~27.2", "29.0 - 29.1": "~29.0", "29.2 - 29.3": "~29.2", + "29.5 - 29.6": "~29.5", }, "babel-preset-jest": { "26.3 - 26.4": "~26.3", @@ -102,11 +103,13 @@ const PACKAGE_VERSION_MAP = { "27.2 - 27.3": "~27.2", "29.0 - 29.1": "~29.0", "29.2 - 29.3": "~29.2", + "29.5 - 29.6": "~29.5", }, "jest-changed-files": { "26.3 - 26.4": "~26.3", "29.0 - 29.1": "~29.0", "29.2 - 29.3": "~29.2", + "29.5 - 29.6": "~29.5", }, "jest-docblock": { "^25.3": "~25.3", @@ -114,7 +117,7 @@ const PACKAGE_VERSION_MAP = { "27.0 - 27.3": "~27.0", "29.0 - 29.1": "~29.0", "29.2 - 29.3": "~29.2", - "29.4 - 29.5": "~29.4", + "29.4 - 29.6": "~29.4", }, "jest-environment-jsdom": { "26.3 - 26.4": "~26.3", @@ -130,7 +133,7 @@ const PACKAGE_VERSION_MAP = { "^28": "~28.0", "29.0 - 29.1": "~29.0", "29.2 - 29.3": "~29.2", - "29.4 - 29.5": "~29.4", + "29.4 - 29.6": "~29.4", }, "jest-haste-map": { "26.3 - 26.4": "~26.3", @@ -148,7 +151,7 @@ const PACKAGE_VERSION_MAP = { "^28": "~28.0", "29.0 - 29.1": "~29.0", "29.2 - 29.3": "~29.2", - "29.4 - 29.5": "~29.4", + "29.4 - 29.6": "~29.4", }, "jest-serializer": { "25.2 - 25.4": "~25.2", From 128d573149b0f6a94176050b1f38d67859e38ca7 Mon Sep 17 00:00:00 2001 From: "David A. Ramos" Date: Fri, 7 Jul 2023 15:36:15 -0700 Subject: [PATCH 02/53] [cypress] Support Cypress 12.17 --- .github/workflows/ci.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 0105775..f4d7def 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -114,6 +114,7 @@ jobs: - "12.14" - "12.15" - "12.16" + - "12.17" steps: - uses: actions/checkout@v3 @@ -217,6 +218,7 @@ jobs: - "12.14" - "12.15" - "12.16" + - "12.17" steps: - uses: actions/checkout@v3 From b4707d103893dd20cd993be66f6945971834714a Mon Sep 17 00:00:00 2001 From: "David A. Ramos" Date: Wed, 5 Jul 2023 02:10:13 -0700 Subject: [PATCH 03/53] [jest] Bump @unflakable/jest-plugin to 0.3.0 --- packages/jest-plugin/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/jest-plugin/package.json b/packages/jest-plugin/package.json index 5e00edc..98f312d 100644 --- a/packages/jest-plugin/package.json +++ b/packages/jest-plugin/package.json @@ -8,7 +8,7 @@ "bugs": "https://github.com/unflakable/unflakable-javascript/issues", "homepage": "https://unflakable.com", "license": "MIT", - "version": "0.2.0", + "version": "0.3.0", "exports": { "./dist/reporter": { "types": "./dist/reporter.d.ts", From 7e4a40f725d1cbe37015c85e2b7e19f7e62f589e Mon Sep 17 00:00:00 2001 From: "David A. Ramos" Date: Wed, 5 Jul 2023 02:07:22 -0700 Subject: [PATCH 04/53] [jest] Make file paths relative to git or Jest root dir Previously, file paths were relative to the current working directory. This changes the behavior to make paths relative to the git repo (if git auto-detect is enabled) or the Jest rootDir otherwise, which matches the behavior of the Cypress plugin. --- packages/cypress-plugin/src/plugin.ts | 21 ++- packages/jest-plugin/src/reporter.ts | 15 +- packages/jest-plugin/src/runner.ts | 50 ++++-- .../test/integration/src/common.ts | 5 + .../test/integration/src/git.test.ts | 6 + .../test/integration/src/runTestCase.ts | 160 ++++++++++-------- packages/plugins-common/src/git.ts | 16 +- 7 files changed, 162 insertions(+), 111 deletions(-) diff --git a/packages/cypress-plugin/src/plugin.ts b/packages/cypress-plugin/src/plugin.ts index ad9b93a..a28057e 100644 --- a/packages/cypress-plugin/src/plugin.ts +++ b/packages/cypress-plugin/src/plugin.ts @@ -15,6 +15,7 @@ import { branchOverride, commitOverride, isTestQuarantined, + loadGitRepo, normalizeTestName, toPosix, UnflakableConfig, @@ -477,15 +478,19 @@ ${ commit === undefined || commit.length === 0) ) { - const { branch: gitBranch, commit: gitCommit } = await autoDetectGit( - console.error.bind(console) - ); + const git = await loadGitRepo(); + if (git !== null) { + const { branch: gitBranch, commit: gitCommit } = await autoDetectGit( + git, + console.error.bind(console) + ); - if (branch === undefined || branch.length === 0) { - branch = gitBranch; - } - if (commit === undefined || commit.length === 0) { - commit = gitCommit; + if (branch === undefined || branch.length === 0) { + branch = gitBranch; + } + if (commit === undefined || commit.length === 0) { + commit = gitCommit; + } } } diff --git a/packages/jest-plugin/src/reporter.ts b/packages/jest-plugin/src/reporter.ts index bff6a34..d3e47cd 100644 --- a/packages/jest-plugin/src/reporter.ts +++ b/packages/jest-plugin/src/reporter.ts @@ -38,8 +38,10 @@ import { autoDetectGit, branchOverride, commitOverride, + getRepoRoot, loadApiKey, loadConfigSync, + loadGitRepo, UnflakableConfig, } from "@unflakable/plugins-common"; @@ -209,7 +211,7 @@ export default class UnflakableReporter extends BaseReporter { private readonly apiKey: string; private readonly unflakableConfig: UnflakableConfig; - private readonly cwd: string; + private readonly rootDir: string; private readonly defaultReporter: DefaultReporter & { // Not defined in Jest < 26.2. onTestCaseResult?: (test: Test, testCaseResult: AssertionResult) => void; @@ -218,7 +220,7 @@ export default class UnflakableReporter extends BaseReporter { constructor(globalConfig: Config.GlobalConfig) { super(); - this.cwd = process.cwd(); + this.rootDir = globalConfig.rootDir; this.unflakableConfig = loadConfigSync(globalConfig.rootDir); this.apiKey = this.unflakableConfig.enabled ? loadApiKey() : ""; @@ -412,6 +414,10 @@ export default class UnflakableReporter extends BaseReporter { unflakableConfig: UnflakableConfig ): Promise { const testSuiteId = unflakableConfig.testSuiteId; + + const git = unflakableConfig.gitAutoDetect ? await loadGitRepo() : null; + const repoRoot = git !== null ? await getRepoRoot(git) : this.rootDir; + const results = Object.entries( groupBy( aggregatedResults.testResults, @@ -428,7 +434,7 @@ export default class UnflakableReporter extends BaseReporter { ) .map( ([, assertionResults]): TestRunRecord => ({ - filename: path.relative(this.cwd, testFilePath), + filename: path.relative(repoRoot, testFilePath), name: testKey(assertionResults[0]), attempts: assertionResults .map((testResult: UnflakableAssertionResult) => ({ @@ -462,13 +468,14 @@ export default class UnflakableReporter extends BaseReporter { commit = commitOverride.value; if ( - unflakableConfig.gitAutoDetect && + git !== null && (branch === undefined || branch.length === 0 || commit === undefined || commit.length === 0) ) { const { branch: gitBranch, commit: gitCommit } = await autoDetectGit( + git, this.log.bind(this) ); diff --git a/packages/jest-plugin/src/runner.ts b/packages/jest-plugin/src/runner.ts index ecc7abd..6940666 100644 --- a/packages/jest-plugin/src/runner.ts +++ b/packages/jest-plugin/src/runner.ts @@ -26,10 +26,12 @@ import chalk from "chalk"; import escapeStringRegexp from "escape-string-regexp"; import { debug as _debug } from "debug"; import { + getRepoRoot, getTestSuiteManifest, isTestQuarantined, loadApiKey, loadConfigSync, + loadGitRepo, QuarantineMode, UnflakableConfig, } from "@unflakable/plugins-common"; @@ -41,23 +43,23 @@ type TestFailure = { test: Test; testResult: TestResult }; const wrapOnResult = ({ attempt, - cwd, manifest, onResult, quarantineMode, + repoRoot, testFailures, }: { attempt: number; - cwd: string; manifest: TestSuiteManifest | undefined; onResult: OnTestSuccess; quarantineMode: QuarantineMode; + repoRoot: string; testFailures: TestFailure[]; }) => async (test: Test, testResult: TestResult): Promise => { const testResults = testResult.testResults.map( (assertionResult: AssertionResult): UnflakableAssertionResult => { - const testFilename = path.relative(cwd, test.path); + const testFilename = path.relative(repoRoot, test.path); if (assertionResult.status === FAILED) { if (manifest === undefined) { debug( @@ -67,19 +69,24 @@ const wrapOnResult = debug( "Not quarantining test failure because quarantineMode is set to `no_quarantine`" ); - } else if ( - isTestQuarantined(manifest, testFilename, testKey(assertionResult)) - ) { + } else { + const isQuarantined = isTestQuarantined( + manifest, + testFilename, + testKey(assertionResult) + ); debug( - `Quarantining failed test ${JSON.stringify( + `Test is ${ + isQuarantined ? "" : "NOT " + }quarantined: ${JSON.stringify( testKey(assertionResult) - )} from file ${testFilename}` + )} in file ${testFilename}` ); return { ...assertionResult, // Use a separate field instead of adding a new `status` to avoid confusing third- // party code that consumes the `Status` enum. - _unflakableIsQuarantined: true, + _unflakableIsQuarantined: isQuarantined, }; } } @@ -127,13 +134,11 @@ const wrapOnResult = class UnflakableRunner { private readonly context?: TestRunnerContext; - private readonly cwd: string; private readonly globalConfig: Config.GlobalConfig; private readonly manifest: Promise; private readonly unflakableConfig: UnflakableConfig; constructor(globalConfig: Config.GlobalConfig, context?: TestRunnerContext) { - this.cwd = process.cwd(); this.unflakableConfig = loadConfigSync(globalConfig.rootDir); const testSuiteId = this.unflakableConfig.enabled @@ -166,6 +171,16 @@ class UnflakableRunner { onFailure: OnTestFailure, options: TestRunnerOptions ): Promise { + const repoRoot = + this.unflakableConfig.enabled && this.unflakableConfig.gitAutoDetect + ? await (async (): Promise => { + const git = await loadGitRepo(); + return git !== null + ? await getRepoRoot(git) + : this.globalConfig.rootDir; + })() + : this.globalConfig.rootDir; + let testFailures = await this.runTestsImpl( tests, watcher, @@ -173,7 +188,7 @@ class UnflakableRunner { onResult, onFailure, options, - this.cwd, + repoRoot, this.globalConfig, this.context, this.unflakableConfig, @@ -245,7 +260,7 @@ class UnflakableRunner { onResult, onFailure, options, - this.cwd, + repoRoot, filteredGlobalConfig, this.context, this.unflakableConfig, @@ -266,7 +281,7 @@ class UnflakableRunner { onResult: OnTestSuccess, onFailure: OnTestFailure, options: TestRunnerOptions, - cwd: string, + repoRoot: string, globalConfig: Config.GlobalConfig, context: TestRunnerContext | undefined, unflakableConfig: UnflakableConfig, @@ -278,10 +293,10 @@ class UnflakableRunner { const onResultImpl = this.unflakableConfig.enabled ? wrapOnResult({ attempt, - cwd, manifest, onResult, quarantineMode: unflakableConfig.quarantineMode, + repoRoot, testFailures, }) : onResult; @@ -396,7 +411,7 @@ class UnflakableRunner { // Then, we run the remaining test files normally below. await tests .reduce((promise, test) => { - const relPath = path.relative(cwd, test.path); + const relPath = path.relative(repoRoot, test.path); const quarantinedTestsInFile = quarantinedTestsByFile[relPath]; if ( quarantinedTestsInFile !== undefined && @@ -446,7 +461,8 @@ class UnflakableRunner { }, Promise.resolve()) .then(() => { const normalTestFiles = tests.filter( - (test) => !(path.relative(cwd, test.path) in quarantinedTestsByFile) + (test) => + !(path.relative(repoRoot, test.path) in quarantinedTestsByFile) ); if (normalTestFiles.length > 0) { debug( diff --git a/packages/jest-plugin/test/integration/src/common.ts b/packages/jest-plugin/test/integration/src/common.ts index 5f6cf8c..61fdc51 100644 --- a/packages/jest-plugin/test/integration/src/common.ts +++ b/packages/jest-plugin/test/integration/src/common.ts @@ -6,6 +6,7 @@ import * as cosmiconfig from "cosmiconfig"; import type { OptionsSync } from "cosmiconfig"; import { FetchMockSandbox, MockCall } from "fetch-mock"; import jestPackage from "jest/package.json"; +import path from "path"; const throwUnimplemented = (): never => { throw new Error("unimplemented"); @@ -82,6 +83,7 @@ export const integrationTest = async (testCase: TestCase): Promise => { expectedCommit: "MOCK_COMMIT", expectedFailureRetries: 2, expectedFlakeTestNameSuffix: "", + expectedRepoRelativePathPrefix: "test/integration-input/", expectedSuiteId: "MOCK_SUITE_ID", expectPluginToBeEnabled: true, expectResultsToBeUploaded: true, @@ -98,6 +100,9 @@ export const integrationTest = async (testCase: TestCase): Promise => { refs: [{ sha: "MOCK_COMMIT", refName: "refs/heads/MOCK_BRANCH" }], commit: "MOCK_COMMIT", isRepo: true, + // Mock the git repo root as packages/jest-plugin so that we're for sure testing the + // mocked output and not using real git commands. + repoRoot: path.resolve("../.."), }, quarantineFlake: false, skipFailures: false, diff --git a/packages/jest-plugin/test/integration/src/git.test.ts b/packages/jest-plugin/test/integration/src/git.test.ts index f6786d1..2fbd6de 100644 --- a/packages/jest-plugin/test/integration/src/git.test.ts +++ b/packages/jest-plugin/test/integration/src/git.test.ts @@ -5,6 +5,7 @@ import { integrationTest, integrationTestSuite, } from "./common"; +import path from "path"; integrationTestSuite(() => { it("no git repo", () => @@ -12,6 +13,8 @@ integrationTestSuite(() => { params: { expectedBranch: undefined, expectedCommit: undefined, + // Without a repo, paths are relative to the Jest rootDir. + expectedRepoRelativePathPrefix: "", git: { isRepo: false, }, @@ -40,6 +43,7 @@ integrationTestSuite(() => { refName: "refs/remote/pull/MOCK_PR_NUMBER/merge", }, ], + repoRoot: path.resolve("../.."), }, expectedCommit: "MOCK_PR_COMMIT", expectedBranch: "pull/MOCK_PR_NUMBER/merge", @@ -70,6 +74,8 @@ integrationTestSuite(() => { }, expectedBranch: undefined, expectedCommit: undefined, + // Without a repo, paths are relative to the Jest rootDir. + expectedRepoRelativePathPrefix: "", }, expectedExitCode: 1, expectedResults: defaultExpectedResults, diff --git a/packages/jest-plugin/test/integration/src/runTestCase.ts b/packages/jest-plugin/test/integration/src/runTestCase.ts index 6d5afd6..e8e7424 100644 --- a/packages/jest-plugin/test/integration/src/runTestCase.ts +++ b/packages/jest-plugin/test/integration/src/runTestCase.ts @@ -27,6 +27,9 @@ import * as cosmiconfig from "cosmiconfig"; import { CosmiconfigResult } from "cosmiconfig/dist/types"; import { gunzipSync } from "zlib"; import { UnflakableConfig } from "@unflakable/plugins-common"; +import _debug from "debug"; + +const debug = _debug("unflakable:integration:run-test-case"); const userAgentRegex = new RegExp( "unflakable-js-api/(?:[-0-9.]|alpha|beta)+ unflakable-jest-plugin/(?:[-0-9.]|alpha|beta)+ \\(Jest [0-9]+\\.[0-9]+\\.[0-9]+; Node v[0-9]+\\.[0-9]+\\.[0-9]\\)" @@ -50,6 +53,7 @@ export type SimpleGitMockParams = commit: string; isRepo: true; refs: SimpleGitMockRef[]; + repoRoot: string; }; export type TestCaseParams = { @@ -61,6 +65,7 @@ export type TestCaseParams = { expectedFailureRetries: number; expectedFlakeTestNameSuffix: string; expectedSuiteId: string; + expectedRepoRelativePathPrefix: string; expectPluginToBeEnabled: boolean; expectResultsToBeUploaded: boolean; expectQuarantinedTestsToBeQuarantined: boolean; @@ -79,44 +84,52 @@ export type TestCaseParams = { const originalStderrWrite = process.stderr.write.bind(process.stderr); const mockSimpleGit = (params: SimpleGitMockParams): void => { - (simpleGit as jest.Mock).mockImplementationOnce( - () => - ({ - checkIsRepo: jest.fn( - () => Promise.resolve(params.isRepo) as GitResponse - ), - revparse: jest.fn((options: string | TaskOptions) => { - if (!params.isRepo) { - throw new Error("not a git repository"); - } else if ( - Array.isArray(options) && - options.length === 2 && - options[0] === "--abbrev-ref" - ) { - return Promise.resolve( - params.abbreviatedRefs[options[1]] ?? "HEAD" - ) as GitResponse; - } else if (options === "HEAD") { - return Promise.resolve(params.commit) as GitResponse; - } else { - throw new Error(`unexpected options ${options.toString()}`); - } - }), - raw: jest.fn((options: string | TaskOptions) => { - if (!params.isRepo) { - throw new Error("not a git repository"); - } else if (deepEqual(options, ["show-ref"])) { - return Promise.resolve( - (params.refs ?? []) - .map((mockRef) => `${mockRef.sha} ${mockRef.refName}`) - .join("\n") + "\n" - ) as GitResponse; - } else { - throw new Error(`unexpected options ${options.toString()}`); - } - }), - } as unknown as SimpleGit) - ); + debug("Mocking simple-git with params %o", params); + const mockSimpleGitImpl = { + checkIsRepo: jest.fn( + () => Promise.resolve(params.isRepo) as GitResponse + ), + revparse: jest.fn((options: string | TaskOptions) => { + if (!params.isRepo) { + throw new Error("not a git repository"); + } else if ( + Array.isArray(options) && + options.length === 2 && + options[0] === "--abbrev-ref" + ) { + return Promise.resolve( + params.abbreviatedRefs[options[1]] ?? "HEAD" + ) as GitResponse; + } else if ( + Array.isArray(options) && + options.length === 1 && + options[0] === "--show-toplevel" + ) { + // Treat the current working directory as the repo root. + debug(`Returning mock git repo root ${params.repoRoot}`); + return Promise.resolve(params.repoRoot); + } else if (options === "HEAD") { + return Promise.resolve(params.commit) as GitResponse; + } else { + throw new Error(`unexpected options ${options.toString()}`); + } + }), + raw: jest.fn((options: string | TaskOptions) => { + if (!params.isRepo) { + throw new Error("not a git repository"); + } else if (deepEqual(options, ["show-ref"])) { + return Promise.resolve( + (params.refs ?? []) + .map((mockRef) => `${mockRef.sha} ${mockRef.refName}`) + .join("\n") + "\n" + ) as GitResponse; + } else { + throw new Error(`unexpected options ${options.toString()}`); + } + }), + } as unknown as SimpleGit; + + (simpleGit as jest.Mock).mockImplementation(() => mockSimpleGitImpl); }; // These are the chalk-formatted strings that include console color codes. @@ -160,6 +173,9 @@ const testResultRegexMatch = ( "" ); +const specRepoPath = (params: TestCaseParams, specNameStub: string): string => + `${params.expectedRepoRelativePathPrefix}src/${specNameStub}.test.ts`; + export type ResultCounts = { passedSuites: number; passedTests: number; @@ -469,25 +485,25 @@ const countResults = ({ })); }; -const uploadResultsMatcher = - ( - { - expectedBranch, - expectedCommit, - expectedFailureRetries, - expectedFlakeTestNameSuffix, - expectQuarantinedTestsToBeQuarantined, - expectQuarantinedTestsToBeSkipped, - failToFetchManifest, - quarantineFlake, - skipFailures, - skipFlake, - skipQuarantined, - testNamePattern, - }: TestCaseParams, - results: ResultCounts - ): MockMatcher => - (_url, { body, headers }) => { +const uploadResultsMatcher = ( + params: TestCaseParams, + results: ResultCounts +): MockMatcher => { + const { + expectedBranch, + expectedCommit, + expectedFailureRetries, + expectedFlakeTestNameSuffix, + expectQuarantinedTestsToBeQuarantined, + expectQuarantinedTestsToBeSkipped, + failToFetchManifest, + quarantineFlake, + skipFailures, + skipFlake, + skipQuarantined, + testNamePattern, + } = params; + return (_url, { body, headers }) => { const parsedBody = JSON.parse( gunzipSync(body as string).toString() ) as CreateTestSuiteRunInlineRequest; @@ -534,7 +550,7 @@ const uploadResultsMatcher = ? ([] as TestRunRecord[]) : ([ { - filename: "../integration-input/src/fail.test.ts", + filename: specRepoPath(params, "fail"), name: ["describe block", "should ([escape regex]?.*$ fail"], attempts: Array( expectedFailureRetries + 1 @@ -555,7 +571,7 @@ const uploadResultsMatcher = ? [] : [ { - filename: "../integration-input/src/flake.test.ts", + filename: specRepoPath(params, "flake"), name: [ `should be flaky 1${expectedFlakeTestNameSuffix}`.substring( 0, @@ -594,7 +610,7 @@ const uploadResultsMatcher = ? [] : [ { - filename: "../integration-input/src/flake.test.ts", + filename: specRepoPath(params, "flake"), name: [ `should be flaky 2${expectedFlakeTestNameSuffix}`.substring( 0, @@ -630,7 +646,7 @@ const uploadResultsMatcher = ? [] : ([ { - filename: "../integration-input/src/mixed.test.ts", + filename: specRepoPath(params, "mixed"), name: ["mixed", "mixed: should be quarantined"], attempts: Array( expectedFailureRetries + 1 @@ -650,7 +666,7 @@ const uploadResultsMatcher = ? [] : ([ { - filename: "../integration-input/src/mixed.test.ts", + filename: specRepoPath(params, "mixed"), name: ["mixed", "mixed: should fail"], attempts: Array( expectedFailureRetries + 1 @@ -664,7 +680,7 @@ const uploadResultsMatcher = "mixed mixed: should pass".match(testNamePattern) === null ? [ { - filename: "../integration-input/src/mixed.test.ts", + filename: specRepoPath(params, "mixed"), name: ["mixed", "mixed: should pass"], attempts: [ { @@ -679,7 +695,7 @@ const uploadResultsMatcher = "should pass".match(testNamePattern) !== null ? [ { - filename: "../integration-input/src/pass.test.ts", + filename: specRepoPath(params, "pass"), name: ["should pass"], attempts: [ { @@ -698,7 +714,7 @@ const uploadResultsMatcher = ? [] : ([ { - filename: "../integration-input/src/quarantined.test.ts", + filename: specRepoPath(params, "quarantined"), name: ["describe block", "should be quarantined"], attempts: Array( expectedFailureRetries + 1 @@ -728,6 +744,7 @@ const uploadResultsMatcher = ); return true; }; +}; const addFetchMockExpectations = ( params: TestCaseParams, @@ -767,19 +784,19 @@ const addFetchMockExpectations = ( quarantined_tests: [ { test_id: "TEST_QUARANTINED", - filename: "../integration-input/src/quarantined.test.ts", + filename: specRepoPath(params, "quarantined"), name: ["describe block", "should be quarantined"], }, { test_id: "TEST_QUARANTINED2", - filename: "../integration-input/src/mixed.test.ts", + filename: specRepoPath(params, "mixed"), name: ["mixed", "mixed: should be quarantined"], }, ...(quarantineFlake ? [ { test_id: "TEST_FLAKE", - filename: "../integration-input/src/flake.test.ts", + filename: specRepoPath(params, "flake"), name: [ `should be flaky 1${expectedFlakeTestNameSuffix}`.substring( 0, @@ -789,7 +806,7 @@ const addFetchMockExpectations = ( }, { test_id: "TEST_FLAKE", - filename: "../integration-input/src/flake.test.ts", + filename: specRepoPath(params, "flake"), name: [ `should be flaky 2${expectedFlakeTestNameSuffix}`.substring( 0, @@ -1363,10 +1380,7 @@ export const runTestCase = async ( "--runner", "@unflakable/jest-plugin/dist/runner", ...(skipFailures - ? [ - "--testPathIgnorePatterns", - "../integration-input/src/invalid\\.test\\.ts", - ] + ? ["--testPathIgnorePatterns", "/src/invalid\\.test\\.ts"] : []), ...(testNamePattern !== undefined ? ["--testNamePattern", testNamePattern] diff --git a/packages/plugins-common/src/git.ts b/packages/plugins-common/src/git.ts index 7a1e104..d5d94cf 100644 --- a/packages/plugins-common/src/git.ts +++ b/packages/plugins-common/src/git.ts @@ -84,22 +84,20 @@ export const loadGitRepo = async (): Promise => { }; export const autoDetectGit = async ( + git: SimpleGit, log: (message: string) => void ): Promise<{ branch: string | undefined; commit: string | undefined; }> => { try { - const git = await loadGitRepo(); - if (git !== null) { - const commit = await getCurrentGitCommit(git); - const branch = await getCurrentGitBranch(git, commit); + const commit = await getCurrentGitCommit(git); + const branch = await getCurrentGitBranch(git, commit); - return { - branch, - commit, - }; - } + return { + branch, + commit, + }; } catch (e) { log( `WARNING: Unflakable failed to auto-detect current git branch and commit: ${ From 058a3e32ad87904c3b3bc08384ad202430393ea3 Mon Sep 17 00:00:00 2001 From: "David A. Ramos" Date: Sat, 8 Jul 2023 16:59:43 -0700 Subject: [PATCH 05/53] [jest] Run Jest integration tests in child process This reworks the Jest integration tests to match the Cypress integration tests: each test is run in its own child process, and stdout/stderr are captured. This approach avoids the need for brittle mocking and prevents tests from interfering with each other. --- .github/workflows/ci.yaml | 23 +- package.json | 6 +- .../test/integration-common/tsconfig.json | 9 - .../integration-input-esm/cypress-config.cjs | 6 +- .../integration-input-esm/cypress-config.js | 4 +- .../integration-input-esm/cypress.config.ts | 4 +- .../test/integration-input-esm/package.json | 2 +- .../cypress-config.mjs | 4 +- .../cypress.config.js | 6 +- .../integration-input-manual/package.json | 2 +- .../test/integration-input/cypress-config.js | 6 +- .../test/integration-input/cypress-config.mjs | 4 +- .../test/integration-input/cypress.config.ts | 4 +- .../test/integration-input/package.json | 2 +- .../test/integration/package.json | 5 +- .../test/integration/src/run-test-case.ts | 543 ++---- .../test/integration/src/test-wrappers.ts | 35 +- packages/jest-plugin/src/runner.ts | 4 +- .../test/integration/jest.config.js | 3 + .../jest-plugin/test/integration/package.json | 11 +- .../test/integration/src/basic.test.ts | 154 +- .../test/integration/src/common.ts | 180 -- .../test/integration/src/config.test.ts | 82 +- .../integration/src/disablePlugin.test.ts | 128 +- .../integration/src/disableUpload.test.ts | 74 +- .../test/integration/src/git.test.ts | 140 +- .../integration/src/ignoreFailures.test.ts | 24 +- .../test/integration/src/longNames.test.ts | 76 +- .../test/integration/src/noQuarantine.test.ts | 52 +- .../integration/src/pluginFailures.test.ts | 118 +- .../test/integration/src/retries.test.ts | 74 +- .../test/integration/src/runTestCase.ts | 1696 +++++------------ .../test/integration/src/skipTests.test.ts | 216 ++- .../test/integration/src/snapshots.test.ts | 36 +- .../test/integration/src/test-wrappers.ts | 108 ++ .../integration/src/testNamePattern.test.ts | 164 +- .../test/integration/src/verify-output.ts | 379 ++++ packages/plugins-common/src/config.ts | 16 +- packages/plugins-common/src/index.ts | 1 + packages/plugins-common/src/manifest.ts | 7 +- .../.eslintrc.js | 2 +- .../package.json | 11 +- .../rollup.config.mjs | 13 +- .../src/config.ts | 36 +- .../src/git.ts | 0 packages/test-common/src/mock-backend.ts | 256 +++ .../src/mock-cosmiconfig.ts | 3 +- packages/test-common/src/mock-git.ts | 7 + packages/test-common/src/spawn.ts | 155 ++ .../src/tsconfig.json | 3 +- packages/test-common/tsconfig.json | 8 + yarn.lock | 179 +- 52 files changed, 2376 insertions(+), 2705 deletions(-) delete mode 100644 packages/cypress-plugin/test/integration-common/tsconfig.json delete mode 100644 packages/jest-plugin/test/integration/src/common.ts create mode 100644 packages/jest-plugin/test/integration/src/test-wrappers.ts create mode 100644 packages/jest-plugin/test/integration/src/verify-output.ts rename packages/{cypress-plugin/test/integration-common => test-common}/.eslintrc.js (62%) rename packages/{cypress-plugin/test/integration-common => test-common}/package.json (68%) rename packages/{cypress-plugin/test/integration-common => test-common}/rollup.config.mjs (80%) rename packages/{cypress-plugin/test/integration-common => test-common}/src/config.ts (60%) rename packages/{cypress-plugin/test/integration-common => test-common}/src/git.ts (100%) create mode 100644 packages/test-common/src/mock-backend.ts rename packages/{cypress-plugin/test/integration-common => test-common}/src/mock-cosmiconfig.ts (77%) create mode 100644 packages/test-common/src/mock-git.ts create mode 100644 packages/test-common/src/spawn.ts rename packages/{cypress-plugin/test/integration-common => test-common}/src/tsconfig.json (79%) create mode 100644 packages/test-common/tsconfig.json diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index f4d7def..307d146 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -153,9 +153,6 @@ jobs: mv package-new.json package.json yarn install --no-immutable - - name: Build test dependencies - run: yarn build:plugins-common && yarn build:cypress-tests - - name: Set Cypress version env: CYPRESS_INSTALL_BINARY: "0" @@ -166,6 +163,9 @@ jobs: - name: Install Cypress binary run: yarn workspace cypress-integration exec cypress install + - name: Build test dependencies + run: yarn build:plugins-common && yarn build:test-common && yarn build:cypress-tests + - name: Test env: # Enable debug logs within the Jest tests that run Cypress. WARNING: these are very @@ -261,9 +261,6 @@ jobs: mv package-new.json package.json yarn install --no-immutable - - name: Build test dependencies - run: yarn build:plugins-common && yarn build:cypress-tests - - name: Set Cypress version env: CYPRESS_INSTALL_BINARY: "0" @@ -274,6 +271,9 @@ jobs: - name: Install Cypress binary run: yarn workspace cypress-integration exec cypress install + - name: Build test dependencies + run: yarn build:plugins-common && yarn build:test-common && yarn build:cypress-tests + - name: Test env: # Enable debug logs within the Jest tests that run Cypress. WARNING: these are very @@ -379,9 +379,18 @@ jobs: if: ${{ startsWith(matrix.jest, '25.') }} run: yarn set resolution "chalk@npm:^3.0.0 || ^4.0.0" 3.0 + - name: Build test dependencies + run: yarn build:plugins-common && yarn build:test-common + - name: Test env: - DEBUG: unflakable:* + # Enable debug logs within the jest-plugin/test/integration Jest tests that invoke Jest + # on the jest-plugin/test/integration-input test cases. WARNING: these are very verbose + # but are useful for seeing the raw chalk terminal codes. + # DEBUG: unflakable:* + + # Enable debug logs within the Jest plugin. + TEST_DEBUG: "unflakable:*" run: | if [ "${{ github.repository }}" == "unflakable/unflakable-javascript" ]; then export UNFLAKABLE_SUITE_ID=29KWCuK12VnU7pkpvWgrGS0woAX diff --git a/package.json b/package.json index c01d2c5..39f540e 100644 --- a/package.json +++ b/package.json @@ -32,8 +32,9 @@ "build:jest-plugin": "yarn workspace @unflakable/jest-plugin build", "build:plugins-common": "yarn workspace @unflakable/plugins-common build", "typecheck:scripts": "tsc --noEmit --types node -p scripts", - "build:tests": "yarn build:cypress-tests && yarn typecheck:jest-tests", - "build:cypress-tests": "yarn workspace cypress-integration-common build && yarn workspace cypress-integration typecheck && yarn workspace cypress-integration-input typecheck && yarn workspace cypress-integration-input-esm typecheck", + "build:test-common": "yarn workspace unflakable-test-common build", + "build:tests": "yarn build:test-common && yarn build:cypress-tests && yarn typecheck:jest-tests", + "build:cypress-tests": "yarn workspace cypress-integration typecheck && yarn workspace cypress-integration-input typecheck && yarn workspace cypress-integration-input-esm typecheck", "typecheck:jest-tests": "yarn workspace jest-integration typecheck && yarn workspace jest-integration-input typecheck", "lint": "eslint .", "prettier": "prettier --write .", @@ -44,7 +45,6 @@ "workspaces": [ "packages/*", "packages/*/test/integration", - "packages/*/test/integration-common", "packages/*/test/integration-input", "packages/*/test/integration-input-esm", "packages/*/test/integration-input-manual" diff --git a/packages/cypress-plugin/test/integration-common/tsconfig.json b/packages/cypress-plugin/test/integration-common/tsconfig.json deleted file mode 100644 index 6008fb3..0000000 --- a/packages/cypress-plugin/test/integration-common/tsconfig.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "extends": "../../../../tsconfig.json", - "compilerOptions": { - // Removes DOM types. - "lib": ["ES2019"], - "types": ["node"] - }, - "include": [".eslintrc.js", "rollup.config.mjs"] -} diff --git a/packages/cypress-plugin/test/integration-input-esm/cypress-config.cjs b/packages/cypress-plugin/test/integration-input-esm/cypress-config.cjs index f09d4d1..cb9736e 100644 --- a/packages/cypress-plugin/test/integration-input-esm/cypress-config.cjs +++ b/packages/cypress-plugin/test/integration-input-esm/cypress-config.cjs @@ -1,11 +1,9 @@ // Copyright (c) 2023 Developer Innovations, LLC -const { - registerSimpleGitMock, -} = require("cypress-integration-common/dist/git"); +const { registerSimpleGitMock } = require("unflakable-test-common/dist/git"); const { registerCosmiconfigMock, -} = require("cypress-integration-common/dist/config"); +} = require("unflakable-test-common/dist/config"); module.exports = { /** diff --git a/packages/cypress-plugin/test/integration-input-esm/cypress-config.js b/packages/cypress-plugin/test/integration-input-esm/cypress-config.js index d659108..ee5d049 100644 --- a/packages/cypress-plugin/test/integration-input-esm/cypress-config.js +++ b/packages/cypress-plugin/test/integration-input-esm/cypress-config.js @@ -3,8 +3,8 @@ import { openDevToolsOnLaunch } from "./config-js/devtools.js"; import { registerTasks } from "./config-js/tasks.js"; import webpackConfig from "./config-js/webpack.js"; -import { registerSimpleGitMock } from "cypress-integration-common/dist/git.js"; -import { registerCosmiconfigMock } from "cypress-integration-common/dist/config.js"; +import { registerSimpleGitMock } from "unflakable-test-common/dist/git.js"; +import { registerCosmiconfigMock } from "unflakable-test-common/dist/config.js"; /** * @type {Cypress.ConfigOptions} diff --git a/packages/cypress-plugin/test/integration-input-esm/cypress.config.ts b/packages/cypress-plugin/test/integration-input-esm/cypress.config.ts index 1778af1..c7f9649 100644 --- a/packages/cypress-plugin/test/integration-input-esm/cypress.config.ts +++ b/packages/cypress-plugin/test/integration-input-esm/cypress.config.ts @@ -6,8 +6,8 @@ import { defineConfig } from "cypress"; import * as devtools from "./config/devtools.js"; import { registerTasks } from "./config/tasks.js"; import webpackConfig from "./config/webpack.js"; -import { registerSimpleGitMock } from "cypress-integration-common/dist/git.js"; -import { registerCosmiconfigMock } from "cypress-integration-common/dist/config.js"; +import { registerSimpleGitMock } from "unflakable-test-common/dist/git.js"; +import { registerCosmiconfigMock } from "unflakable-test-common/dist/config.js"; export default defineConfig({ component: { diff --git a/packages/cypress-plugin/test/integration-input-esm/package.json b/packages/cypress-plugin/test/integration-input-esm/package.json index d591569..a455fa8 100644 --- a/packages/cypress-plugin/test/integration-input-esm/package.json +++ b/packages/cypress-plugin/test/integration-input-esm/package.json @@ -8,7 +8,6 @@ "@types/react-dom": "^18.2.4", "@unflakable/cypress-plugin": "workspace:^", "cypress": "10 - 12", - "cypress-integration-common": "workspace:^", "mocha": "=7.0.1", "mocha-junit-reporter": "^2.2.0", "process": "^0.11.10", @@ -16,6 +15,7 @@ "react-dom": "^18.2.0", "ts-loader": "^9.4.3", "typescript": "^4.9.5", + "unflakable-test-common": "workspace:^", "webpack": "^5.84.1" }, "scripts": { diff --git a/packages/cypress-plugin/test/integration-input-manual/cypress-config.mjs b/packages/cypress-plugin/test/integration-input-manual/cypress-config.mjs index 1200a83..db5d6b2 100644 --- a/packages/cypress-plugin/test/integration-input-manual/cypress-config.mjs +++ b/packages/cypress-plugin/test/integration-input-manual/cypress-config.mjs @@ -3,8 +3,8 @@ import devtools from "./config/devtools.js"; import tasks from "./config/tasks.js"; import webpackConfig from "./config/webpack.js"; -import { registerSimpleGitMock } from "cypress-integration-common/dist/git.js"; -import { registerCosmiconfigMock } from "cypress-integration-common/dist/config.js"; +import { registerSimpleGitMock } from "unflakable-test-common/dist/git.js"; +import { registerCosmiconfigMock } from "unflakable-test-common/dist/config.js"; import { registerUnflakable } from "@unflakable/cypress-plugin"; /** diff --git a/packages/cypress-plugin/test/integration-input-manual/cypress.config.js b/packages/cypress-plugin/test/integration-input-manual/cypress.config.js index 0a7d411..bb3bc5a 100644 --- a/packages/cypress-plugin/test/integration-input-manual/cypress.config.js +++ b/packages/cypress-plugin/test/integration-input-manual/cypress.config.js @@ -3,12 +3,10 @@ const { openDevToolsOnLaunch } = require("./config/devtools"); const { registerTasks } = require("./config/tasks"); const webpackConfig = require("./config/webpack"); -const { - registerSimpleGitMock, -} = require("cypress-integration-common/dist/git"); +const { registerSimpleGitMock } = require("unflakable-test-common/dist/git"); const { registerCosmiconfigMock, -} = require("cypress-integration-common/dist/config"); +} = require("unflakable-test-common/dist/config"); const { registerUnflakable } = require("@unflakable/cypress-plugin"); diff --git a/packages/cypress-plugin/test/integration-input-manual/package.json b/packages/cypress-plugin/test/integration-input-manual/package.json index f880b09..0591f0d 100644 --- a/packages/cypress-plugin/test/integration-input-manual/package.json +++ b/packages/cypress-plugin/test/integration-input-manual/package.json @@ -10,7 +10,6 @@ "@types/react-dom": "^18.2.4", "@unflakable/cypress-plugin": "workspace:^", "cypress": "10 - 12", - "cypress-integration-common": "workspace:^", "cypress-multi-reporters": "^1.6.3", "mocha": "=7.0.1", "mocha-junit-reporter": "^2.2.0", @@ -19,6 +18,7 @@ "react-dom": "^18.2.0", "ts-loader": "^9.4.3", "typescript": "^4.9.5", + "unflakable-test-common": "workspace:^", "webpack": "^5.84.1" } } diff --git a/packages/cypress-plugin/test/integration-input/cypress-config.js b/packages/cypress-plugin/test/integration-input/cypress-config.js index 421f448..50319f0 100644 --- a/packages/cypress-plugin/test/integration-input/cypress-config.js +++ b/packages/cypress-plugin/test/integration-input/cypress-config.js @@ -3,12 +3,10 @@ const { openDevToolsOnLaunch } = require("config-js/devtools"); const webpackConfig = require("config-js/webpack"); const { registerTasks } = require("config-js/tasks"); -const { - registerSimpleGitMock, -} = require("cypress-integration-common/dist/git"); +const { registerSimpleGitMock } = require("unflakable-test-common/dist/git"); const { registerCosmiconfigMock, -} = require("cypress-integration-common/dist/config"); +} = require("unflakable-test-common/dist/config"); module.exports = { /** diff --git a/packages/cypress-plugin/test/integration-input/cypress-config.mjs b/packages/cypress-plugin/test/integration-input/cypress-config.mjs index cb12ca2..b6e6c0f 100644 --- a/packages/cypress-plugin/test/integration-input/cypress-config.mjs +++ b/packages/cypress-plugin/test/integration-input/cypress-config.mjs @@ -3,8 +3,8 @@ import devtools from "./config-js/devtools.js"; import tasks from "./config-js/tasks.js"; import webpackConfig from "./config-js/webpack.js"; -import { registerSimpleGitMock } from "cypress-integration-common/dist/git.js"; -import { registerCosmiconfigMock } from "cypress-integration-common/dist/config.js"; +import { registerSimpleGitMock } from "unflakable-test-common/dist/git.js"; +import { registerCosmiconfigMock } from "unflakable-test-common/dist/config.js"; /** * @type {Cypress.ConfigOptions} diff --git a/packages/cypress-plugin/test/integration-input/cypress.config.ts b/packages/cypress-plugin/test/integration-input/cypress.config.ts index 88a41f0..1a6359b 100644 --- a/packages/cypress-plugin/test/integration-input/cypress.config.ts +++ b/packages/cypress-plugin/test/integration-input/cypress.config.ts @@ -1,7 +1,7 @@ // Copyright (c) 2023 Developer Innovations, LLC -import { registerSimpleGitMock } from "cypress-integration-common/dist/git"; -import { registerCosmiconfigMock } from "cypress-integration-common/dist/config"; +import { registerSimpleGitMock } from "unflakable-test-common/dist/git"; +import { registerCosmiconfigMock } from "unflakable-test-common/dist/config"; import { defineConfig } from "cypress"; // This intentionally uses the CommonJS relative import syntax that doesn't start with `./` in // order to test that our inclusion of the user config file resolves relative path imports (via diff --git a/packages/cypress-plugin/test/integration-input/package.json b/packages/cypress-plugin/test/integration-input/package.json index 1f5bc39..8722d91 100644 --- a/packages/cypress-plugin/test/integration-input/package.json +++ b/packages/cypress-plugin/test/integration-input/package.json @@ -7,7 +7,6 @@ "@types/react-dom": "^18.2.4", "@unflakable/cypress-plugin": "workspace:^", "cypress": "10 - 12", - "cypress-integration-common": "workspace:^", "mocha": "=7.0.1", "mocha-junit-reporter": "^2.2.0", "process": "^0.11.10", @@ -15,6 +14,7 @@ "react-dom": "^18.2.0", "ts-loader": "^9.4.3", "typescript": "^4.9.5", + "unflakable-test-common": "workspace:^", "webpack": "^5.84.1" }, "scripts": { diff --git a/packages/cypress-plugin/test/integration/package.json b/packages/cypress-plugin/test/integration/package.json index 15c65f1..a36693b 100644 --- a/packages/cypress-plugin/test/integration/package.json +++ b/packages/cypress-plugin/test/integration/package.json @@ -7,16 +7,15 @@ "@unflakable/js-api": "workspace:^", "@unflakable/plugins-common": "workspace:^", "cypress": "10 - 12", - "cypress-integration-common": "workspace:^", "debug": "^4.3.3", "escape-string-regexp": "^4.0.0", "jest": "^29.5.0", "jest-environment-node": "^29.5.0", "jest-expect-message": "^1.1.3", "mockttp": "^3.7.5", - "tree-kill": "^1.2.2", "ts-jest": "^29.1.0", - "typescript": "^4.9.5" + "typescript": "^4.9.5", + "unflakable-test-common": "workspace:^" }, "scripts": { "test": "jest --useStderr --verbose --runInBand", diff --git a/packages/cypress-plugin/test/integration/src/run-test-case.ts b/packages/cypress-plugin/test/integration/src/run-test-case.ts index 2a66fc4..5bf324d 100644 --- a/packages/cypress-plugin/test/integration/src/run-test-case.ts +++ b/packages/cypress-plugin/test/integration/src/run-test-case.ts @@ -1,44 +1,38 @@ // Copyright (c) 2023 Developer Innovations, LLC import { - CreateTestSuiteRunFromUploadRequest, CreateTestSuiteRunInlineRequest, TEST_NAME_ENTRY_MAX_LENGTH, TestAttemptResult, TestRunRecord, - TestSuiteManifest, - TestSuiteRunPendingSummary, } from "@unflakable/js-api"; import { gunzipSync } from "zlib"; import { UnflakableConfig } from "@unflakable/plugins-common"; -import { - CompletedRequest, - getLocal as getLocalHttpServer, - MockedEndpoint, -} from "mockttp"; +import { CompletedRequest } from "mockttp"; import _debug from "debug"; -import { execFile, spawn } from "child_process"; -import type { - CallbackResponseMessageResult, - CallbackResponseResult, -} from "mockttp/dist/rules/requests/request-handler-definitions"; -import { promisify, TextDecoder } from "util"; +import { execFile } from "child_process"; +import { promisify } from "util"; import { CONFIG_MOCK_ENV_VAR, CosmiconfigMockParams, -} from "cypress-integration-common/dist/config"; +} from "unflakable-test-common/dist/config"; import { GIT_MOCK_ENV_VAR, SimpleGitMockParams, -} from "cypress-integration-common/dist/git"; +} from "unflakable-test-common/dist/git"; import path from "path"; import { SummaryTotals } from "./parse-output"; import { expect as expectExt } from "@jest/globals"; import "./matchers"; import { verifyOutput } from "./verify-output"; -import treeKill from "tree-kill"; - -const debug = _debug("unflakable:integration-test:run-test-case"); +import { + MockBackend, + UnmatchedEndpoints, +} from "unflakable-test-common/dist/mock-backend"; +import { + AsyncTestError, + spawnTestWithTimeout, +} from "unflakable-test-common/dist/spawn"; // Jest times out after 120 seconds, so we bail early here to allow time to print the // captured output before Jest kills the test. @@ -172,14 +166,7 @@ export const specPattern = (params: TestCaseParams): string => { const specRepoPath = (params: TestCaseParams, specNameStub: string): string => params.expectedRepoRelativePathPrefix + specProjectPath(params, specNameStub); -export const apiServer = getLocalHttpServer({ - // debug: true, - suggestChanges: false, -}); -export const objectStoreServer = getLocalHttpServer({ - // debug: true, - suggestChanges: false, -}); +export const mockBackend = new MockBackend(); export const MOCK_RUN_ID = "MOCK_RUN_ID"; const TIMESTAMP_REGEX = @@ -455,20 +442,18 @@ const verifyUploadResults = ( ); }; -const addFetchMockExpectations = async ( +const addBackendExpectations = async ( params: TestCaseParams, summaryTotals: SummaryTotals, onError: (e: unknown) => void -): Promise<{ - unmatchedApiRequestEndpoint: MockedEndpoint; - unmatchedObjectStoreRequestEndpoint: MockedEndpoint; -}> => { +): Promise => { const { expectedApiKey, expectedBranch, expectedCommit, expectedFlakeTestNameSuffix, expectedSuiteId, + expectPluginToBeEnabled, expectResultsToBeUploaded, failToFetchManifest, failToUploadResults, @@ -477,258 +462,102 @@ const addFetchMockExpectations = async ( quarantineHookSkip, } = params; - const onUnmatchedRequest = ( - request: CompletedRequest - ): CallbackResponseResult => { - onError(new Error(`Unexpected request ${request.method} ${request.path}`)); - return { statusCode: 500 }; - }; - - const unmatchedApiRequestEndpoint = await apiServer - .forUnmatchedRequest() - .thenCallback(onUnmatchedRequest); - const unmatchedObjectStoreRequestEndpoint = await objectStoreServer - .forUnmatchedRequest() - .thenCallback(onUnmatchedRequest); - - if (!params.expectPluginToBeEnabled) { - return { - unmatchedApiRequestEndpoint, - unmatchedObjectStoreRequestEndpoint, - }; - } - - await apiServer - .forGet(`/api/v1/test-suites/${expectedSuiteId}/manifest`) - .times(failToFetchManifest ? 3 : 1) - .withHeaders({ - Authorization: `Bearer ${expectedApiKey}`, - }) - .thenCallback((request): CallbackResponseResult => { - try { - expect(request.headers["user-agent"]).toMatch(userAgentRegex); - - if (failToFetchManifest) { - return "reset"; - } - - const responseBody: TestSuiteManifest = { - quarantined_tests: [ - { - test_id: "TEST_QUARANTINED", - filename: specRepoPath(params, "quarantined"), - name: ["describe block", "should be quarantined"], - }, + const manifest = { + quarantined_tests: [ + { + test_id: "TEST_QUARANTINED", + filename: specRepoPath(params, "quarantined"), + name: ["describe block", "should be quarantined"], + }, + { + test_id: "TEST_MIXED_QUARANTINED_FAIL", + filename: specRepoPath(params, "mixed/mixed"), + name: [ + "spec with mixed test results", + "mixed: failure should be quarantined", + ], + }, + { + test_id: "TEST_MIXED_QUARANTINED_FLAKE", + filename: specRepoPath(params, "mixed/mixed"), + name: [ + "spec with mixed test results", + "mixed: flake should be quarantined", + ], + }, + { + test_id: "TEST_QUARANTINED_PENDING", + filename: specRepoPath(params, "pending"), + name: ["suite name", "suite test should be quarantined and pending"], + }, + ...(quarantineFlake + ? [ { - test_id: "TEST_MIXED_QUARANTINED_FAIL", - filename: specRepoPath(params, "mixed/mixed"), + test_id: "TEST_FLAKE", + filename: specRepoPath(params, "flake"), name: [ - "spec with mixed test results", - "mixed: failure should be quarantined", + `should be flaky${expectedFlakeTestNameSuffix}`.substring( + 0, + TEST_NAME_ENTRY_MAX_LENGTH + ), ], }, { - test_id: "TEST_MIXED_QUARANTINED_FLAKE", + test_id: "TEST_MIXED_FLAKE", filename: specRepoPath(params, "mixed/mixed"), - name: [ - "spec with mixed test results", - "mixed: flake should be quarantined", - ], + name: ["spec with mixed test results", "mixed: should be flaky"], }, + ] + : []), + ...(quarantineHookFail + ? [ { - test_id: "TEST_QUARANTINED_PENDING", - filename: specRepoPath(params, "pending"), - name: [ - "suite name", - "suite test should be quarantined and pending", - ], - }, - ...(quarantineFlake - ? [ - { - test_id: "TEST_FLAKE", - filename: specRepoPath(params, "flake"), - name: [ - `should be flaky${expectedFlakeTestNameSuffix}`.substring( - 0, - TEST_NAME_ENTRY_MAX_LENGTH - ), - ], - }, - { - test_id: "TEST_MIXED_FLAKE", - filename: specRepoPath(params, "mixed/mixed"), - name: [ - "spec with mixed test results", - "mixed: should be flaky", - ], - }, - ] - : []), - ...(quarantineHookFail - ? [ - { - test_id: "TEST_HOOK_FAIL", - filename: specRepoPath(params, "hook-fail"), - name: ["describe block", "should fail due to hook"], - }, - ] - : []), - ...(quarantineHookSkip - ? [ - { - test_id: "TEST_HOOK_SKIP", - filename: specRepoPath(params, "hook-fail"), - name: ["describe block", "should be skipped"], - }, - ] - : []), - ], - }; - - return { - statusCode: 200, - json: responseBody, - }; - } catch (e: unknown) { - onError(e); - return { statusCode: 500 }; - } - }); - - if (expectResultsToBeUploaded) { - const uploadPath = - `/unflakable-backend-mock-test-uploads/teams/MOCK_TEAM_ID/suites/${expectedSuiteId}/runs/` + - `upload/MOCK_UPLOAD_ID`; - const uploadQuery = "?X-Amz-Signature=MOCK_SIGNATURE"; - - await apiServer - .forPost(`/api/v1/test-suites/${expectedSuiteId}/runs/upload`) - .once() - .withHeaders({ - Authorization: `Bearer ${expectedApiKey}`, - "Content-Type": "application/json", - }) - .thenCallback(async (request) => { - try { - expect(await request.body.getText()).toBe(""); - return { - statusCode: 201, - headers: { - Location: `http://localhost:${objectStoreServer.port}${uploadPath}${uploadQuery}`, + test_id: "TEST_HOOK_FAIL", + filename: specRepoPath(params, "hook-fail"), + name: ["describe block", "should fail due to hook"], }, - json: { - upload_id: "MOCK_UPLOAD_ID", + ] + : []), + ...(quarantineHookSkip + ? [ + { + test_id: "TEST_HOOK_SKIP", + filename: specRepoPath(params, "hook-fail"), + name: ["describe block", "should be skipped"], }, - }; - } catch (e) { - onError(e); - return { - statusCode: 500, - }; - } - }); - - let runRequest: CreateTestSuiteRunInlineRequest | null = null; - await objectStoreServer - .forPut(uploadPath) - .once() - .withExactQuery(uploadQuery) - .withHeaders({ - "Content-Encoding": "gzip", - "Content-Type": "application/json", - }) - .thenCallback((request): CallbackResponseMessageResult => { - try { - runRequest = JSON.parse( - gunzipSync(request.body.buffer).toString() - ) as CreateTestSuiteRunInlineRequest; - - verifyUploadResults(params, summaryTotals, request); - - return { - statusCode: 200, - }; - } catch (e) { - onError(e); - return { statusCode: 500 }; - } - }); - - await apiServer - .forPost(`/api/v1/test-suites/${expectedSuiteId}/runs`) - .times(failToUploadResults ? 3 : 1) - .withHeaders({ - Authorization: `Bearer ${expectedApiKey}`, - "Content-Type": "application/json", - }) - .thenCallback(async (request): Promise => { - try { - const body = await request.body.getText(); - expect(body).not.toBeNull(); - - const parsedBody = ((): CreateTestSuiteRunFromUploadRequest => { - try { - return JSON.parse( - body as string - ) as CreateTestSuiteRunFromUploadRequest; - } catch (e) { - throw new Error(`Invalid request body: ${JSON.stringify(body)}`, { - cause: e, - }); - } - })(); - - expect(parsedBody.upload_id).toBe("MOCK_UPLOAD_ID"); - expect(runRequest).not.toBeNull(); - - if (failToUploadResults) { - return "reset"; - } - - const parsedRequest = runRequest as CreateTestSuiteRunInlineRequest; - - return { - json: { - run_id: MOCK_RUN_ID, - suite_id: expectedSuiteId, - ...(expectedBranch !== undefined - ? { - branch: expectedBranch, - } - : {}), - ...(expectedCommit !== undefined - ? { - commit: expectedCommit, - } - : {}), - start_time: parsedRequest.start_time, - end_time: parsedRequest.end_time, - num_tests: - summaryTotals.numFailing + - summaryTotals.numFlaky + - summaryTotals.numQuarantined + - summaryTotals.numPassing, - num_pass: summaryTotals.numPassing, - num_fail: summaryTotals.numFailing, - num_flake: summaryTotals.numFlaky, - num_quarantined: summaryTotals.numQuarantined, - } as TestSuiteRunPendingSummary, - statusCode: 201, - }; - } catch (e) { - onError(e); - return { - statusCode: 500, - }; - } - }); - } - - return { - unmatchedApiRequestEndpoint, - unmatchedObjectStoreRequestEndpoint, + ] + : []), + ], }; + + return mockBackend.addExpectations( + onError, + failToFetchManifest ? null : manifest, + (request) => verifyUploadResults(params, summaryTotals, request), + failToUploadResults + ? null + : { + run_id: MOCK_RUN_ID, + suite_id: expectedSuiteId, + ...(expectedBranch !== undefined + ? { + branch: expectedBranch, + } + : {}), + ...(expectedCommit !== undefined + ? { + commit: expectedCommit, + } + : {}), + }, + userAgentRegex, + { + expectPluginToBeEnabled, + expectResultsToBeUploaded, + expectedApiKey, + expectedSuiteId, + } + ); }; // Similar to debug's default formatter, but with timestamps instead of the +Nms at the end of each @@ -770,16 +599,19 @@ export const runTestCase = async ( multipleHookErrors, } = params; - const testError = { error: undefined as unknown | undefined }; + const asyncTestError: AsyncTestError = { error: undefined }; - const { unmatchedApiRequestEndpoint, unmatchedObjectStoreRequestEndpoint } = - await addFetchMockExpectations(params, summaryTotals, (error) => { - if (testError.error === undefined) { - testError.error = error ?? new Error("undefined error"); + const unmatchedRequestEndpoints = await addBackendExpectations( + params, + summaryTotals, + (error) => { + if (asyncTestError.error === undefined) { + asyncTestError.error = error ?? new Error("undefined error"); } else { console.error("Multiple failed fetch expectations", error); } - }); + } + ); const configMockParams: CosmiconfigMockParams = { searchFrom: path.resolve(projectPath(params)), @@ -826,7 +658,7 @@ export const runTestCase = async ( const args = [ "--require", - require.resolve("cypress-integration-common/dist/mock-cosmiconfig"), + require.resolve("unflakable-test-common/dist/mock-cosmiconfig"), cypressPluginBin, ...(params.project === "integration-input-manual" ? ["--no-auto-config", "--no-auto-support"] @@ -860,7 +692,7 @@ export const runTestCase = async ( // NODE_OPTIONS: "--loader=testdouble", // Needed for resolving `cypress-unflakable` path. PATH: process.env.PATH, - UNFLAKABLE_API_BASE_URL: `http://localhost:${apiServer.port}`, + UNFLAKABLE_API_BASE_URL: `http://localhost:${mockBackend.apiServerPort}`, [CONFIG_MOCK_ENV_VAR]: JSON.stringify(configMockParams), [GIT_MOCK_ENV_VAR]: JSON.stringify(params.git), // Windows requires these environment variables to be propagated. @@ -874,141 +706,22 @@ export const runTestCase = async ( : {}), }; - debug( - `Spawning test:\n args = %o\n environment = %o\n cwd = %s`, + await spawnTestWithTimeout( args, env, - projectPath(params) - ); - - const cypressChild = spawn("node", args, { - cwd: projectPath(params), - env, - }); - - const onOutput = ( - name: string, - onLine: (line: string, now: Date) => void, - escapeDebugOutput: boolean - ): ((data: Buffer) => void) => { - const debugExt = debug.extend(name); - const decoder = new TextDecoder("utf-8", { fatal: true }); - - const pending = { s: "" }; - - // Don't eat the last line of output. - cypressChild.on("exit", () => { - if (pending.s !== "") { - onLine(pending.s, new Date()); - debugExt(escapeDebugOutput ? JSON.stringify(pending.s) : pending.s); - } - }); - - return (data: Buffer): void => { - const now = new Date(); - // In case data terminates in the middle of a Unicode sequence, we need to use a stateful - // TextDecoder with `stream: true`. Otherwise, invalid UTF-8 sequences at the end get - // converted to 0xFFFD, which breaks the tests non-deterministically (i.e., makes them flaky). - const lines = decoder.decode(data, { stream: true }).split("\n"); - - // If the last line is empty, then `dataStr` ends in a linebreak. Otherwise, we have a - // partial line that we want to defer until the next call. - lines.slice(0, lines.length - 1).forEach((line, idx) => { - const lineWithPending = idx === 0 ? pending.s + line : line; - onLine(lineWithPending, now); - debugExt( - escapeDebugOutput ? JSON.stringify(lineWithPending) : lineWithPending - ); - }); - - pending.s = lines[lines.length - 1]; - }; - }; - - const stdoutLines = [] as string[]; - const combinedLines = [] as string[]; - - cypressChild.stderr.on( - "data", - onOutput( - "stderr", - (line, now) => { - combinedLines.push(`${now.toISOString()} ${line}`); - }, - // Don't escape stderr output since it likely comes from debug output in the subprocess, which - // is intended for human consumption and not for verifying test results. - false - ) - ); - cypressChild.stdout.on( - "data", - onOutput( - "stdout", - (line, now) => { - stdoutLines.push(line); - combinedLines.push(`${now.toISOString()} ${line}`); - }, - // Escape special characters in debug output so that we can more easily understand test - // failures related to unexpected output. - true - ) + projectPath(params), + TEST_TIMEOUT_MS, + async (stdoutLines) => { + verifyOutput( + params, + stdoutLines, + summaryTotals, + mockBackend.apiServerPort + ); + await mockBackend.checkExpectations(unmatchedRequestEndpoints); + }, + expectedExitCode, + false, + asyncTestError ); - - type ChildResult = { - code: number | null; - signal: NodeJS.Signals | null; - }; - - try { - const { code, signal } = await new Promise( - (resolve, reject) => { - const watchdog = setTimeout(() => { - console.error( - `Test timed out after ${TEST_TIMEOUT_MS}ms; killing Cypress process tree` - ); - const timeoutError = new Error( - `Test timed out after ${TEST_TIMEOUT_MS}ms` - ); - if (testError.error === undefined) { - testError.error = timeoutError; - } - treeKill(cypressChild.pid, "SIGKILL", () => reject(timeoutError)); - }, TEST_TIMEOUT_MS); - - cypressChild.on("error", (err) => { - clearTimeout(watchdog); - reject(err); - }); - cypressChild.on("exit", (code, signal) => { - clearTimeout(watchdog); - resolve({ code, signal }); - }); - } - ); - - if (testError.error !== undefined) { - throw testError.error; - } - - verifyOutput(params, stdoutLines, summaryTotals, apiServer.port); - - expect(signal).toBe(null); - expect(code).toBe(expectedExitCode); - - expect(await apiServer.getPendingEndpoints()).toStrictEqual([ - unmatchedApiRequestEndpoint, - ]); - expect(await objectStoreServer.getPendingEndpoints()).toStrictEqual([ - unmatchedObjectStoreRequestEndpoint, - ]); - } catch (e: unknown) { - // Jest doesn't have a built-in setting for printing console logs only for failed tests, so we - // just defer the output until this catch block and attach it to the error. See - // https://github.com/jestjs/jest/issues/4156. We don't call console.log() directly here because - // that output gets printed before the failed test, whereas the error gets printed immediately - // after, which makes it easy to associate with the corresponding test. - throw new Error(`Test failed with output:\n\n${combinedLines.join("\n")}`, { - cause: e, - }); - } }; diff --git a/packages/cypress-plugin/test/integration/src/test-wrappers.ts b/packages/cypress-plugin/test/integration/src/test-wrappers.ts index f3697b7..f0f420d 100644 --- a/packages/cypress-plugin/test/integration/src/test-wrappers.ts +++ b/packages/cypress-plugin/test/integration/src/test-wrappers.ts @@ -1,16 +1,11 @@ // Copyright (c) 2023 Developer Innovations, LLC -import { - apiServer, - objectStoreServer, - runTestCase, - TestCaseParams, -} from "./run-test-case"; +import { mockBackend, runTestCase, TestCaseParams } from "./run-test-case"; import path from "path"; -import _debug from "debug"; import cypressPackage from "cypress/package.json"; import { SummaryTotals } from "./parse-output"; import * as os from "os"; +import * as util from "util"; declare global { // eslint-disable-next-line @typescript-eslint/no-namespace @@ -24,8 +19,6 @@ declare global { } } -const debug = _debug("unflakable:integration-test:test-wrappers"); - export type TestCase = { params: Partial; expectedExitCode?: number; @@ -110,30 +103,14 @@ export const integrationTest = ( ) .then(done) .catch((e) => { - done(e as string | { message: string }); + // Ensures any chained `cause` gets printed. + done(util.inspect(e, { colors: true, depth: 5 })); }); }; export const integrationTestSuite = (runTests: () => void): void => { - beforeEach(async () => { - await apiServer.start(); - debug( - `Listening for mock API requests on http://localhost:${apiServer.port}` - ); - - await objectStoreServer.start(); - debug( - `Listening for mock S3 requests on http://localhost:${objectStoreServer.port}` - ); - }); - - afterEach(async () => { - debug(`Stopping mock API server`); - await apiServer.stop(); - - debug(`Stopping mock S3 server`); - return objectStoreServer.stop(); - }); + beforeEach(() => mockBackend.start()); + afterEach(() => mockBackend.stop()); const cypressMinorVersion = cypressPackage.version.match(/^[^.]+\.[^.]+/); const nodeMajorVersion = process.version.match(/^[^.]+/); diff --git a/packages/jest-plugin/src/runner.ts b/packages/jest-plugin/src/runner.ts index 6940666..75d3e18 100644 --- a/packages/jest-plugin/src/runner.ts +++ b/packages/jest-plugin/src/runner.ts @@ -218,8 +218,8 @@ class UnflakableRunner { 0 )} failed test(s) from ${testFailures.length} file(s) -- ${ attempts - attempt - 1 - } ${attempts - attempt - 1 === 1 ? "retry" : "retries"} remaining\n` - ) + } ${attempts - attempt - 1 === 1 ? "retry" : "retries"} remaining` + ) + "\n" ); // Similar to how we skip quarantined tests when quarantineMode is "skip_tests", we need to diff --git a/packages/jest-plugin/test/integration/jest.config.js b/packages/jest-plugin/test/integration/jest.config.js index e7329eb..33e1278 100644 --- a/packages/jest-plugin/test/integration/jest.config.js +++ b/packages/jest-plugin/test/integration/jest.config.js @@ -12,5 +12,8 @@ module.exports = { ], }, + // NB: This should be greater than TEST_TIMEOUT_MS used by the watchdog in runTestCase(). + testTimeout: 40000, + verbose: true, }; diff --git a/packages/jest-plugin/test/integration/package.json b/packages/jest-plugin/test/integration/package.json index 5a1e554..2d6b21b 100644 --- a/packages/jest-plugin/test/integration/package.json +++ b/packages/jest-plugin/test/integration/package.json @@ -5,16 +5,15 @@ "@types/temp": "^0.9.1", "@unflakable/jest-plugin": "workspace:^", "@unflakable/js-api": "workspace:^", - "cross-env": "^7.0.3", - "deep-equal": "^2.0.5", - "fetch-mock-jest": "^1.5.1", + "escape-string-regexp": "^4.0.0", "jest": "25.1.0 - 29", - "jest-cli": "25.1.0 - 29", "jest-environment-node": "25.1.0 - 29", - "temp": "^0.9.4" + "mockttp": "^3.7.5", + "temp": "^0.9.4", + "unflakable-test-common": "workspace:^" }, "scripts": { - "test": "cross-env NODE_OPTIONS=\"--require ./src/force-color.js\" jest --useStderr --verbose", + "test": "jest --useStderr --verbose", "typecheck": "tsc --build" } } diff --git a/packages/jest-plugin/test/integration/src/basic.test.ts b/packages/jest-plugin/test/integration/src/basic.test.ts index 7047ad9..eb98a3f 100644 --- a/packages/jest-plugin/test/integration/src/basic.test.ts +++ b/packages/jest-plugin/test/integration/src/basic.test.ts @@ -4,80 +4,96 @@ import { defaultExpectedResults, integrationTest, integrationTestSuite, -} from "./common"; +} from "./test-wrappers"; -integrationTestSuite(() => { - it("environment variables config + git auto-detect", () => - integrationTest({ - params: {}, - expectedExitCode: 1, - expectedResults: defaultExpectedResults, - })); - - it("quarantine flaky test", () => - integrationTest({ - params: { - quarantineFlake: true, - }, - expectedExitCode: 1, - expectedResults: { - failedSuites: 3, - failedTests: 2, - flakyTests: 0, - passedSuites: 1, - passedTests: 2, - quarantinedSuites: 2, - quarantinedTests: 4, - skippedSuites: 0, - skippedTests: 0, - passedSnapshots: 1, - failedSnapshots: 0, - totalSnapshots: 1, +integrationTestSuite((mockBackend) => { + it("environment variables config + git auto-detect", (done) => + integrationTest( + { + params: {}, + expectedExitCode: 1, + expectedResults: defaultExpectedResults, }, - })); + mockBackend, + done + )); - it("skip failures", () => - integrationTest({ - params: { - skipFailures: true, + it("quarantine flaky test", (done) => + integrationTest( + { + params: { + quarantineFlake: true, + }, + expectedExitCode: 1, + expectedResults: { + failedSuites: 3, + failedTests: 2, + flakyTests: 0, + passedSuites: 1, + passedTests: 2, + quarantinedSuites: 2, + quarantinedTests: 4, + skippedSuites: 0, + skippedTests: 0, + passedSnapshots: 1, + failedSnapshots: 0, + totalSnapshots: 1, + }, }, - expectedExitCode: 1, - expectedResults: { - failedSuites: 1, - failedTests: 0, - flakyTests: 2, - passedSuites: 1, - passedTests: 2, - quarantinedSuites: 2, - quarantinedTests: 2, - skippedSuites: 1, - skippedTests: 2, - passedSnapshots: 1, - failedSnapshots: 0, - totalSnapshots: 1, - }, - })); + mockBackend, + done + )); - it("run should succeed when skipping failures and quarantining flaky test", () => - integrationTest({ - params: { - quarantineFlake: true, - skipFailures: true, + it("skip failures", (done) => + integrationTest( + { + params: { + skipFailures: true, + }, + expectedExitCode: 1, + expectedResults: { + failedSuites: 1, + failedTests: 0, + flakyTests: 2, + passedSuites: 1, + passedTests: 2, + quarantinedSuites: 2, + quarantinedTests: 2, + skippedSuites: 1, + skippedTests: 2, + passedSnapshots: 1, + failedSnapshots: 0, + totalSnapshots: 1, + }, }, - expectedExitCode: 0, - expectedResults: { - failedSuites: 0, - failedTests: 0, - flakyTests: 0, - passedSuites: 1, - passedTests: 2, - quarantinedSuites: 3, - quarantinedTests: 4, - skippedSuites: 1, - skippedTests: 2, - passedSnapshots: 1, - failedSnapshots: 0, - totalSnapshots: 1, + mockBackend, + done + )); + + it("run should succeed when skipping failures and quarantining flaky test", (done) => + integrationTest( + { + params: { + quarantineFlake: true, + skipFailures: true, + }, + expectedExitCode: 0, + expectedResults: { + failedSuites: 0, + failedTests: 0, + flakyTests: 0, + passedSuites: 1, + passedTests: 2, + quarantinedSuites: 3, + quarantinedTests: 4, + skippedSuites: 1, + skippedTests: 2, + passedSnapshots: 1, + failedSnapshots: 0, + totalSnapshots: 1, + }, }, - })); + mockBackend, + done + )); }); diff --git a/packages/jest-plugin/test/integration/src/common.ts b/packages/jest-plugin/test/integration/src/common.ts deleted file mode 100644 index 61fdc51..0000000 --- a/packages/jest-plugin/test/integration/src/common.ts +++ /dev/null @@ -1,180 +0,0 @@ -// Copyright (c) 2022-2023 Developer Innovations, LLC - -import mockFetchJest from "fetch-mock-jest"; -import { ResultCounts, runTestCase, TestCaseParams } from "./runTestCase"; -import * as cosmiconfig from "cosmiconfig"; -import type { OptionsSync } from "cosmiconfig"; -import { FetchMockSandbox, MockCall } from "fetch-mock"; -import jestPackage from "jest/package.json"; -import path from "path"; - -const throwUnimplemented = (): never => { - throw new Error("unimplemented"); -}; - -// Mocking `fs` is brittle, and making actual filesystem modifications to, e.g., package.json, can -// leave around artifacts that produce hard-to-debug side effects. Instead, we directly mock -// the cosmiconfig package that the jest-plugin uses for reading its config and hope that it's -// being used correctly. We still test one case of the actual implementation through dogfooding. -const mockConfigExplorer: ReturnType = { - clearCaches: jest.fn(throwUnimplemented), - clearLoadCache: jest.fn(throwUnimplemented), - clearSearchCache: jest.fn(throwUnimplemented), - load: jest.fn(throwUnimplemented), - search: jest.fn(throwUnimplemented), -}; -const mockCosmiconfig: typeof cosmiconfig = { - ...jest.requireActual("cosmiconfig"), - cosmiconfigSync: jest.fn((moduleName: string, options?: OptionsSync) => { - expect(moduleName).toBe("unflakable"); - expect(options?.searchPlaces).toContain("package.json"); - expect(options?.searchPlaces).toContain("unflakable.json"); - expect(options?.searchPlaces).toContain("unflakable.js"); - expect(options?.searchPlaces).toContain("unflakable.yaml"); - expect(options?.searchPlaces).toContain("unflakable.yml"); - return mockConfigExplorer; - }), -}; -jest.mock("cosmiconfig", () => mockCosmiconfig); - -// Jest calls exit() (the `exit` NPM package, not process.exit directly) if our custom reporter or -// runners throw an exception. This in turn causes the whole test run to exit rather than reporting -// the specific test as a failure. Instead, we mock exit() to log a message and continue execution. -const mockExit = jest.fn(); -jest.mock("exit", () => mockExit); - -jest.mock("node-fetch", () => mockFetchJest.sandbox()); - -jest.mock("simple-git"); -jest.setTimeout(30000); - -const originalStderrWrite = process.stderr.write.bind(process.stderr); - -export type TestCase = { - params: Partial; - expectedExitCode: number; - expectedResults: ResultCounts; -}; - -export const defaultExpectedResults: ResultCounts = { - failedSuites: 4, - failedTests: 2, - flakyTests: 2, - passedSuites: 1, - passedTests: 2, - quarantinedSuites: 1, - quarantinedTests: 2, - skippedSuites: 0, - skippedTests: 0, - passedSnapshots: 1, - failedSnapshots: 0, - totalSnapshots: 1, -}; - -export const integrationTest = async (testCase: TestCase): Promise => { - const mockFetch = jest.requireMock< - jest.MockInstance & FetchMockSandbox - >("node-fetch"); - await runTestCase( - { - config: null, - expectedApiKey: "MOCK_API_KEY", - expectedBranch: "MOCK_BRANCH", - expectedCommit: "MOCK_COMMIT", - expectedFailureRetries: 2, - expectedFlakeTestNameSuffix: "", - expectedRepoRelativePathPrefix: "test/integration-input/", - expectedSuiteId: "MOCK_SUITE_ID", - expectPluginToBeEnabled: true, - expectResultsToBeUploaded: true, - expectQuarantinedTestsToBeQuarantined: true, - expectQuarantinedTestsToBeSkipped: false, - expectSnapshots: false, - failToFetchManifest: false, - failToUploadResults: false, - git: { - abbreviatedRefs: { - HEAD: "MOCK_BRANCH", - "refs/heads/MOCK_BRANCH": "MOCK_BRANCH", - }, - refs: [{ sha: "MOCK_COMMIT", refName: "refs/heads/MOCK_BRANCH" }], - commit: "MOCK_COMMIT", - isRepo: true, - // Mock the git repo root as packages/jest-plugin so that we're for sure testing the - // mocked output and not using real git commands. - repoRoot: path.resolve("../.."), - }, - quarantineFlake: false, - skipFailures: false, - skipFlake: false, - skipQuarantined: false, - testNamePattern: undefined, - ...testCase.params, - envVars: { - UNFLAKABLE_API_KEY: "MOCK_API_KEY", - UNFLAKABLE_SUITE_ID: "MOCK_SUITE_ID", - ...testCase.params.envVars, - }, - }, - testCase.expectedExitCode, - testCase.expectedResults, - mockConfigExplorer, - mockExit, - mockFetch - ); -}; - -export const integrationTestSuite = (runTests: () => void): void => { - beforeEach(() => { - (mockConfigExplorer.search as jest.Mock).mockClear(); - mockExit.mockClear(); - - jest - .requireMock("node-fetch") - .reset() - .catch((url, request) => { - throw new Error( - `Unexpected ${ - request.method?.toUpperCase().toString() ?? "undefined" - } request to ${url} ${ - request.body !== null && request.body !== undefined - ? `with body ${request.body.toString()}` - : "without body" - } and headers ${JSON.stringify(request.headers ?? {})}` - ); - }); - - // Don't propagate environment variables from the calling environment to the underlying tests, - // which can lead to different results across environments and leak state between tests that - // manipulate the environment. - process.env = { - NODE_ENV: "test", - FORCE_COLOR: "3", - }; - - let elapsedMs = 0; - Date.now = jest.fn(() => { - const date = - new Date(Date.UTC(2022, 0, 23, 4, 5, 6, 789)).valueOf() + elapsedMs; - elapsedMs += 200; - return date; - }); - - // Restore original. - process.stderr.write = originalStderrWrite; - }); - - const jestMinorVersion = jestPackage.version.match(/^[^.]+\.[^.]+/); - const nodeMajorVersion = process.version.match(/^[^.]+/); - - describe(`Jest ${ - jestMinorVersion !== null ? jestMinorVersion[0] : jestPackage.version - }`, () => { - // Only use Node major version for test name. - describe(`Node ${ - nodeMajorVersion !== null ? nodeMajorVersion[0] : process.version - }`, () => { - runTests(); - }); - }); -}; diff --git a/packages/jest-plugin/test/integration/src/config.test.ts b/packages/jest-plugin/test/integration/src/config.test.ts index 3140496..0f56c1e 100644 --- a/packages/jest-plugin/test/integration/src/config.test.ts +++ b/packages/jest-plugin/test/integration/src/config.test.ts @@ -4,48 +4,60 @@ import { defaultExpectedResults, integrationTest, integrationTestSuite, -} from "./common"; +} from "./test-wrappers"; -integrationTestSuite(() => { - it("set test suite ID via environment", () => - integrationTest({ - params: { - envVars: { - UNFLAKABLE_SUITE_ID: "MOCK_SUITE_ID_ENV", +integrationTestSuite((mockBackend) => { + it("set test suite ID via environment", (done) => + integrationTest( + { + params: { + envVars: { + UNFLAKABLE_SUITE_ID: "MOCK_SUITE_ID_ENV", + }, + expectedSuiteId: "MOCK_SUITE_ID_ENV", }, - expectedSuiteId: "MOCK_SUITE_ID_ENV", + expectedExitCode: 1, + expectedResults: defaultExpectedResults, }, - expectedExitCode: 1, - expectedResults: defaultExpectedResults, - })); + mockBackend, + done + )); - it("set test suite ID via config", () => - integrationTest({ - params: { - config: { - testSuiteId: "MOCK_SUITE_ID_CONFIG", + it("set test suite ID via config", (done) => + integrationTest( + { + params: { + config: { + testSuiteId: "MOCK_SUITE_ID_CONFIG", + }, + envVars: { + UNFLAKABLE_SUITE_ID: undefined, + }, + expectedSuiteId: "MOCK_SUITE_ID_CONFIG", }, - envVars: { - UNFLAKABLE_SUITE_ID: undefined, - }, - expectedSuiteId: "MOCK_SUITE_ID_CONFIG", + expectedExitCode: 1, + expectedResults: defaultExpectedResults, }, - expectedExitCode: 1, - expectedResults: defaultExpectedResults, - })); + mockBackend, + done + )); - it("set test suite ID via environment (override config)", () => - integrationTest({ - params: { - config: { - testSuiteId: "MOCK_SUITE_ID_CONFIG", - }, - envVars: { - UNFLAKABLE_SUITE_ID: "MOCK_SUITE_ID_ENV", + it("set test suite ID via environment (override config)", (done) => + integrationTest( + { + params: { + config: { + testSuiteId: "MOCK_SUITE_ID_CONFIG", + }, + envVars: { + UNFLAKABLE_SUITE_ID: "MOCK_SUITE_ID_ENV", + }, + expectedSuiteId: "MOCK_SUITE_ID_ENV", }, - expectedSuiteId: "MOCK_SUITE_ID_ENV", + expectedExitCode: 1, + expectedResults: defaultExpectedResults, }, - expectedExitCode: 1, - expectedResults: defaultExpectedResults, - })); + mockBackend, + done + )); }); diff --git a/packages/jest-plugin/test/integration/src/disablePlugin.test.ts b/packages/jest-plugin/test/integration/src/disablePlugin.test.ts index 6d2a341..bd42b0e 100644 --- a/packages/jest-plugin/test/integration/src/disablePlugin.test.ts +++ b/packages/jest-plugin/test/integration/src/disablePlugin.test.ts @@ -4,71 +4,83 @@ import { defaultExpectedResults, integrationTest, integrationTestSuite, -} from "./common"; +} from "./test-wrappers"; -integrationTestSuite(() => { - it("disable plugin via config", () => - integrationTest({ - params: { - config: { - enabled: false, +integrationTestSuite((mockBackend) => { + it("disable plugin via config", (done) => + integrationTest( + { + params: { + config: { + enabled: false, + }, + expectPluginToBeEnabled: false, + }, + expectedExitCode: 1, + expectedResults: { + failedSuites: 5, + failedTests: 6, + flakyTests: 0, + passedSuites: 1, + passedTests: 2, + quarantinedSuites: 0, + quarantinedTests: 0, + skippedSuites: 0, + skippedTests: 0, + passedSnapshots: 1, + failedSnapshots: 0, + totalSnapshots: 1, }, - expectPluginToBeEnabled: false, - }, - expectedExitCode: 1, - expectedResults: { - failedSuites: 5, - failedTests: 6, - flakyTests: 0, - passedSuites: 1, - passedTests: 2, - quarantinedSuites: 0, - quarantinedTests: 0, - skippedSuites: 0, - skippedTests: 0, - passedSnapshots: 1, - failedSnapshots: 0, - totalSnapshots: 1, }, - })); + mockBackend, + done + )); - it("disable plugin via environment", () => - integrationTest({ - params: { - envVars: { - UNFLAKABLE_ENABLED: "false", + it("disable plugin via environment", (done) => + integrationTest( + { + params: { + envVars: { + UNFLAKABLE_ENABLED: "false", + }, + expectPluginToBeEnabled: false, + }, + expectedExitCode: 1, + expectedResults: { + failedSuites: 5, + failedTests: 6, + flakyTests: 0, + passedSuites: 1, + passedTests: 2, + quarantinedSuites: 0, + quarantinedTests: 0, + skippedSuites: 0, + skippedTests: 0, + passedSnapshots: 1, + failedSnapshots: 0, + totalSnapshots: 1, }, - expectPluginToBeEnabled: false, - }, - expectedExitCode: 1, - expectedResults: { - failedSuites: 5, - failedTests: 6, - flakyTests: 0, - passedSuites: 1, - passedTests: 2, - quarantinedSuites: 0, - quarantinedTests: 0, - skippedSuites: 0, - skippedTests: 0, - passedSnapshots: 1, - failedSnapshots: 0, - totalSnapshots: 1, }, - })); + mockBackend, + done + )); - it("enable plugin via environment (override config)", () => - integrationTest({ - params: { - config: { - enabled: false, - }, - envVars: { - UNFLAKABLE_ENABLED: "true", + it("enable plugin via environment (override config)", (done) => + integrationTest( + { + params: { + config: { + enabled: false, + }, + envVars: { + UNFLAKABLE_ENABLED: "true", + }, + expectPluginToBeEnabled: true, }, - expectPluginToBeEnabled: true, + expectedExitCode: 1, + expectedResults: defaultExpectedResults, }, - expectedExitCode: 1, - expectedResults: defaultExpectedResults, - })); + mockBackend, + done + )); }); diff --git a/packages/jest-plugin/test/integration/src/disableUpload.test.ts b/packages/jest-plugin/test/integration/src/disableUpload.test.ts index 3e2253e..38706c2 100644 --- a/packages/jest-plugin/test/integration/src/disableUpload.test.ts +++ b/packages/jest-plugin/test/integration/src/disableUpload.test.ts @@ -4,44 +4,56 @@ import { defaultExpectedResults, integrationTest, integrationTestSuite, -} from "./common"; +} from "./test-wrappers"; -integrationTestSuite(() => { - it("disable upload via config", () => - integrationTest({ - params: { - config: { - uploadResults: false, +integrationTestSuite((mockBackend) => { + it("disable upload via config", (done) => + integrationTest( + { + params: { + config: { + uploadResults: false, + }, + expectResultsToBeUploaded: false, }, - expectResultsToBeUploaded: false, + expectedExitCode: 1, + expectedResults: defaultExpectedResults, }, - expectedExitCode: 1, - expectedResults: defaultExpectedResults, - })); + mockBackend, + done + )); - it("disable upload via environment", () => - integrationTest({ - params: { - envVars: { - UNFLAKABLE_UPLOAD_RESULTS: "false", + it("disable upload via environment", (done) => + integrationTest( + { + params: { + envVars: { + UNFLAKABLE_UPLOAD_RESULTS: "false", + }, + expectResultsToBeUploaded: false, }, - expectResultsToBeUploaded: false, + expectedExitCode: 1, + expectedResults: defaultExpectedResults, }, - expectedExitCode: 1, - expectedResults: defaultExpectedResults, - })); + mockBackend, + done + )); - it("enable upload via environment (override config)", () => - integrationTest({ - params: { - config: { - uploadResults: false, - }, - envVars: { - UNFLAKABLE_UPLOAD_RESULTS: "true", + it("enable upload via environment (override config)", (done) => + integrationTest( + { + params: { + config: { + uploadResults: false, + }, + envVars: { + UNFLAKABLE_UPLOAD_RESULTS: "true", + }, }, + expectedExitCode: 1, + expectedResults: defaultExpectedResults, }, - expectedExitCode: 1, - expectedResults: defaultExpectedResults, - })); + mockBackend, + done + )); }); diff --git a/packages/jest-plugin/test/integration/src/git.test.ts b/packages/jest-plugin/test/integration/src/git.test.ts index 2fbd6de..0f8487d 100644 --- a/packages/jest-plugin/test/integration/src/git.test.ts +++ b/packages/jest-plugin/test/integration/src/git.test.ts @@ -1,83 +1,99 @@ // Copyright (c) 2022-2023 Developer Innovations, LLC +import path from "path"; import { defaultExpectedResults, integrationTest, integrationTestSuite, -} from "./common"; -import path from "path"; +} from "./test-wrappers"; -integrationTestSuite(() => { - it("no git repo", () => - integrationTest({ - params: { - expectedBranch: undefined, - expectedCommit: undefined, - // Without a repo, paths are relative to the Jest rootDir. - expectedRepoRelativePathPrefix: "", - git: { - isRepo: false, +integrationTestSuite((mockBackend) => { + it("no git repo", (done) => + integrationTest( + { + params: { + expectedBranch: undefined, + expectedCommit: undefined, + // Without a repo, paths are relative to the Jest rootDir. + expectedRepoRelativePathPrefix: "", + git: { + isRepo: false, + }, }, + expectedExitCode: 1, + expectedResults: defaultExpectedResults, }, - expectedExitCode: 1, - expectedResults: defaultExpectedResults, - })); + mockBackend, + done + )); // This tests the environment present in GitHub Actions for a `pull_request` event. - it("git repo with detached HEAD", () => - integrationTest({ - params: { - git: { - abbreviatedRefs: { - // Mock a detached HEAD. - HEAD: "HEAD", - "refs/remote/pull/MOCK_PR_NUMBER/merge": - "pull/MOCK_PR_NUMBER/merge", - }, - commit: "MOCK_PR_COMMIT", - isRepo: true, - // Mock the `git show-ref` response. - refs: [ - { - sha: "MOCK_PR_COMMIT", - refName: "refs/remote/pull/MOCK_PR_NUMBER/merge", + it("git repo with detached HEAD", (done) => + integrationTest( + { + params: { + git: { + abbreviatedRefs: { + // Mock a detached HEAD. + HEAD: "HEAD", + "refs/remote/pull/MOCK_PR_NUMBER/merge": + "pull/MOCK_PR_NUMBER/merge", }, - ], - repoRoot: path.resolve("../.."), + commit: "MOCK_PR_COMMIT", + isRepo: true, + // Mock the `git show-ref` response. + refs: [ + { + sha: "MOCK_PR_COMMIT", + refName: "refs/remote/pull/MOCK_PR_NUMBER/merge", + }, + ], + repoRoot: path.resolve("../.."), + }, + expectedCommit: "MOCK_PR_COMMIT", + expectedBranch: "pull/MOCK_PR_NUMBER/merge", }, - expectedCommit: "MOCK_PR_COMMIT", - expectedBranch: "pull/MOCK_PR_NUMBER/merge", + expectedExitCode: 1, + expectedResults: defaultExpectedResults, }, - expectedExitCode: 1, - expectedResults: defaultExpectedResults, - })); + mockBackend, + done + )); - it("read branch/commit from environment", () => - integrationTest({ - params: { - envVars: { - UNFLAKABLE_BRANCH: "MOCK_BRANCH2", - UNFLAKABLE_COMMIT: "MOCK_COMMIT2", + it("read branch/commit from environment", (done) => + integrationTest( + { + params: { + envVars: { + UNFLAKABLE_BRANCH: "MOCK_BRANCH2", + UNFLAKABLE_COMMIT: "MOCK_COMMIT2", + }, + expectedBranch: "MOCK_BRANCH2", + expectedCommit: "MOCK_COMMIT2", }, - expectedBranch: "MOCK_BRANCH2", - expectedCommit: "MOCK_COMMIT2", + expectedExitCode: 1, + expectedResults: defaultExpectedResults, }, - expectedExitCode: 1, - expectedResults: defaultExpectedResults, - })); + mockBackend, + done + )); - it("disable git auto-detection", () => - integrationTest({ - params: { - config: { - gitAutoDetect: false, + it("disable git auto-detection", (done) => + integrationTest( + { + params: { + config: { + gitAutoDetect: false, + }, + expectedBranch: undefined, + expectedCommit: undefined, + // Without a repo, paths are relative to the Jest rootDir. + expectedRepoRelativePathPrefix: "", }, - expectedBranch: undefined, - expectedCommit: undefined, - // Without a repo, paths are relative to the Jest rootDir. - expectedRepoRelativePathPrefix: "", + expectedExitCode: 1, + expectedResults: defaultExpectedResults, }, - expectedExitCode: 1, - expectedResults: defaultExpectedResults, - })); + mockBackend, + done + )); }); diff --git a/packages/jest-plugin/test/integration/src/ignoreFailures.test.ts b/packages/jest-plugin/test/integration/src/ignoreFailures.test.ts index d666d16..e2e68c4 100644 --- a/packages/jest-plugin/test/integration/src/ignoreFailures.test.ts +++ b/packages/jest-plugin/test/integration/src/ignoreFailures.test.ts @@ -4,17 +4,21 @@ import { defaultExpectedResults, integrationTest, integrationTestSuite, -} from "./common"; +} from "./test-wrappers"; -integrationTestSuite(() => { - it("set quarantineMode to ignore_failures explicitly", () => - integrationTest({ - params: { - config: { - quarantineMode: "ignore_failures", +integrationTestSuite((mockBackend) => { + it("set quarantineMode to ignore_failures explicitly", (done) => + integrationTest( + { + params: { + config: { + quarantineMode: "ignore_failures", + }, }, + expectedExitCode: 1, + expectedResults: defaultExpectedResults, }, - expectedExitCode: 1, - expectedResults: defaultExpectedResults, - })); + mockBackend, + done + )); }); diff --git a/packages/jest-plugin/test/integration/src/longNames.test.ts b/packages/jest-plugin/test/integration/src/longNames.test.ts index a95265b..e2a4f66 100644 --- a/packages/jest-plugin/test/integration/src/longNames.test.ts +++ b/packages/jest-plugin/test/integration/src/longNames.test.ts @@ -4,45 +4,53 @@ import { defaultExpectedResults, integrationTest, integrationTestSuite, -} from "./common"; +} from "./test-wrappers"; -integrationTestSuite(() => { - it("test names longer than 4096 chars should be truncated", () => - integrationTest({ - params: { - expectedFlakeTestNameSuffix: "*".repeat(4096), - envVars: { - FLAKE_TEST_NAME_SUFFIX: "*".repeat(4096), +integrationTestSuite((mockBackend) => { + it("test names longer than 4096 chars should be truncated", (done) => + integrationTest( + { + params: { + expectedFlakeTestNameSuffix: "*".repeat(4096), + envVars: { + FLAKE_TEST_NAME_SUFFIX: "*".repeat(4096), + }, }, + expectedExitCode: 1, + expectedResults: defaultExpectedResults, }, - expectedExitCode: 1, - expectedResults: defaultExpectedResults, - })); + mockBackend, + done + )); // Include an emoji here for good measure for our dogfooding. - it("quarantining should work for tests with names longer than 4096 chars 😅", () => - integrationTest({ - params: { - expectedFlakeTestNameSuffix: "*".repeat(4096), - envVars: { - FLAKE_TEST_NAME_SUFFIX: "*".repeat(4096), + it("quarantining should work for tests with names longer than 4096 chars 😅", (done) => + integrationTest( + { + params: { + expectedFlakeTestNameSuffix: "*".repeat(4096), + envVars: { + FLAKE_TEST_NAME_SUFFIX: "*".repeat(4096), + }, + quarantineFlake: true, + }, + expectedExitCode: 1, + expectedResults: { + failedSuites: 3, + failedTests: 2, + flakyTests: 0, + passedSuites: 1, + passedTests: 2, + quarantinedSuites: 2, + quarantinedTests: 4, + skippedSuites: 0, + skippedTests: 0, + passedSnapshots: 1, + failedSnapshots: 0, + totalSnapshots: 1, }, - quarantineFlake: true, - }, - expectedExitCode: 1, - expectedResults: { - failedSuites: 3, - failedTests: 2, - flakyTests: 0, - passedSuites: 1, - passedTests: 2, - quarantinedSuites: 2, - quarantinedTests: 4, - skippedSuites: 0, - skippedTests: 0, - passedSnapshots: 1, - failedSnapshots: 0, - totalSnapshots: 1, }, - })); + mockBackend, + done + )); }); diff --git a/packages/jest-plugin/test/integration/src/noQuarantine.test.ts b/packages/jest-plugin/test/integration/src/noQuarantine.test.ts index 379eee0..c397b18 100644 --- a/packages/jest-plugin/test/integration/src/noQuarantine.test.ts +++ b/packages/jest-plugin/test/integration/src/noQuarantine.test.ts @@ -1,30 +1,34 @@ // Copyright (c) 2022-2023 Developer Innovations, LLC -import { integrationTest, integrationTestSuite } from "./common"; +import { integrationTest, integrationTestSuite } from "./test-wrappers"; -integrationTestSuite(() => { - it("set quarantineMode to no_quarantine", () => - integrationTest({ - params: { - config: { - quarantineMode: "no_quarantine", +integrationTestSuite((mockBackend) => { + it("set quarantineMode to no_quarantine", (done) => + integrationTest( + { + params: { + config: { + quarantineMode: "no_quarantine", + }, + expectQuarantinedTestsToBeQuarantined: false, + }, + expectedExitCode: 1, + expectedResults: { + failedSuites: 5, + failedTests: 4, + flakyTests: 2, + passedSuites: 1, + passedTests: 2, + quarantinedSuites: 0, + quarantinedTests: 0, + skippedSuites: 0, + skippedTests: 0, + passedSnapshots: 1, + failedSnapshots: 0, + totalSnapshots: 1, }, - expectQuarantinedTestsToBeQuarantined: false, - }, - expectedExitCode: 1, - expectedResults: { - failedSuites: 5, - failedTests: 4, - flakyTests: 2, - passedSuites: 1, - passedTests: 2, - quarantinedSuites: 0, - quarantinedTests: 0, - skippedSuites: 0, - skippedTests: 0, - passedSnapshots: 1, - failedSnapshots: 0, - totalSnapshots: 1, }, - })); + mockBackend, + done + )); }); diff --git a/packages/jest-plugin/test/integration/src/pluginFailures.test.ts b/packages/jest-plugin/test/integration/src/pluginFailures.test.ts index 32634b8..3b2305c 100644 --- a/packages/jest-plugin/test/integration/src/pluginFailures.test.ts +++ b/packages/jest-plugin/test/integration/src/pluginFailures.test.ts @@ -4,63 +4,75 @@ import { defaultExpectedResults, integrationTest, integrationTestSuite, -} from "./common"; +} from "./test-wrappers"; -integrationTestSuite(() => { - it("run should not fail due to error fetching manifest", () => - integrationTest({ - params: { - failToFetchManifest: true, - skipFailures: true, - skipFlake: true, - skipQuarantined: true, +integrationTestSuite((mockBackend) => { + it("run should not fail due to error fetching manifest", (done) => + integrationTest( + { + params: { + failToFetchManifest: true, + skipFailures: true, + skipFlake: true, + skipQuarantined: true, + }, + expectedExitCode: 0, + expectedResults: { + failedSuites: 0, + failedTests: 0, + flakyTests: 0, + passedSuites: 2, + passedTests: 2, + quarantinedSuites: 0, + quarantinedTests: 0, + skippedSuites: 3, + skippedTests: 6, + passedSnapshots: 0, + failedSnapshots: 0, + totalSnapshots: 0, + }, }, - expectedExitCode: 0, - expectedResults: { - failedSuites: 0, - failedTests: 0, - flakyTests: 0, - passedSuites: 2, - passedTests: 2, - quarantinedSuites: 0, - quarantinedTests: 0, - skippedSuites: 3, - skippedTests: 6, - passedSnapshots: 0, - failedSnapshots: 0, - totalSnapshots: 0, - }, - })); + mockBackend, + done + )); - it("reporter should print results even if upload fails", () => - integrationTest({ - params: { - failToUploadResults: true, + it("reporter should print results even if upload fails", (done) => + integrationTest( + { + params: { + failToUploadResults: true, + }, + expectedExitCode: 1, + expectedResults: defaultExpectedResults, }, - expectedExitCode: 1, - expectedResults: defaultExpectedResults, - })); + mockBackend, + done + )); - it("reporter should print results even if both manifest fetch and upload fail", () => - integrationTest({ - params: { - failToFetchManifest: true, - failToUploadResults: true, - }, - expectedExitCode: 1, - expectedResults: { - failedSuites: 5, - failedTests: 4, - flakyTests: 2, - passedSuites: 1, - passedTests: 2, - quarantinedSuites: 0, - quarantinedTests: 0, - skippedSuites: 0, - skippedTests: 0, - passedSnapshots: 1, - failedSnapshots: 0, - totalSnapshots: 1, + it("reporter should print results even if both manifest fetch and upload fail", (done) => + integrationTest( + { + params: { + failToFetchManifest: true, + failToUploadResults: true, + }, + expectedExitCode: 1, + expectedResults: { + failedSuites: 5, + failedTests: 4, + flakyTests: 2, + passedSuites: 1, + passedTests: 2, + quarantinedSuites: 0, + quarantinedTests: 0, + skippedSuites: 0, + skippedTests: 0, + passedSnapshots: 1, + failedSnapshots: 0, + totalSnapshots: 1, + }, }, - })); + mockBackend, + done + )); }); diff --git a/packages/jest-plugin/test/integration/src/retries.test.ts b/packages/jest-plugin/test/integration/src/retries.test.ts index 10f6c9c..da85f1f 100644 --- a/packages/jest-plugin/test/integration/src/retries.test.ts +++ b/packages/jest-plugin/test/integration/src/retries.test.ts @@ -4,43 +4,51 @@ import { defaultExpectedResults, integrationTest, integrationTestSuite, -} from "./common"; +} from "./test-wrappers"; -integrationTestSuite(() => { - it("disable failure retries", () => - integrationTest({ - params: { - config: { - failureRetries: 0, +integrationTestSuite((mockBackend) => { + it("disable failure retries", (done) => + integrationTest( + { + params: { + config: { + failureRetries: 0, + }, + expectedFailureRetries: 0, + }, + expectedExitCode: 1, + expectedResults: { + failedSuites: 4, + failedTests: 4, + flakyTests: 0, + passedSuites: 1, + passedTests: 2, + quarantinedSuites: 1, + quarantinedTests: 2, + skippedSuites: 0, + skippedTests: 0, + passedSnapshots: 1, + failedSnapshots: 0, + totalSnapshots: 1, }, - expectedFailureRetries: 0, - }, - expectedExitCode: 1, - expectedResults: { - failedSuites: 4, - failedTests: 4, - flakyTests: 0, - passedSuites: 1, - passedTests: 2, - quarantinedSuites: 1, - quarantinedTests: 2, - skippedSuites: 0, - skippedTests: 0, - passedSnapshots: 1, - failedSnapshots: 0, - totalSnapshots: 1, }, - })); + mockBackend, + done + )); - it("set failure retries to 1", () => - integrationTest({ - params: { - config: { - failureRetries: 1, + it("set failure retries to 1", (done) => + integrationTest( + { + params: { + config: { + failureRetries: 1, + }, + expectedFailureRetries: 1, }, - expectedFailureRetries: 1, + expectedExitCode: 1, + expectedResults: defaultExpectedResults, }, - expectedExitCode: 1, - expectedResults: defaultExpectedResults, - })); + mockBackend, + done + )); }); diff --git a/packages/jest-plugin/test/integration/src/runTestCase.ts b/packages/jest-plugin/test/integration/src/runTestCase.ts index e8e7424..ac9fc49 100644 --- a/packages/jest-plugin/test/integration/src/runTestCase.ts +++ b/packages/jest-plugin/test/integration/src/runTestCase.ts @@ -2,34 +2,36 @@ import * as temp from "temp"; import { - FetchMockSandbox, - MockCall, - MockRequest, - MockResponse, - MockMatcher, -} from "fetch-mock"; -import { run } from "jest-cli"; -import escapeStringRegexp from "escape-string-regexp"; -import { - CreateTestSuiteRunFromUploadRequest, CreateTestSuiteRunInlineRequest, TEST_NAME_ENTRY_MAX_LENGTH, - TestAttemptResult, TestRunAttemptRecord, TestRunRecord, TestSuiteManifest, - TestSuiteRunPendingSummary, } from "@unflakable/js-api"; -import { simpleGit, SimpleGit } from "simple-git"; -import type { Response as GitResponse, TaskOptions } from "simple-git"; -import deepEqual from "deep-equal"; -import * as cosmiconfig from "cosmiconfig"; -import { CosmiconfigResult } from "cosmiconfig/dist/types"; import { gunzipSync } from "zlib"; import { UnflakableConfig } from "@unflakable/plugins-common"; -import _debug from "debug"; +import { verifyOutput } from "./verify-output"; +import { + CONFIG_MOCK_ENV_VAR, + CosmiconfigMockParams, +} from "unflakable-test-common/dist/config"; +import path from "path"; +import { promisify } from "util"; +import { execFile } from "child_process"; +import { + MockBackend, + UnmatchedEndpoints, +} from "unflakable-test-common/dist/mock-backend"; +import { CompletedRequest } from "mockttp"; +import { GIT_MOCK_ENV_VAR } from "unflakable-test-common/dist/git"; +import { + AsyncTestError, + spawnTestWithTimeout, +} from "unflakable-test-common/dist/spawn"; -const debug = _debug("unflakable:integration:run-test-case"); +// Jest times out after 40 seconds, so we bail early here to allow time to print the +// captured output before Jest kills the test. +const TEST_TIMEOUT_MS = 30000; const userAgentRegex = new RegExp( "unflakable-js-api/(?:[-0-9.]|alpha|beta)+ unflakable-jest-plugin/(?:[-0-9.]|alpha|beta)+ \\(Jest [0-9]+\\.[0-9]+\\.[0-9]+; Node v[0-9]+\\.[0-9]+\\.[0-9]\\)" @@ -81,101 +83,13 @@ export type TestCaseParams = { testNamePattern: string | undefined; }; -const originalStderrWrite = process.stderr.write.bind(process.stderr); - -const mockSimpleGit = (params: SimpleGitMockParams): void => { - debug("Mocking simple-git with params %o", params); - const mockSimpleGitImpl = { - checkIsRepo: jest.fn( - () => Promise.resolve(params.isRepo) as GitResponse - ), - revparse: jest.fn((options: string | TaskOptions) => { - if (!params.isRepo) { - throw new Error("not a git repository"); - } else if ( - Array.isArray(options) && - options.length === 2 && - options[0] === "--abbrev-ref" - ) { - return Promise.resolve( - params.abbreviatedRefs[options[1]] ?? "HEAD" - ) as GitResponse; - } else if ( - Array.isArray(options) && - options.length === 1 && - options[0] === "--show-toplevel" - ) { - // Treat the current working directory as the repo root. - debug(`Returning mock git repo root ${params.repoRoot}`); - return Promise.resolve(params.repoRoot); - } else if (options === "HEAD") { - return Promise.resolve(params.commit) as GitResponse; - } else { - throw new Error(`unexpected options ${options.toString()}`); - } - }), - raw: jest.fn((options: string | TaskOptions) => { - if (!params.isRepo) { - throw new Error("not a git repository"); - } else if (deepEqual(options, ["show-ref"])) { - return Promise.resolve( - (params.refs ?? []) - .map((mockRef) => `${mockRef.sha} ${mockRef.refName}`) - .join("\n") + "\n" - ) as GitResponse; - } else { - throw new Error(`unexpected options ${options.toString()}`); - } - }), - } as unknown as SimpleGit; - - (simpleGit as jest.Mock).mockImplementation(() => mockSimpleGitImpl); -}; - -// These are the chalk-formatted strings that include console color codes. -const FAIL = - "\u001b[0m\u001b[7m\u001b[1m\u001b[31m FAIL \u001b[39m\u001b[22m\u001b[27m\u001b[0m"; -const PASS = - "\u001b[0m\u001b[7m\u001b[1m\u001b[32m PASS \u001b[39m\u001b[22m\u001b[27m\u001b[0m"; -const QUARANTINED = - "\u001b[0m\u001b[7m\u001b[1m\u001b[33m QUARANTINED \u001b[39m\u001b[22m\u001b[27m\u001b[0m"; -const formatTestFilename = (path: string, filename: string): string => - `\u001b[2m${path}\u001b[22m\u001b[1m${filename}\u001b[22m`; - -const testResultRegexMatch = ( - result: TestAttemptResult | "skipped", - testName: string, - indent?: number -): RegExp => - new RegExp( - `^${" ".repeat(indent ?? 4)}${escapeStringRegexp( - result === "pass" - ? // Green - "\u001b[32m✓\u001b[39m" - : result === "fail" - ? // Red - "\u001b[31m✕\u001b[39m" - : result === "quarantined" - ? // Yellow - "\u001b[33m✕\u001b[39m" - : result === "skipped" - ? // Yellow - "\u001b[33m○\u001b[39m" - : "" - )} \u001b\\[2m${result === "skipped" ? "skipped " : ""}${escapeStringRegexp( - testName - )}${ - result === "quarantined" - ? escapeStringRegexp("\u001b[33m [quarantined]\u001b[39m") - : "" - // Test duration is only included if the test takes at least 1ms. - }( \\([0-9]+ ms\\))?\u001b\\[22m$`, - "" - ); - const specRepoPath = (params: TestCaseParams, specNameStub: string): string => `${params.expectedRepoRelativePathPrefix}src/${specNameStub}.test.ts`; +export const MOCK_RUN_ID = "MOCK_RUN_ID"; +const TIMESTAMP_REGEX = + /^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{3}Z$/; + export type ResultCounts = { passedSuites: number; passedTests: number; @@ -191,304 +105,11 @@ export type ResultCounts = { totalSnapshots: number; }; -const countResults = ({ - expectPluginToBeEnabled, - expectQuarantinedTestsToBeQuarantined, - expectQuarantinedTestsToBeSkipped, - expectedFailureRetries, - expectedFlakeTestNameSuffix, - expectSnapshots, - failToFetchManifest, - quarantineFlake, - skipFailures, - skipFlake, - skipQuarantined, - testNamePattern, -}: TestCaseParams): ResultCounts => { - const flakyTest1ShouldRun = - !skipFlake && - (!expectPluginToBeEnabled || - !quarantineFlake || - !expectQuarantinedTestsToBeSkipped || - failToFetchManifest) && - (testNamePattern === undefined || - `should be flaky 1${expectedFlakeTestNameSuffix}`.match( - testNamePattern - ) !== null); - const flakyTest2ShouldRun = - !skipFlake && - (!expectPluginToBeEnabled || - !quarantineFlake || - !expectQuarantinedTestsToBeSkipped || - failToFetchManifest) && - (testNamePattern === undefined || - `should be flaky 2${expectedFlakeTestNameSuffix}`.match( - testNamePattern - ) !== null); - - const mixedFailTestShouldRun = - !skipFailures && - (testNamePattern === undefined || - "mixed mixed: should fail".match(testNamePattern) !== null); - const mixedQuarantinedTestShouldRun = - !expectQuarantinedTestsToBeSkipped && - !skipQuarantined && - (testNamePattern === undefined || - "mixed mixed: should be quarantined".match(testNamePattern) !== null); - const mixedPassTestShouldRun = - testNamePattern === undefined || - "mixed mixed: should pass".match(testNamePattern) !== null; - - const quarantinedTestShouldRun = - !expectQuarantinedTestsToBeSkipped && - !skipQuarantined && - (testNamePattern === undefined || - "describe block should be quarantined".match(testNamePattern) !== null); - - return [ - // fail.test.ts - { - failedSuites: - (testNamePattern === undefined || - "describe block should ([escape regex]?.*$ fail".match( - testNamePattern - ) !== null) && - !skipFailures - ? 1 - : 0, - failedTests: - (testNamePattern === undefined || - "describe block should ([escape regex]?.*$ fail".match( - testNamePattern - ) !== null) && - !skipFailures - ? 1 - : 0, - flakyTests: 0, - passedSuites: 0, - passedTests: 0, - quarantinedSuites: 0, - quarantinedTests: 0, - skippedSuites: - (testNamePattern !== undefined && - "describe block should ([escape regex]?.*$ fail".match( - testNamePattern - ) === null) || - skipFailures - ? 1 - : 0, - skippedTests: - (testNamePattern !== undefined && - "describe block should ([escape regex]?.*$ fail".match( - testNamePattern - ) === null) || - skipFailures - ? 1 - : 0, - passedSnapshots: 0, - failedSnapshots: expectSnapshots ? 1 : 0, - totalSnapshots: expectSnapshots ? 1 : 0, - }, - // flake.test.ts - { - failedSuites: - (!expectPluginToBeEnabled || !quarantineFlake) && - (flakyTest1ShouldRun || flakyTest2ShouldRun) - ? 1 - : 0, - failedTests: - !expectPluginToBeEnabled || expectedFailureRetries === 0 - ? (flakyTest1ShouldRun ? 1 : 0) + (flakyTest2ShouldRun ? 1 : 0) - : 0, - flakyTests: - expectPluginToBeEnabled && - expectedFailureRetries > 0 && - !quarantineFlake - ? (flakyTest1ShouldRun ? 1 : 0) + (flakyTest2ShouldRun ? 1 : 0) - : 0, - passedSuites: 0, - passedTests: 0, - quarantinedSuites: - expectPluginToBeEnabled && - quarantineFlake && - expectQuarantinedTestsToBeQuarantined && - (flakyTest1ShouldRun || flakyTest2ShouldRun) - ? 1 - : 0, - quarantinedTests: - expectPluginToBeEnabled && - quarantineFlake && - expectQuarantinedTestsToBeQuarantined - ? (flakyTest1ShouldRun ? 1 : 0) + (flakyTest2ShouldRun ? 1 : 0) - : 0, - skippedSuites: !flakyTest1ShouldRun && !flakyTest2ShouldRun ? 1 : 0, - skippedTests: - (!flakyTest1ShouldRun ? 1 : 0) + (!flakyTest2ShouldRun ? 1 : 0), - passedSnapshots: 0, - failedSnapshots: expectSnapshots ? 2 : 0, - totalSnapshots: expectSnapshots ? 2 : 0, - }, - // invalid.test.ts - { - // If skipFailures is enabled, then we exclude the whole file using a path regex. - failedSuites: skipFailures ? 0 : 1, - failedTests: 0, - flakyTests: 0, - passedSuites: 0, - passedTests: 0, - quarantinedSuites: 0, - quarantinedTests: 0, - skippedSuites: 0, - skippedTests: 0, - passedSnapshots: 0, - failedSnapshots: 0, - totalSnapshots: 0, - }, - // mixed.test.ts - { - failedSuites: - ((!expectPluginToBeEnabled || - !expectQuarantinedTestsToBeQuarantined || - failToFetchManifest) && - mixedQuarantinedTestShouldRun) || - mixedFailTestShouldRun - ? 1 - : 0, - failedTests: - ((!expectPluginToBeEnabled || - !expectQuarantinedTestsToBeQuarantined || - failToFetchManifest) && - mixedQuarantinedTestShouldRun - ? 1 - : 0) + (mixedFailTestShouldRun ? 1 : 0), - flakyTests: 0, - passedSuites: - !mixedFailTestShouldRun && - !mixedQuarantinedTestShouldRun && - mixedPassTestShouldRun - ? 1 - : 0, - passedTests: mixedPassTestShouldRun ? 1 : 0, - quarantinedSuites: - expectPluginToBeEnabled && - expectQuarantinedTestsToBeQuarantined && - !failToFetchManifest && - !mixedFailTestShouldRun && - mixedQuarantinedTestShouldRun - ? 1 - : 0, - quarantinedTests: - expectPluginToBeEnabled && - expectQuarantinedTestsToBeQuarantined && - !failToFetchManifest && - mixedQuarantinedTestShouldRun - ? 1 - : 0, - skippedSuites: - !mixedQuarantinedTestShouldRun && - !mixedFailTestShouldRun && - !mixedPassTestShouldRun - ? 1 - : 0, - skippedTests: - (!mixedQuarantinedTestShouldRun ? 1 : 0) + - (!mixedFailTestShouldRun ? 1 : 0) + - (!mixedPassTestShouldRun ? 1 : 0), - passedSnapshots: - !expectSnapshots && mixedQuarantinedTestShouldRun ? 1 : 0, - failedSnapshots: expectSnapshots && mixedQuarantinedTestShouldRun ? 1 : 0, - totalSnapshots: mixedQuarantinedTestShouldRun ? 1 : 0, - }, - // pass.test.ts - { - failedSuites: 0, - failedTests: 0, - flakyTests: 0, - passedSuites: - testNamePattern === undefined || - "should pass".match(testNamePattern) !== null - ? 1 - : 0, - passedTests: - testNamePattern === undefined || - "should pass".match(testNamePattern) !== null - ? 1 - : 0, - quarantinedSuites: 0, - quarantinedTests: 0, - skippedSuites: - testNamePattern !== undefined && - "should pass".match(testNamePattern) === null - ? 1 - : 0, - skippedTests: - testNamePattern !== undefined && - "should pass".match(testNamePattern) === null - ? 1 - : 0, - passedSnapshots: expectSnapshots ? 1 : 0, - failedSnapshots: 0, - totalSnapshots: expectSnapshots ? 1 : 0, - }, - // quarantined.test.ts - { - failedSuites: - (!expectPluginToBeEnabled || - !expectQuarantinedTestsToBeQuarantined || - failToFetchManifest) && - quarantinedTestShouldRun - ? 1 - : 0, - failedTests: - (!expectPluginToBeEnabled || - !expectQuarantinedTestsToBeQuarantined || - failToFetchManifest) && - quarantinedTestShouldRun - ? 1 - : 0, - flakyTests: 0, - passedSuites: 0, - passedTests: 0, - quarantinedSuites: - expectPluginToBeEnabled && - expectQuarantinedTestsToBeQuarantined && - !failToFetchManifest && - quarantinedTestShouldRun - ? 1 - : 0, - quarantinedTests: - expectPluginToBeEnabled && - expectQuarantinedTestsToBeQuarantined && - !failToFetchManifest && - quarantinedTestShouldRun - ? 1 - : 0, - skippedSuites: !quarantinedTestShouldRun ? 1 : 0, - skippedTests: !quarantinedTestShouldRun ? 1 : 0, - passedSnapshots: 0, - failedSnapshots: 0, - totalSnapshots: 0, - }, - ].reduce((a: ResultCounts, b: ResultCounts) => ({ - failedSuites: a.failedSuites + b.failedSuites, - failedTests: a.failedTests + b.failedTests, - flakyTests: a.flakyTests + b.flakyTests, - passedSuites: a.passedSuites + b.passedSuites, - passedTests: a.passedTests + b.passedTests, - quarantinedSuites: a.quarantinedSuites + b.quarantinedSuites, - quarantinedTests: a.quarantinedTests + b.quarantinedTests, - skippedSuites: a.skippedSuites + b.skippedSuites, - skippedTests: a.skippedTests + b.skippedTests, - passedSnapshots: a.passedSnapshots + b.passedSnapshots, - failedSnapshots: a.failedSnapshots + b.failedSnapshots, - totalSnapshots: a.totalSnapshots + b.totalSnapshots, - })); -}; - -const uploadResultsMatcher = ( +const verifyUploadResults = ( params: TestCaseParams, - results: ResultCounts -): MockMatcher => { + expectedResults: ResultCounts, + request: CompletedRequest +): void => { const { expectedBranch, expectedCommit, @@ -503,773 +124,329 @@ const uploadResultsMatcher = ( skipQuarantined, testNamePattern, } = params; - return (_url, { body, headers }) => { - const parsedBody = JSON.parse( - gunzipSync(body as string).toString() - ) as CreateTestSuiteRunInlineRequest; - - expect((headers as { [key in string]: string })["User-Agent"]).toMatch( - userAgentRegex - ); - - const testNamePatternRegex = - testNamePattern !== undefined ? new RegExp(testNamePattern) : undefined; - parsedBody.test_runs.sort((a, b) => - a.filename < b.filename - ? -1 - : a.filename > b.filename - ? 1 - : a.name < b.name - ? -1 - : a.name > b.name - ? 1 - : a < b - ? -1 - : a > b - ? 1 - : 0 - ); + const parsedBody = JSON.parse( + gunzipSync(request.body.buffer).toString() + ) as CreateTestSuiteRunInlineRequest; + + expect(request.headers["user-agent"]).toMatch(userAgentRegex); + + const testNamePatternRegex = + testNamePattern !== undefined ? new RegExp(testNamePattern) : undefined; + + parsedBody.test_runs.sort((a, b) => + a.filename < b.filename + ? -1 + : a.filename > b.filename + ? 1 + : a.name < b.name + ? -1 + : a.name > b.name + ? 1 + : a < b + ? -1 + : a > b + ? 1 + : 0 + ); - expect(parsedBody).toEqual({ - ...(expectedBranch !== undefined - ? { - branch: expectedBranch, - } - : {}), - ...(expectedCommit !== undefined - ? { - commit: expectedCommit, - } - : {}), - start_time: "2022-01-23T04:05:06.789Z", - end_time: expect.stringMatching(/2022-01-23T04:05:..\..89Z/) as string, - test_runs: expect.arrayContaining( - [ - ...(skipFailures - ? ([] as TestRunRecord[]) - : ([ - { - filename: specRepoPath(params, "fail"), - name: ["describe block", "should ([escape regex]?.*$ fail"], - attempts: Array( - expectedFailureRetries + 1 - ).fill({ + expect(parsedBody).toEqual({ + ...(expectedBranch !== undefined + ? { + branch: expectedBranch, + } + : {}), + ...(expectedCommit !== undefined + ? { + commit: expectedCommit, + } + : {}), + start_time: expect.stringMatching(TIMESTAMP_REGEX) as string, + end_time: expect.stringMatching(TIMESTAMP_REGEX) as string, + test_runs: expect.arrayContaining( + [ + ...(skipFailures + ? ([] as TestRunRecord[]) + : ([ + { + filename: specRepoPath(params, "fail"), + name: ["describe block", "should ([escape regex]?.*$ fail"], + attempts: Array( + expectedFailureRetries + 1 + ).fill({ + duration_ms: expect.any(Number) as number, + result: "fail", + }), + }, + ] as TestRunRecord[])), + ...(skipFlake || + (expectQuarantinedTestsToBeSkipped && + quarantineFlake && + !failToFetchManifest) || + (testNamePattern !== undefined && + `should be flaky 1${expectedFlakeTestNameSuffix}`.match( + testNamePattern + ) === null) + ? [] + : [ + { + filename: specRepoPath(params, "flake"), + name: [ + `should be flaky 1${expectedFlakeTestNameSuffix}`.substring( + 0, + TEST_NAME_ENTRY_MAX_LENGTH + ), + ], + attempts: [ + { duration_ms: expect.any(Number) as number, - result: "fail", - }), - }, - ] as TestRunRecord[])), - ...(skipFlake || - (expectQuarantinedTestsToBeSkipped && - quarantineFlake && - !failToFetchManifest) || - (testNamePattern !== undefined && - `should be flaky 1${expectedFlakeTestNameSuffix}`.match( - testNamePattern - ) === null) - ? [] - : [ - { - filename: specRepoPath(params, "flake"), - name: [ - `should be flaky 1${expectedFlakeTestNameSuffix}`.substring( - 0, - TEST_NAME_ENTRY_MAX_LENGTH - ), - ], - attempts: [ - { - duration_ms: expect.any(Number) as number, - result: - quarantineFlake && - !failToFetchManifest && - expectQuarantinedTestsToBeQuarantined - ? "quarantined" - : "fail", - }, - ...(expectedFailureRetries > 0 - ? [ - { - duration_ms: expect.any(Number) as number, - result: "pass", - }, - ] - : []), - ], - } as TestRunRecord, - ]), - ...(skipFlake || - (expectQuarantinedTestsToBeSkipped && - quarantineFlake && - !failToFetchManifest) || - (testNamePattern !== undefined && - `should be flaky 2${expectedFlakeTestNameSuffix}`.match( - testNamePattern - ) === null) - ? [] - : [ - { - filename: specRepoPath(params, "flake"), - name: [ - `should be flaky 2${expectedFlakeTestNameSuffix}`.substring( - 0, - TEST_NAME_ENTRY_MAX_LENGTH - ), - ], - attempts: [ - { - duration_ms: expect.any(Number) as number, - result: - quarantineFlake && - !failToFetchManifest && - expectQuarantinedTestsToBeQuarantined - ? "quarantined" - : "fail", - }, - ...(expectedFailureRetries > 0 - ? [ - { - duration_ms: expect.any(Number) as number, - result: "pass", - }, - ] - : []), - ], - } as TestRunRecord, - ]), - ...(skipQuarantined || - (expectQuarantinedTestsToBeSkipped && !failToFetchManifest) || - (testNamePattern !== undefined && - "mixed mixed: should be quarantined".match(testNamePattern) !== - null) - ? [] - : ([ - { - filename: specRepoPath(params, "mixed"), - name: ["mixed", "mixed: should be quarantined"], - attempts: Array( - expectedFailureRetries + 1 - ).fill({ + result: + quarantineFlake && + !failToFetchManifest && + expectQuarantinedTestsToBeQuarantined + ? "quarantined" + : "fail", + }, + ...(expectedFailureRetries > 0 + ? [ + { + duration_ms: expect.any(Number) as number, + result: "pass", + }, + ] + : []), + ], + } as TestRunRecord, + ]), + ...(skipFlake || + (expectQuarantinedTestsToBeSkipped && + quarantineFlake && + !failToFetchManifest) || + (testNamePattern !== undefined && + `should be flaky 2${expectedFlakeTestNameSuffix}`.match( + testNamePattern + ) === null) + ? [] + : [ + { + filename: specRepoPath(params, "flake"), + name: [ + `should be flaky 2${expectedFlakeTestNameSuffix}`.substring( + 0, + TEST_NAME_ENTRY_MAX_LENGTH + ), + ], + attempts: [ + { duration_ms: expect.any(Number) as number, result: - failToFetchManifest || - !expectQuarantinedTestsToBeQuarantined - ? "fail" - : "quarantined", - }), - }, - ] as TestRunRecord[])), - ...(skipFailures || - (testNamePattern !== undefined && - "mixed mixed: should fail".match(testNamePattern) !== null) - ? [] - : ([ - { - filename: specRepoPath(params, "mixed"), - name: ["mixed", "mixed: should fail"], - attempts: Array( - expectedFailureRetries + 1 - ).fill({ + quarantineFlake && + !failToFetchManifest && + expectQuarantinedTestsToBeQuarantined + ? "quarantined" + : "fail", + }, + ...(expectedFailureRetries > 0 + ? [ + { + duration_ms: expect.any(Number) as number, + result: "pass", + }, + ] + : []), + ], + } as TestRunRecord, + ]), + ...(skipQuarantined || + (expectQuarantinedTestsToBeSkipped && !failToFetchManifest) || + (testNamePattern !== undefined && + "mixed mixed: should be quarantined".match(testNamePattern) !== null) + ? [] + : ([ + { + filename: specRepoPath(params, "mixed"), + name: ["mixed", "mixed: should be quarantined"], + attempts: Array( + expectedFailureRetries + 1 + ).fill({ + duration_ms: expect.any(Number) as number, + result: + failToFetchManifest || + !expectQuarantinedTestsToBeQuarantined + ? "fail" + : "quarantined", + }), + }, + ] as TestRunRecord[])), + ...(skipFailures || + (testNamePattern !== undefined && + "mixed mixed: should fail".match(testNamePattern) !== null) + ? [] + : ([ + { + filename: specRepoPath(params, "mixed"), + name: ["mixed", "mixed: should fail"], + attempts: Array( + expectedFailureRetries + 1 + ).fill({ + duration_ms: expect.any(Number) as number, + result: "fail", + }), + }, + ] as TestRunRecord[])), + ...(testNamePattern === undefined || + "mixed mixed: should pass".match(testNamePattern) === null + ? [ + { + filename: specRepoPath(params, "mixed"), + name: ["mixed", "mixed: should pass"], + attempts: [ + { duration_ms: expect.any(Number) as number, - result: "fail", - }), - }, - ] as TestRunRecord[])), - ...(testNamePattern === undefined || - "mixed mixed: should pass".match(testNamePattern) === null - ? [ - { - filename: specRepoPath(params, "mixed"), - name: ["mixed", "mixed: should pass"], - attempts: [ - { - duration_ms: expect.any(Number) as number, - result: "pass", - }, - ], - } as TestRunRecord, - ] - : []), - ...(testNamePattern === undefined || - "should pass".match(testNamePattern) !== null - ? [ - { - filename: specRepoPath(params, "pass"), - name: ["should pass"], - attempts: [ - { - duration_ms: expect.any(Number) as number, - result: "pass", - }, - ], - } as TestRunRecord, - ] - : []), - ...(skipQuarantined || - (expectQuarantinedTestsToBeSkipped && !failToFetchManifest) || - (testNamePattern !== undefined && - "describe block should be quarantined".match(testNamePattern) === - null) - ? [] - : ([ - { - filename: specRepoPath(params, "quarantined"), - name: ["describe block", "should be quarantined"], - attempts: Array( - expectedFailureRetries + 1 - ).fill({ + result: "pass", + }, + ], + } as TestRunRecord, + ] + : []), + ...(testNamePattern === undefined || + "should pass".match(testNamePattern) !== null + ? [ + { + filename: specRepoPath(params, "pass"), + name: ["should pass"], + attempts: [ + { duration_ms: expect.any(Number) as number, - result: - failToFetchManifest || - !expectQuarantinedTestsToBeQuarantined - ? "fail" - : "quarantined", - }), - }, - ] as TestRunRecord[])), - ].filter( - (runRecord) => - testNamePatternRegex === undefined || - testNamePatternRegex.test(runRecord.name.join(" ")) - ) - ) as TestRunRecord[], - }); - // Make sure there aren't any extra tests reported. - expect(parsedBody.test_runs).toHaveLength( - results.failedTests + - results.flakyTests + - results.quarantinedTests + - results.passedTests - ); - return true; - }; + result: "pass", + }, + ], + } as TestRunRecord, + ] + : []), + ...(skipQuarantined || + (expectQuarantinedTestsToBeSkipped && !failToFetchManifest) || + (testNamePattern !== undefined && + "describe block should be quarantined".match(testNamePattern) === + null) + ? [] + : ([ + { + filename: specRepoPath(params, "quarantined"), + name: ["describe block", "should be quarantined"], + attempts: Array( + expectedFailureRetries + 1 + ).fill({ + duration_ms: expect.any(Number) as number, + result: + failToFetchManifest || + !expectQuarantinedTestsToBeQuarantined + ? "fail" + : "quarantined", + }), + }, + ] as TestRunRecord[])), + ].filter( + (runRecord) => + testNamePatternRegex === undefined || + testNamePatternRegex.test(runRecord.name.join(" ")) + ) + ) as TestRunRecord[], + }); + // Make sure there aren't any extra tests reported. + expect(parsedBody.test_runs).toHaveLength( + expectedResults.failedTests + + expectedResults.flakyTests + + expectedResults.quarantinedTests + + expectedResults.passedTests + ); }; -const addFetchMockExpectations = ( +const addBackendExpectations = async ( params: TestCaseParams, - results: ResultCounts, - fetchMock: jest.MockInstance & FetchMockSandbox -): void => { + expectedResults: ResultCounts, + mockBackend: MockBackend, + onError: (e: unknown) => void +): Promise => { const { expectedApiKey, expectedBranch, expectedCommit, expectedFlakeTestNameSuffix, expectedSuiteId, + expectPluginToBeEnabled, expectResultsToBeUploaded, failToFetchManifest, failToUploadResults, quarantineFlake, } = params; - fetchMock.get( - { - url: `https://app.unflakable.com/api/v1/test-suites/${expectedSuiteId}/manifest`, - headers: { - Authorization: `Bearer ${expectedApiKey}`, - }, - matcher: (_url, { headers }) => { - expect((headers as { [key in string]: string })["User-Agent"]).toMatch( - userAgentRegex - ); - return true; - }, - }, - failToFetchManifest - ? { - throws: new Error("mock request failure"), - } - : { - body: { - quarantined_tests: [ - { - test_id: "TEST_QUARANTINED", - filename: specRepoPath(params, "quarantined"), - name: ["describe block", "should be quarantined"], - }, - { - test_id: "TEST_QUARANTINED2", - filename: specRepoPath(params, "mixed"), - name: ["mixed", "mixed: should be quarantined"], - }, - ...(quarantineFlake - ? [ - { - test_id: "TEST_FLAKE", - filename: specRepoPath(params, "flake"), - name: [ - `should be flaky 1${expectedFlakeTestNameSuffix}`.substring( - 0, - TEST_NAME_ENTRY_MAX_LENGTH - ), - ], - }, - { - test_id: "TEST_FLAKE", - filename: specRepoPath(params, "flake"), - name: [ - `should be flaky 2${expectedFlakeTestNameSuffix}`.substring( - 0, - TEST_NAME_ENTRY_MAX_LENGTH - ), - ], - }, - ] - : []), - ], - } as TestSuiteManifest, - status: 200, - }, - { - repeat: failToFetchManifest ? 3 : 1, - } - ); - if (expectResultsToBeUploaded) { - const uploadUrl = - "https://s3.mock.amazonaws.com/unflakable-backend-mock-test-uploads/teams/MOCK_TEAM_ID/" + - `suites/${expectedSuiteId}/runs/upload/MOCK_UPLOAD_ID?X-Amz-Signature=MOCK_SIGNATURE`; - fetchMock.postOnce( - { - url: `https://app.unflakable.com/api/v1/test-suites/${expectedSuiteId}/runs/upload`, - headers: { - Authorization: `Bearer ${expectedApiKey}`, - "Content-Type": "application/json", - }, - matcher: (_url, { body }) => { - expect(body).toBe(undefined); - return true; - }, - }, - (): MockResponse => ({ - body: { - upload_id: "MOCK_UPLOAD_ID", - }, - headers: { - Location: uploadUrl, - }, - status: 201, - }) - ); - let runRequest: CreateTestSuiteRunInlineRequest | null = null; - fetchMock.putOnce( + const manifest: TestSuiteManifest = { + quarantined_tests: [ { - url: uploadUrl, - headers: { - "Content-Encoding": "gzip", - "Content-Type": "application/json", - }, - matcher: uploadResultsMatcher(params, results), + test_id: "TEST_QUARANTINED", + filename: specRepoPath(params, "quarantined"), + name: ["describe block", "should be quarantined"], }, - (_url: string, { body }: MockRequest): MockResponse => { - runRequest = JSON.parse( - gunzipSync(body as string).toString() - ) as CreateTestSuiteRunInlineRequest; - - return { - status: 200, - }; - } - ); - fetchMock.post( { - url: `https://app.unflakable.com/api/v1/test-suites/${expectedSuiteId}/runs`, - headers: { - Authorization: `Bearer ${expectedApiKey}`, - "Content-Type": "application/json", - }, - matcher: (_url, { body }) => { - const parsedBody = JSON.parse( - body as string - ) as CreateTestSuiteRunFromUploadRequest; - expect(parsedBody.upload_id).toBe("MOCK_UPLOAD_ID"); - return true; - }, + test_id: "TEST_QUARANTINED2", + filename: specRepoPath(params, "mixed"), + name: ["mixed", "mixed: should be quarantined"], }, - (): MockResponse => { - expect(runRequest).not.toBeNull(); - const parsedRequest = runRequest as CreateTestSuiteRunInlineRequest; - - if (failToUploadResults) { - return { - throws: new Error("mock request failure"), - }; - } - - return { - body: { - run_id: "MOCK_RUN_ID", - suite_id: expectedSuiteId, - ...(expectedBranch !== undefined - ? { - branch: expectedBranch, - } - : {}), - ...(expectedCommit !== undefined - ? { - commit: expectedCommit, - } - : {}), - start_time: parsedRequest.start_time, - end_time: parsedRequest.end_time, - num_tests: - results.failedTests + - results.flakyTests + - results.quarantinedTests + - results.passedTests, - num_pass: results.passedTests, - num_fail: results.failedSuites, - num_flake: results.flakyTests, - num_quarantined: results.quarantinedSuites, - } as TestSuiteRunPendingSummary, - status: 201, - }; - }, - { - repeat: failToUploadResults ? 3 : 1, - } - ); - } -}; - -const verifyOutput = ( - { - expectPluginToBeEnabled, - expectQuarantinedTestsToBeQuarantined, - expectQuarantinedTestsToBeSkipped, - expectResultsToBeUploaded, - expectedFailureRetries, - expectedFlakeTestNameSuffix, - expectedSuiteId, - failToFetchManifest, - failToUploadResults, - quarantineFlake, - skipFailures, - skipFlake, - skipQuarantined, - testNamePattern, - }: TestCaseParams, - stderrLines: (Uint8Array | string)[], - results: ResultCounts -): void => { - // Make sure expected output is present and chalk-formatted correctly. - - /* eslint-disable @typescript-eslint/unbound-method */ - - // Test our VerboseReporter customization. - (testNamePattern === undefined || - "should pass".match(testNamePattern) !== null - ? expect(stderrLines).toContain - : expect(stderrLines).not.toContain)( - `${PASS} ${formatTestFilename("../integration-input/src/", "pass.test.ts")}` - ); - (testNamePattern === undefined || - "should pass".match(testNamePattern) !== null - ? expect(stderrLines).toContainEqual - : expect(stderrLines).not.toContainEqual)( - // This test doesn't have a describe() block, so it's only indented 2 spaces. - expect.stringMatching(testResultRegexMatch("pass", "should pass", 2)) - ); - - (!skipFailures && - (testNamePattern === undefined || - "describe block should ([escape regex]?.*$ fail".match( - testNamePattern - ) !== null) - ? expect(stderrLines).toContain - : expect(stderrLines).not.toContain)( - `${FAIL} ${formatTestFilename("../integration-input/src/", "fail.test.ts")}` - ); - (!skipFailures && - (testNamePattern === undefined || - "describe block should ([escape regex]?.*$ fail".match( - testNamePattern - ) !== null) - ? expect(stderrLines).toContainEqual - : expect(stderrLines).not.toContainEqual)( - expect.stringMatching( - testResultRegexMatch("fail", "should ([escape regex]?.*$ fail") - ) - ); - - const flakyTest1Name = `should be flaky 1${expectedFlakeTestNameSuffix}`; - const flakyTest1ShouldRun = - !skipFlake && - (!quarantineFlake || - failToFetchManifest || - !expectQuarantinedTestsToBeSkipped) && - (testNamePattern === undefined || - flakyTest1Name.match(testNamePattern) !== null); - (flakyTest1ShouldRun - ? expect(stderrLines).toContain - : expect(stderrLines).not.toContain)( - `${ - quarantineFlake && - !failToFetchManifest && - expectQuarantinedTestsToBeQuarantined - ? `${QUARANTINED} ` - : "" - }${FAIL} ${formatTestFilename( - "../integration-input/src/", - "flake.test.ts" - )}` - ); - // This test should fail then pass (though we're not verifying the order here). - (flakyTest1ShouldRun - ? expect(stderrLines).toContainEqual - : expect(stderrLines).not.toContainEqual)( - expect.stringMatching( - testResultRegexMatch( - quarantineFlake && !failToFetchManifest ? "quarantined" : "fail", - flakyTest1Name, - 2 - ) - ) - ); - (expectPluginToBeEnabled && expectedFailureRetries > 0 && flakyTest1ShouldRun - ? expect(stderrLines).toContainEqual - : expect(stderrLines).not.toContainEqual)( - expect.stringMatching(testResultRegexMatch("pass", flakyTest1Name, 2)) - ); - - const flakyTest2Name = `should be flaky 2${expectedFlakeTestNameSuffix}`; - const flakyTest2ShouldRun = - !skipFlake && - (!quarantineFlake || - failToFetchManifest || - !expectQuarantinedTestsToBeSkipped) && - (testNamePattern === undefined || - flakyTest2Name.match(testNamePattern) !== null); - (flakyTest2ShouldRun - ? expect(stderrLines).toContainEqual - : expect(stderrLines).not.toContainEqual)( - expect.stringMatching( - testResultRegexMatch( - quarantineFlake && !failToFetchManifest ? "quarantined" : "fail", - flakyTest2Name, - 2 - ) - ) - ); - (expectPluginToBeEnabled && expectedFailureRetries > 0 && flakyTest2ShouldRun - ? expect(stderrLines).toContainEqual - : expect(stderrLines).not.toContainEqual)( - expect.stringMatching(testResultRegexMatch("pass", flakyTest2Name, 2)) - ); - - (!skipQuarantined && - (!expectQuarantinedTestsToBeSkipped || failToFetchManifest) && - (testNamePattern === undefined || - "describe block should be quarantined".match(testNamePattern) !== null) - ? expect(stderrLines).toContain - : expect(stderrLines).not.toContain)( - `${ - expectPluginToBeEnabled && - !failToFetchManifest && - expectQuarantinedTestsToBeQuarantined && - !expectQuarantinedTestsToBeSkipped - ? `${QUARANTINED} ` - : "" - }${FAIL} ${formatTestFilename( - "../integration-input/src/", - "quarantined.test.ts" - )}` - ); - (!skipQuarantined && - (testNamePattern === undefined || - "describe block should be quarantined".match(testNamePattern) !== null) && - !expectQuarantinedTestsToBeSkipped - ? expect(stderrLines).toContainEqual - : expect(stderrLines).not.toContainEqual)( - expect.stringMatching( - testResultRegexMatch( - expectPluginToBeEnabled && - !failToFetchManifest && - expectQuarantinedTestsToBeQuarantined - ? "quarantined" - : "fail", - "should be quarantined" - ) - ) - ); - - const mixedFailTestShouldRun = - !skipFailures && - (testNamePattern === undefined || - "mixed mixed: should fail".match(testNamePattern) !== null); - const mixedQuarantinedTestShouldRun = - !expectQuarantinedTestsToBeSkipped && - !skipQuarantined && - (testNamePattern === undefined || - "mixed mixed: should be quarantined".match(testNamePattern) !== null); - const mixedPassTestShouldRun = - testNamePattern === undefined || - "mixed mixed: should pass".match(testNamePattern) !== null; - - // Mixed file containing both a failed test and a quarantined one. - (((!expectPluginToBeEnabled || - failToFetchManifest || - !expectQuarantinedTestsToBeQuarantined) && - mixedQuarantinedTestShouldRun) || - mixedFailTestShouldRun - ? expect(stderrLines).toContain - : expect(stderrLines).not.toContain)( - `${FAIL} ${formatTestFilename( - "../integration-input/src/", - "mixed.test.ts" - )}` - ); - (mixedQuarantinedTestShouldRun - ? expect(stderrLines).toContainEqual - : expect(stderrLines).not.toContainEqual)( - expect.stringMatching( - testResultRegexMatch( - expectPluginToBeEnabled && - !failToFetchManifest && - expectQuarantinedTestsToBeQuarantined - ? "quarantined" - : "fail", - "mixed: should be quarantined" - ) - ) - ); - (mixedFailTestShouldRun - ? expect(stderrLines).toContainEqual - : expect(stderrLines).not.toContainEqual)( - expect.stringMatching(testResultRegexMatch("fail", "mixed: should fail")) - ); - - expect( - stderrLines.filter((line) => - testResultRegexMatch("pass", "mixed: should pass").test(line as string) - ) - ).toHaveLength(mixedPassTestShouldRun ? 1 : 0); - - // The passed test gets skipped during the retries. - if (mixedFailTestShouldRun || mixedQuarantinedTestShouldRun) { - expect( - stderrLines.filter((line) => - testResultRegexMatch("skipped", "mixed: should pass").test( - line as string - ) - ) - ).toHaveLength( - testNamePattern !== undefined && - "mixed mixed: should pass".match(testNamePattern) === null && - expectPluginToBeEnabled - ? expectedFailureRetries + 1 - : expectPluginToBeEnabled - ? expectedFailureRetries - : testNamePattern !== undefined && - "mixed mixed: should pass".match(testNamePattern) === null - ? 1 - : 0 - ); - } - - // Test our SummaryReporter customization. - expect(stderrLines).toContain( - `\u001b[1mTest Suites: \u001b[22m${ - results.failedSuites !== 0 - ? `\u001b[1m\u001b[31m${results.failedSuites} failed\u001b[39m\u001b[22m, ` - : "" - }${ - results.quarantinedSuites !== 0 - ? `\u001b[1m\u001b[33m${results.quarantinedSuites} quarantined\u001b[39m\u001b[22m, ` - : "" - }${ - results.skippedSuites !== 0 - ? `\u001b[1m\u001b[33m${results.skippedSuites} skipped\u001b[39m\u001b[22m, ` - : "" - }${ - results.passedSuites !== 0 - ? `\u001b[1m\u001b[32m${results.passedSuites} passed\u001b[39m\u001b[22m, ` - : "" - }${ - results.skippedSuites !== 0 - ? `${ - results.failedSuites + - results.quarantinedSuites + - results.passedSuites - } of ${ - results.failedSuites + - results.quarantinedSuites + - results.passedSuites + - results.skippedSuites - }` - : results.failedSuites + - results.quarantinedSuites + - results.passedSuites - } total` - ); - - expect(stderrLines).toContain( - `\u001b[1mTests: \u001b[22m${ - results.failedTests !== 0 - ? `\u001b[1m\u001b[31m${results.failedTests} failed\u001b[39m\u001b[22m, ` - : "" - }${ - results.flakyTests !== 0 - ? `\u001b[1m\u001b[95m${results.flakyTests} flaky\u001b[39m\u001b[22m, ` - : "" - }${ - results.quarantinedTests !== 0 - ? `\u001b[1m\u001b[33m${results.quarantinedTests} quarantined\u001b[39m\u001b[22m, ` - : "" - }${ - results.skippedTests !== 0 - ? `\u001b[1m\u001b[33m${results.skippedTests} skipped\u001b[39m\u001b[22m, ` - : "" - }${ - results.passedTests !== 0 - ? `\u001b[1m\u001b[32m${results.passedTests} passed\u001b[39m\u001b[22m, ` - : "" - }${ - results.failedTests + - results.flakyTests + - results.quarantinedTests + - results.passedTests + - results.skippedTests - } total` - ); - - expect(stderrLines).toContain( - `\u001b[1mSnapshots: \u001b[22m${ - results.failedSnapshots > 0 - ? `\u001b[1m\u001b[31m${results.failedSnapshots} failed\u001b[39m\u001b[22m, ` - : "" - }${ - results.passedSnapshots > 0 - ? `\u001b[1m\u001b[32m${results.passedSnapshots} passed\u001b[39m\u001b[22m, ` - : "" - }${results.totalSnapshots} total` - ); - // None of the snapshots should be obsolete. - expect(stderrLines).not.toContainEqual( - expect.stringMatching(new RegExp("[0-9]+ snapshot(:?s)? obsolete")) - ); - - // The duration here is based on the mocked time, so it should be deterministic. - expect(stderrLines).toContainEqual( - expect.stringMatching( - new RegExp( - `${escapeStringRegexp("\u001b[1mTime:\u001b[22m ")}[0-9.]+ s` - ) - ) - ); - - expect(stderrLines).toContain( - `\u001b[2mRan all test suites\u001b[22m\u001b[2m${ - testNamePattern !== undefined - ? ` with tests matching \u001b[22m"${testNamePattern}"\u001b[2m` - : "" - }.\u001b[22m` - ); + ...(quarantineFlake + ? [ + { + test_id: "TEST_FLAKE", + filename: specRepoPath(params, "flake"), + name: [ + `should be flaky 1${expectedFlakeTestNameSuffix}`.substring( + 0, + TEST_NAME_ENTRY_MAX_LENGTH + ), + ], + }, + { + test_id: "TEST_FLAKE", + filename: specRepoPath(params, "flake"), + name: [ + `should be flaky 2${expectedFlakeTestNameSuffix}`.substring( + 0, + TEST_NAME_ENTRY_MAX_LENGTH + ), + ], + }, + ] + : []), + ], + }; - (expectPluginToBeEnabled && expectResultsToBeUploaded && !failToUploadResults - ? expect(stderrLines).toContain - : expect(stderrLines).not.toContain)( - `Unflakable report: https://app.unflakable.com/test-suites/${expectedSuiteId}/runs/MOCK_RUN_ID` + return mockBackend.addExpectations( + onError, + failToFetchManifest ? null : manifest, + (request) => verifyUploadResults(params, expectedResults, request), + failToUploadResults + ? null + : { + run_id: MOCK_RUN_ID, + suite_id: expectedSuiteId, + ...(expectedBranch !== undefined + ? { + branch: expectedBranch, + } + : {}), + ...(expectedCommit !== undefined + ? { + commit: expectedCommit, + } + : {}), + }, + userAgentRegex, + { + expectPluginToBeEnabled, + expectResultsToBeUploaded, + expectedApiKey, + expectedSuiteId, + } ); }; @@ -1277,134 +454,111 @@ export const runTestCase = async ( params: TestCaseParams, expectedExitCode: number, expectedResults: ResultCounts, - mockConfigExplorer: ReturnType, - mockExit: jest.Mock, - fetchMock: jest.MockInstance & FetchMockSandbox + mockBackend: MockBackend ): Promise => { - const { - expectPluginToBeEnabled, - failToUploadResults, - git, - skipFailures, - skipFlake, - skipQuarantined, - testNamePattern, - } = params; + const { git, skipFailures, skipFlake, skipQuarantined, testNamePattern } = + params; + + const asyncTestError: AsyncTestError = { error: undefined }; + + const unmatchedRequestEndpoints = await addBackendExpectations( + params, + expectedResults, + mockBackend, + (error) => { + if (asyncTestError.error === undefined) { + asyncTestError.error = error ?? new Error("undefined error"); + } else { + console.error("Multiple failed fetch expectations", error); + } + } + ); - (mockConfigExplorer.search as jest.Mock).mockImplementation( - (searchFrom?: string): CosmiconfigResult => { - expect(searchFrom).toMatch( - new RegExp("packages/jest-plugin/test/integration-input$") - ); - return params.config !== null + const integrationInputPath = path.join("..", "integration-input"); + const configMockParams: CosmiconfigMockParams = { + searchFrom: path.resolve(integrationInputPath), + searchResult: + params.config !== null ? { config: params.config, - filepath: - "MOCK_BASE/packages/jest-plugin/test/integration-input/package.json", + filepath: "MOCK_BASE/packages/jest-plugin/test/unflakable.yml", } - : null; - } - ); - - mockSimpleGit(git); - - const results = countResults(params); - - // The flaky test needs external state to know when it's being retried so that it can pass. - process.env.FLAKY_TEST_TEMP = temp.path(); - - if (skipFailures) { - process.env.SKIP_FAILURES = "true"; - } else { - delete process.env.SKIP_FAILURES; - } - - if (skipFlake) { - process.env.SKIP_FLAKE = "true"; - } else { - delete process.env.SKIP_FLAKE; - } - - if (skipQuarantined) { - process.env.SKIP_QUARANTINED = "true"; - } else { - delete process.env.SKIP_QUARANTINED; - } - - Object.entries(params.envVars).forEach(([name, value]) => { - if (value !== undefined) { - process.env[name] = value; - } else { - delete process.env[name]; - } - }); - - if (expectPluginToBeEnabled) { - addFetchMockExpectations(params, results, fetchMock); - } - - let stderrLines: (Uint8Array | string)[] = []; - process.stderr.write = jest.fn( - ( - buffer: Uint8Array | string, - encoding?: BufferEncoding, - cb?: (err?: Error) => void - ): boolean => { - stderrLines = [ - ...stderrLines, - ...(typeof buffer === "string" - ? buffer.split("\n") - : [JSON.stringify(buffer)]), - ]; - // split() adds an empty string if the delimiter is the last character; remove it here. - if (stderrLines[stderrLines.length - 1] === "") { - stderrLines.pop(); - } - //originalStderrWrite(JSON.stringify(buffer)); - return originalStderrWrite(buffer, encoding, cb); - } - ) as typeof process.stderr.write; - - try { - const runPromise = run( - [ - // Needed so that exit() gets called and we can assert that it's the correct exit code. - // NB: Despite the warning, we don't pass --detectOpenHandles since that would also enable - // --runInBand and not test the plugin's ability to deal with parallel tests. - "--forceExit", - // --no-cache disables the cache that stores the past timings, which makes the output - // non-deterministic since it gets bolded if it exceeds the expected time. - "--no-cache", - "--reporters", - "@unflakable/jest-plugin/dist/reporter", - "--runner", - "@unflakable/jest-plugin/dist/runner", - ...(skipFailures - ? ["--testPathIgnorePatterns", "/src/invalid\\.test\\.ts"] - : []), - ...(testNamePattern !== undefined - ? ["--testNamePattern", testNamePattern] - : []), - ], - "../integration-input" - ); - - if (failToUploadResults) { - await expect(runPromise).rejects.toThrow(); - } else { - await runPromise; - } - } finally { - process.stderr.write = originalStderrWrite; - } - - // There shouldn't be any unexpected mock fetch calls. - expect(fetchMock.calls("unmatched")).toHaveLength(0); - expect(fetchMock).toBeDone(); + : null, + }; - expect(results).toEqual(expectedResults); - verifyOutput(params, stderrLines, results); + // We don't directly invoke `jest` because we need to pass `--require` to Node.JS in order to + // mock cosmiconfig for testing. Instead, we resolve the binary to an absolute path using `yarn + // bin` and then invoke node directly. + const jestBin = ( + await promisify(execFile)("yarn", ["bin", "jest"], { + cwd: integrationInputPath, + // yarn.CMD isn't executable without a shell on Windows. + shell: process.platform === "win32", + }) + ).stdout.trimEnd(); + + const args = [ + "--require", + require.resolve("./force-color.js"), + "--require", + require.resolve("unflakable-test-common/dist/mock-cosmiconfig"), + "--require", + require.resolve("unflakable-test-common/dist/mock-git"), + jestBin, + // --no-cache disables the cache that stores the past timings, which makes the output + // non-deterministic since it gets bolded if it exceeds the expected time. + "--no-cache", + "--reporters", + "@unflakable/jest-plugin/dist/reporter", + "--runner", + "@unflakable/jest-plugin/dist/runner", + ...(skipFailures + ? ["--testPathIgnorePatterns", "/src/invalid\\.test\\.ts"] + : []), + ...(testNamePattern !== undefined + ? ["--testNamePattern", testNamePattern] + : []), + ]; + + const env = { + ...params.envVars, + DEBUG: process.env.TEST_DEBUG, + // The flaky test needs external state to know when it's being retried so that it can pass. + FLAKY_TEST_TEMP: temp.path(), + PATH: process.env.PATH, + UNFLAKABLE_API_BASE_URL: `http://localhost:${mockBackend.apiServerPort}`, + [CONFIG_MOCK_ENV_VAR]: JSON.stringify(configMockParams), + [GIT_MOCK_ENV_VAR]: JSON.stringify(git), + ...(skipFailures ? { SKIP_FAILURES: "1" } : {}), + ...(skipFlake ? { SKIP_FLAKE: "1" } : {}), + ...(skipQuarantined ? { SKIP_QUARANTINED: "1" } : {}), + // Windows requires these environment variables to be propagated. + ...(process.platform === "win32" + ? { + APPDATA: process.env.APPDATA, + LOCALAPPDATA: process.env.LOCALAPPDATA, + TMP: process.env.TMP, + TEMP: process.env.TEMP, + } + : {}), + }; - expect(mockExit).toHaveBeenCalledTimes(1); - expect(mockExit).toHaveBeenCalledWith(expectedExitCode); + await spawnTestWithTimeout( + args, + env, + integrationInputPath, + TEST_TIMEOUT_MS, + async (_stdoutLines, stderrLines) => { + verifyOutput( + params, + stderrLines, + expectedResults, + mockBackend.apiServerPort + ); + await mockBackend.checkExpectations(unmatchedRequestEndpoints); + }, + expectedExitCode, + true, + asyncTestError + ); }; diff --git a/packages/jest-plugin/test/integration/src/skipTests.test.ts b/packages/jest-plugin/test/integration/src/skipTests.test.ts index 580bd08..1302612 100644 --- a/packages/jest-plugin/test/integration/src/skipTests.test.ts +++ b/packages/jest-plugin/test/integration/src/skipTests.test.ts @@ -1,117 +1,133 @@ // Copyright (c) 2022-2023 Developer Innovations, LLC -import { integrationTest, integrationTestSuite } from "./common"; +import { integrationTest, integrationTestSuite } from "./test-wrappers"; -integrationTestSuite(() => { - it("set quarantineMode to skip_tests", () => - integrationTest({ - params: { - config: { - quarantineMode: "skip_tests", +integrationTestSuite((mockBackend) => { + it("set quarantineMode to skip_tests", (done) => + integrationTest( + { + params: { + config: { + quarantineMode: "skip_tests", + }, + expectQuarantinedTestsToBeSkipped: true, + }, + expectedExitCode: 1, + expectedResults: { + failedSuites: 4, + failedTests: 2, + flakyTests: 2, + passedSuites: 1, + passedTests: 2, + quarantinedSuites: 0, + quarantinedTests: 0, + skippedSuites: 1, + skippedTests: 2, + passedSnapshots: 0, + failedSnapshots: 0, + totalSnapshots: 0, }, - expectQuarantinedTestsToBeSkipped: true, - }, - expectedExitCode: 1, - expectedResults: { - failedSuites: 4, - failedTests: 2, - flakyTests: 2, - passedSuites: 1, - passedTests: 2, - quarantinedSuites: 0, - quarantinedTests: 0, - skippedSuites: 1, - skippedTests: 2, - passedSnapshots: 0, - failedSnapshots: 0, - totalSnapshots: 0, }, - })); + mockBackend, + done + )); - it("test names longer than 4096 chars should be truncated w/ quarantineMode set to skip_tests", () => - integrationTest({ - params: { - config: { - quarantineMode: "skip_tests", + it("test names longer than 4096 chars should be truncated w/ quarantineMode set to skip_tests", (done) => + integrationTest( + { + params: { + config: { + quarantineMode: "skip_tests", + }, + envVars: { + FLAKE_TEST_NAME_SUFFIX: "*".repeat(4096), + }, + expectedFlakeTestNameSuffix: "*".repeat(4096), + expectQuarantinedTestsToBeSkipped: true, }, - envVars: { - FLAKE_TEST_NAME_SUFFIX: "*".repeat(4096), + expectedExitCode: 1, + expectedResults: { + failedSuites: 4, + failedTests: 2, + flakyTests: 2, + passedSuites: 1, + passedTests: 2, + quarantinedSuites: 0, + quarantinedTests: 0, + skippedSuites: 1, + skippedTests: 2, + passedSnapshots: 0, + failedSnapshots: 0, + totalSnapshots: 0, }, - expectedFlakeTestNameSuffix: "*".repeat(4096), - expectQuarantinedTestsToBeSkipped: true, - }, - expectedExitCode: 1, - expectedResults: { - failedSuites: 4, - failedTests: 2, - flakyTests: 2, - passedSuites: 1, - passedTests: 2, - quarantinedSuites: 0, - quarantinedTests: 0, - skippedSuites: 1, - skippedTests: 2, - passedSnapshots: 0, - failedSnapshots: 0, - totalSnapshots: 0, }, - })); + mockBackend, + done + )); - it("quarantining should work for tests with names longer than 4096 chars w/ quarantineMode set to skip_tests", () => - integrationTest({ - params: { - config: { - quarantineMode: "skip_tests", + it("quarantining should work for tests with names longer than 4096 chars w/ quarantineMode set to skip_tests", (done) => + integrationTest( + { + params: { + config: { + quarantineMode: "skip_tests", + }, + envVars: { + FLAKE_TEST_NAME_SUFFIX: "*".repeat(4096), + }, + expectedFlakeTestNameSuffix: "*".repeat(4096), + expectQuarantinedTestsToBeSkipped: true, + quarantineFlake: true, }, - envVars: { - FLAKE_TEST_NAME_SUFFIX: "*".repeat(4096), + expectedExitCode: 1, + expectedResults: { + failedSuites: 3, + failedTests: 2, + flakyTests: 0, + passedSuites: 1, + passedTests: 2, + quarantinedSuites: 0, + quarantinedTests: 0, + skippedSuites: 2, + skippedTests: 4, + passedSnapshots: 0, + failedSnapshots: 0, + totalSnapshots: 0, }, - expectedFlakeTestNameSuffix: "*".repeat(4096), - expectQuarantinedTestsToBeSkipped: true, - quarantineFlake: true, }, - expectedExitCode: 1, - expectedResults: { - failedSuites: 3, - failedTests: 2, - flakyTests: 0, - passedSuites: 1, - passedTests: 2, - quarantinedSuites: 0, - quarantinedTests: 0, - skippedSuites: 2, - skippedTests: 4, - passedSnapshots: 0, - failedSnapshots: 0, - totalSnapshots: 0, - }, - })); + mockBackend, + done + )); - it("set quarantineMode to skip_tests and --testNamePattern that skips all other tests", () => - integrationTest({ - params: { - config: { - quarantineMode: "skip_tests", + it("set quarantineMode to skip_tests and --testNamePattern that skips all other tests", (done) => + integrationTest( + { + params: { + config: { + quarantineMode: "skip_tests", + }, + expectQuarantinedTestsToBeSkipped: true, + expectResultsToBeUploaded: false, + skipFailures: true, + testNamePattern: "should be quarantined", + }, + expectedExitCode: 0, + expectedResults: { + failedSuites: 0, + failedTests: 0, + flakyTests: 0, + passedSuites: 0, + passedTests: 0, + quarantinedSuites: 0, + quarantinedTests: 0, + skippedSuites: 5, + skippedTests: 8, + passedSnapshots: 0, + failedSnapshots: 0, + totalSnapshots: 0, }, - expectQuarantinedTestsToBeSkipped: true, - expectResultsToBeUploaded: false, - skipFailures: true, - testNamePattern: "should be quarantined", - }, - expectedExitCode: 0, - expectedResults: { - failedSuites: 0, - failedTests: 0, - flakyTests: 0, - passedSuites: 0, - passedTests: 0, - quarantinedSuites: 0, - quarantinedTests: 0, - skippedSuites: 5, - skippedTests: 8, - passedSnapshots: 0, - failedSnapshots: 0, - totalSnapshots: 0, }, - })); + mockBackend, + done + )); }); diff --git a/packages/jest-plugin/test/integration/src/snapshots.test.ts b/packages/jest-plugin/test/integration/src/snapshots.test.ts index 2748bd8..7c4948c 100644 --- a/packages/jest-plugin/test/integration/src/snapshots.test.ts +++ b/packages/jest-plugin/test/integration/src/snapshots.test.ts @@ -4,23 +4,27 @@ import { defaultExpectedResults, integrationTest, integrationTestSuite, -} from "./common"; +} from "./test-wrappers"; -integrationTestSuite(() => { - it("use snapshot tests", () => - integrationTest({ - params: { - envVars: { - TEST_SNAPSHOTS: "true", +integrationTestSuite((mockBackend) => { + it("use snapshot tests", (done) => + integrationTest( + { + params: { + envVars: { + TEST_SNAPSHOTS: "true", + }, + expectSnapshots: true, + }, + expectedExitCode: 1, + expectedResults: { + ...defaultExpectedResults, + passedSnapshots: 1, + failedSnapshots: 4, + totalSnapshots: 5, }, - expectSnapshots: true, - }, - expectedExitCode: 1, - expectedResults: { - ...defaultExpectedResults, - passedSnapshots: 1, - failedSnapshots: 4, - totalSnapshots: 5, }, - })); + mockBackend, + done + )); }); diff --git a/packages/jest-plugin/test/integration/src/test-wrappers.ts b/packages/jest-plugin/test/integration/src/test-wrappers.ts new file mode 100644 index 0000000..243e4a9 --- /dev/null +++ b/packages/jest-plugin/test/integration/src/test-wrappers.ts @@ -0,0 +1,108 @@ +// Copyright (c) 2023 Developer Innovations, LLC + +import jestPackage from "jest/package.json"; +import path from "path"; +import { ResultCounts, runTestCase, TestCaseParams } from "./runTestCase"; +import { MockBackend } from "unflakable-test-common/dist/mock-backend"; +import * as util from "util"; + +export type TestCase = { + params: Partial; + expectedExitCode: number; + expectedResults: ResultCounts; +}; + +export const defaultExpectedResults: ResultCounts = { + failedSuites: 4, + failedTests: 2, + flakyTests: 2, + passedSuites: 1, + passedTests: 2, + quarantinedSuites: 1, + quarantinedTests: 2, + skippedSuites: 0, + skippedTests: 0, + passedSnapshots: 1, + failedSnapshots: 0, + totalSnapshots: 1, +}; + +export const integrationTest = ( + testCase: TestCase, + mockBackend: MockBackend, + done: jest.DoneCallback +): void => { + void runTestCase( + { + config: null, + expectedApiKey: "MOCK_API_KEY", + expectedBranch: "MOCK_BRANCH", + expectedCommit: "MOCK_COMMIT", + expectedFailureRetries: 2, + expectedFlakeTestNameSuffix: "", + expectedRepoRelativePathPrefix: "test/integration-input/", + expectedSuiteId: "MOCK_SUITE_ID", + expectPluginToBeEnabled: true, + expectResultsToBeUploaded: true, + expectQuarantinedTestsToBeQuarantined: true, + expectQuarantinedTestsToBeSkipped: false, + expectSnapshots: false, + failToFetchManifest: false, + failToUploadResults: false, + git: { + abbreviatedRefs: { + HEAD: "MOCK_BRANCH", + "refs/heads/MOCK_BRANCH": "MOCK_BRANCH", + }, + refs: [{ sha: "MOCK_COMMIT", refName: "refs/heads/MOCK_BRANCH" }], + commit: "MOCK_COMMIT", + isRepo: true, + // Mock the git repo root as packages/jest-plugin so that we're for sure testing the + // mocked output and not using real git commands. + repoRoot: path.resolve("../.."), + }, + quarantineFlake: false, + skipFailures: false, + skipFlake: false, + skipQuarantined: false, + testNamePattern: undefined, + ...testCase.params, + envVars: { + UNFLAKABLE_API_KEY: "MOCK_API_KEY", + UNFLAKABLE_SUITE_ID: "MOCK_SUITE_ID", + ...testCase.params.envVars, + }, + }, + testCase.expectedExitCode, + testCase.expectedResults, + mockBackend + ) + .then(done) + .catch((e) => { + // Ensures any chained `cause` gets printed. + done(util.inspect(e, { colors: true, depth: 5 })); + }); +}; + +export const integrationTestSuite = ( + runTests: (mockBackend: MockBackend) => void +): void => { + const mockBackend = new MockBackend(); + + beforeEach(() => mockBackend.start()); + afterEach(() => mockBackend.stop()); + + const jestMinorVersion = jestPackage.version.match(/^[^.]+\.[^.]+/); + const nodeMajorVersion = process.version.match(/^[^.]+/); + + describe(`Jest ${ + jestMinorVersion !== null ? jestMinorVersion[0] : jestPackage.version + }`, () => { + // Only use Node major version for test name. + describe(`Node ${ + nodeMajorVersion !== null ? nodeMajorVersion[0] : process.version + }`, () => { + runTests(mockBackend); + }); + }); +}; diff --git a/packages/jest-plugin/test/integration/src/testNamePattern.test.ts b/packages/jest-plugin/test/integration/src/testNamePattern.test.ts index 1669ba7..cd39f5e 100644 --- a/packages/jest-plugin/test/integration/src/testNamePattern.test.ts +++ b/packages/jest-plugin/test/integration/src/testNamePattern.test.ts @@ -4,87 +4,103 @@ import { defaultExpectedResults, integrationTest, integrationTestSuite, -} from "./common"; +} from "./test-wrappers"; -integrationTestSuite(() => { - it("use --testNamePattern to have Jest filter tests", () => - integrationTest({ - params: { - testNamePattern: "(should .*fail|should be flaky)", - }, - expectedExitCode: 1, - expectedResults: { - failedSuites: 4, - failedTests: 2, - flakyTests: 2, - passedSuites: 0, - passedTests: 0, - quarantinedSuites: 0, - quarantinedTests: 0, - skippedSuites: 2, - skippedTests: 4, - passedSnapshots: 0, - failedSnapshots: 0, - totalSnapshots: 0, +integrationTestSuite((mockBackend) => { + it("use --testNamePattern to have Jest filter tests", (done) => + integrationTest( + { + params: { + testNamePattern: "(should .*fail|should be flaky)", + }, + expectedExitCode: 1, + expectedResults: { + failedSuites: 4, + failedTests: 2, + flakyTests: 2, + passedSuites: 0, + passedTests: 0, + quarantinedSuites: 0, + quarantinedTests: 0, + skippedSuites: 2, + skippedTests: 4, + passedSnapshots: 0, + failedSnapshots: 0, + totalSnapshots: 0, + }, }, - })); + mockBackend, + done + )); - it("use --testNamePattern with no matches", () => - integrationTest({ - params: { - expectResultsToBeUploaded: false, - skipFailures: true, - testNamePattern: "no matches", - }, - expectedExitCode: 0, - expectedResults: { - failedSuites: 0, - failedTests: 0, - flakyTests: 0, - passedSuites: 0, - passedTests: 0, - quarantinedSuites: 0, - quarantinedTests: 0, - skippedSuites: 5, - skippedTests: 8, - passedSnapshots: 0, - failedSnapshots: 0, - totalSnapshots: 0, + it("use --testNamePattern with no matches", (done) => + integrationTest( + { + params: { + expectResultsToBeUploaded: false, + skipFailures: true, + testNamePattern: "no matches", + }, + expectedExitCode: 0, + expectedResults: { + failedSuites: 0, + failedTests: 0, + flakyTests: 0, + passedSuites: 0, + passedTests: 0, + quarantinedSuites: 0, + quarantinedTests: 0, + skippedSuites: 5, + skippedTests: 8, + passedSnapshots: 0, + failedSnapshots: 0, + totalSnapshots: 0, + }, }, - })); + mockBackend, + done + )); - it("use a permissive --testNamePattern and ensure only failed tests are retried", () => - integrationTest({ - params: { - testNamePattern: ".*", + it("use a permissive --testNamePattern and ensure only failed tests are retried", (done) => + integrationTest( + { + params: { + testNamePattern: ".*", + }, + expectedExitCode: 1, + expectedResults: defaultExpectedResults, }, - expectedExitCode: 1, - expectedResults: defaultExpectedResults, - })); + mockBackend, + done + )); - it("use --testNamePattern with plugin disabled", () => - integrationTest({ - params: { - envVars: { - UNFLAKABLE_ENABLED: "false", + it("use --testNamePattern with plugin disabled", (done) => + integrationTest( + { + params: { + envVars: { + UNFLAKABLE_ENABLED: "false", + }, + expectPluginToBeEnabled: false, + testNamePattern: "(should .*fail|should be flaky)", + }, + expectedExitCode: 1, + expectedResults: { + failedSuites: 4, + failedTests: 4, + flakyTests: 0, + passedSuites: 0, + passedTests: 0, + quarantinedSuites: 0, + quarantinedTests: 0, + skippedSuites: 2, + skippedTests: 4, + passedSnapshots: 0, + failedSnapshots: 0, + totalSnapshots: 0, }, - expectPluginToBeEnabled: false, - testNamePattern: "(should .*fail|should be flaky)", - }, - expectedExitCode: 1, - expectedResults: { - failedSuites: 4, - failedTests: 4, - flakyTests: 0, - passedSuites: 0, - passedTests: 0, - quarantinedSuites: 0, - quarantinedTests: 0, - skippedSuites: 2, - skippedTests: 4, - passedSnapshots: 0, - failedSnapshots: 0, - totalSnapshots: 0, }, - })); + mockBackend, + done + )); }); diff --git a/packages/jest-plugin/test/integration/src/verify-output.ts b/packages/jest-plugin/test/integration/src/verify-output.ts new file mode 100644 index 0000000..bbcb461 --- /dev/null +++ b/packages/jest-plugin/test/integration/src/verify-output.ts @@ -0,0 +1,379 @@ +// Copyright (c) 2023 Developer Innovations, LLC + +// These are the chalk-formatted strings that include console color codes. +import escapeStringRegexp from "escape-string-regexp"; +import { MOCK_RUN_ID, ResultCounts, TestCaseParams } from "./runTestCase"; +import { TestAttemptResult } from "@unflakable/js-api"; + +const FAIL = + "\u001b[0m\u001b[7m\u001b[1m\u001b[31m FAIL \u001b[39m\u001b[22m\u001b[27m\u001b[0m"; +const PASS = + "\u001b[0m\u001b[7m\u001b[1m\u001b[32m PASS \u001b[39m\u001b[22m\u001b[27m\u001b[0m"; +const QUARANTINED = + "\u001b[0m\u001b[7m\u001b[1m\u001b[33m QUARANTINED \u001b[39m\u001b[22m\u001b[27m\u001b[0m"; +const formatTestFilename = (path: string, filename: string): string => + `\u001b[2m${path}\u001b[22m\u001b[1m${filename}\u001b[22m`; + +const testResultRegexMatch = ( + result: TestAttemptResult | "skipped", + testName: string, + indent?: number +): RegExp => + new RegExp( + `^${" ".repeat(indent ?? 4)}${escapeStringRegexp( + result === "pass" + ? // Green + "\u001b[32m✓\u001b[39m" + : result === "fail" + ? // Red + "\u001b[31m✕\u001b[39m" + : result === "quarantined" + ? // Yellow + "\u001b[33m✕\u001b[39m" + : result === "skipped" + ? // Yellow + "\u001b[33m○\u001b[39m" + : "" + )} \u001b\\[2m${result === "skipped" ? "skipped " : ""}${escapeStringRegexp( + testName + )}${ + result === "quarantined" + ? escapeStringRegexp("\u001b[33m [quarantined]\u001b[39m") + : "" + // Test duration is only included if the test takes at least 1ms. + }( \\([0-9]+ ms\\))?\u001b\\[22m$`, + "" + ); + +export const verifyOutput = ( + { + expectPluginToBeEnabled, + expectQuarantinedTestsToBeQuarantined, + expectQuarantinedTestsToBeSkipped, + expectResultsToBeUploaded, + expectedFailureRetries, + expectedFlakeTestNameSuffix, + expectedSuiteId, + failToFetchManifest, + failToUploadResults, + quarantineFlake, + skipFailures, + skipFlake, + skipQuarantined, + testNamePattern, + }: TestCaseParams, + stderrLines: (Uint8Array | string)[], + expectedResults: ResultCounts, + apiServerPort: number +): void => { + // Make sure expected output is present and chalk-formatted correctly. + + /* eslint-disable @typescript-eslint/unbound-method */ + + // Test our VerboseReporter customization. + (testNamePattern === undefined || + "should pass".match(testNamePattern) !== null + ? expect(stderrLines).toContain + : expect(stderrLines).not.toContain)( + `${PASS} ${formatTestFilename("src/", "pass.test.ts")}` + ); + (testNamePattern === undefined || + "should pass".match(testNamePattern) !== null + ? expect(stderrLines).toContainEqual + : expect(stderrLines).not.toContainEqual)( + // This test doesn't have a describe() block, so it's only indented 2 spaces. + expect.stringMatching(testResultRegexMatch("pass", "should pass", 2)) + ); + + (!skipFailures && + (testNamePattern === undefined || + "describe block should ([escape regex]?.*$ fail".match( + testNamePattern + ) !== null) + ? expect(stderrLines).toContain + : expect(stderrLines).not.toContain)( + `${FAIL} ${formatTestFilename("src/", "fail.test.ts")}` + ); + (!skipFailures && + (testNamePattern === undefined || + "describe block should ([escape regex]?.*$ fail".match( + testNamePattern + ) !== null) + ? expect(stderrLines).toContainEqual + : expect(stderrLines).not.toContainEqual)( + expect.stringMatching( + testResultRegexMatch("fail", "should ([escape regex]?.*$ fail") + ) + ); + + const flakyTest1Name = `should be flaky 1${expectedFlakeTestNameSuffix}`; + const flakyTest1ShouldRun = + !skipFlake && + (!quarantineFlake || + failToFetchManifest || + !expectQuarantinedTestsToBeSkipped) && + (testNamePattern === undefined || + flakyTest1Name.match(testNamePattern) !== null); + (flakyTest1ShouldRun + ? expect(stderrLines).toContain + : expect(stderrLines).not.toContain)( + `${ + quarantineFlake && + !failToFetchManifest && + expectQuarantinedTestsToBeQuarantined + ? `${QUARANTINED} ` + : "" + }${FAIL} ${formatTestFilename("src/", "flake.test.ts")}` + ); + // This test should fail then pass (though we're not verifying the order here). + (flakyTest1ShouldRun + ? expect(stderrLines).toContainEqual + : expect(stderrLines).not.toContainEqual)( + expect.stringMatching( + testResultRegexMatch( + quarantineFlake && !failToFetchManifest ? "quarantined" : "fail", + flakyTest1Name, + 2 + ) + ) + ); + (expectPluginToBeEnabled && expectedFailureRetries > 0 && flakyTest1ShouldRun + ? expect(stderrLines).toContainEqual + : expect(stderrLines).not.toContainEqual)( + expect.stringMatching(testResultRegexMatch("pass", flakyTest1Name, 2)) + ); + + const flakyTest2Name = `should be flaky 2${expectedFlakeTestNameSuffix}`; + const flakyTest2ShouldRun = + !skipFlake && + (!quarantineFlake || + failToFetchManifest || + !expectQuarantinedTestsToBeSkipped) && + (testNamePattern === undefined || + flakyTest2Name.match(testNamePattern) !== null); + (flakyTest2ShouldRun + ? expect(stderrLines).toContainEqual + : expect(stderrLines).not.toContainEqual)( + expect.stringMatching( + testResultRegexMatch( + quarantineFlake && !failToFetchManifest ? "quarantined" : "fail", + flakyTest2Name, + 2 + ) + ) + ); + (expectPluginToBeEnabled && expectedFailureRetries > 0 && flakyTest2ShouldRun + ? expect(stderrLines).toContainEqual + : expect(stderrLines).not.toContainEqual)( + expect.stringMatching(testResultRegexMatch("pass", flakyTest2Name, 2)) + ); + + (!skipQuarantined && + (!expectQuarantinedTestsToBeSkipped || failToFetchManifest) && + (testNamePattern === undefined || + "describe block should be quarantined".match(testNamePattern) !== null) + ? expect(stderrLines).toContain + : expect(stderrLines).not.toContain)( + `${ + expectPluginToBeEnabled && + !failToFetchManifest && + expectQuarantinedTestsToBeQuarantined && + !expectQuarantinedTestsToBeSkipped + ? `${QUARANTINED} ` + : "" + }${FAIL} ${formatTestFilename("src/", "quarantined.test.ts")}` + ); + (!skipQuarantined && + (testNamePattern === undefined || + "describe block should be quarantined".match(testNamePattern) !== null) && + !expectQuarantinedTestsToBeSkipped + ? expect(stderrLines).toContainEqual + : expect(stderrLines).not.toContainEqual)( + expect.stringMatching( + testResultRegexMatch( + expectPluginToBeEnabled && + !failToFetchManifest && + expectQuarantinedTestsToBeQuarantined + ? "quarantined" + : "fail", + "should be quarantined" + ) + ) + ); + + const mixedFailTestShouldRun = + !skipFailures && + (testNamePattern === undefined || + "mixed mixed: should fail".match(testNamePattern) !== null); + const mixedQuarantinedTestShouldRun = + !expectQuarantinedTestsToBeSkipped && + !skipQuarantined && + (testNamePattern === undefined || + "mixed mixed: should be quarantined".match(testNamePattern) !== null); + const mixedPassTestShouldRun = + testNamePattern === undefined || + "mixed mixed: should pass".match(testNamePattern) !== null; + + // Mixed file containing both a failed test and a quarantined one. + (((!expectPluginToBeEnabled || + failToFetchManifest || + !expectQuarantinedTestsToBeQuarantined) && + mixedQuarantinedTestShouldRun) || + mixedFailTestShouldRun + ? expect(stderrLines).toContain + : expect(stderrLines).not.toContain)( + `${FAIL} ${formatTestFilename("src/", "mixed.test.ts")}` + ); + (mixedQuarantinedTestShouldRun + ? expect(stderrLines).toContainEqual + : expect(stderrLines).not.toContainEqual)( + expect.stringMatching( + testResultRegexMatch( + expectPluginToBeEnabled && + !failToFetchManifest && + expectQuarantinedTestsToBeQuarantined + ? "quarantined" + : "fail", + "mixed: should be quarantined" + ) + ) + ); + (mixedFailTestShouldRun + ? expect(stderrLines).toContainEqual + : expect(stderrLines).not.toContainEqual)( + expect.stringMatching(testResultRegexMatch("fail", "mixed: should fail")) + ); + + expect( + stderrLines.filter((line) => + testResultRegexMatch("pass", "mixed: should pass").test(line as string) + ) + ).toHaveLength(mixedPassTestShouldRun ? 1 : 0); + + // The passed test gets skipped during the retries. + if (mixedFailTestShouldRun || mixedQuarantinedTestShouldRun) { + expect( + stderrLines.filter((line) => + testResultRegexMatch("skipped", "mixed: should pass").test( + line as string + ) + ) + ).toHaveLength( + testNamePattern !== undefined && + "mixed mixed: should pass".match(testNamePattern) === null && + expectPluginToBeEnabled + ? expectedFailureRetries + 1 + : expectPluginToBeEnabled + ? expectedFailureRetries + : testNamePattern !== undefined && + "mixed mixed: should pass".match(testNamePattern) === null + ? 1 + : 0 + ); + } + + // Test our SummaryReporter customization. + expect(stderrLines).toContain( + `\u001b[1mTest Suites: \u001b[22m${ + expectedResults.failedSuites !== 0 + ? `\u001b[1m\u001b[31m${expectedResults.failedSuites} failed\u001b[39m\u001b[22m, ` + : "" + }${ + expectedResults.quarantinedSuites !== 0 + ? `\u001b[1m\u001b[33m${expectedResults.quarantinedSuites} quarantined\u001b[39m\u001b[22m, ` + : "" + }${ + expectedResults.skippedSuites !== 0 + ? `\u001b[1m\u001b[33m${expectedResults.skippedSuites} skipped\u001b[39m\u001b[22m, ` + : "" + }${ + expectedResults.passedSuites !== 0 + ? `\u001b[1m\u001b[32m${expectedResults.passedSuites} passed\u001b[39m\u001b[22m, ` + : "" + }${ + expectedResults.skippedSuites !== 0 + ? `${ + expectedResults.failedSuites + + expectedResults.quarantinedSuites + + expectedResults.passedSuites + } of ${ + expectedResults.failedSuites + + expectedResults.quarantinedSuites + + expectedResults.passedSuites + + expectedResults.skippedSuites + }` + : expectedResults.failedSuites + + expectedResults.quarantinedSuites + + expectedResults.passedSuites + } total` + ); + + expect(stderrLines).toContain( + `\u001b[1mTests: \u001b[22m${ + expectedResults.failedTests !== 0 + ? `\u001b[1m\u001b[31m${expectedResults.failedTests} failed\u001b[39m\u001b[22m, ` + : "" + }${ + expectedResults.flakyTests !== 0 + ? `\u001b[1m\u001b[95m${expectedResults.flakyTests} flaky\u001b[39m\u001b[22m, ` + : "" + }${ + expectedResults.quarantinedTests !== 0 + ? `\u001b[1m\u001b[33m${expectedResults.quarantinedTests} quarantined\u001b[39m\u001b[22m, ` + : "" + }${ + expectedResults.skippedTests !== 0 + ? `\u001b[1m\u001b[33m${expectedResults.skippedTests} skipped\u001b[39m\u001b[22m, ` + : "" + }${ + expectedResults.passedTests !== 0 + ? `\u001b[1m\u001b[32m${expectedResults.passedTests} passed\u001b[39m\u001b[22m, ` + : "" + }${ + expectedResults.failedTests + + expectedResults.flakyTests + + expectedResults.quarantinedTests + + expectedResults.passedTests + + expectedResults.skippedTests + } total` + ); + + expect(stderrLines).toContain( + `\u001b[1mSnapshots: \u001b[22m${ + expectedResults.failedSnapshots > 0 + ? `\u001b[1m\u001b[31m${expectedResults.failedSnapshots} failed\u001b[39m\u001b[22m, ` + : "" + }${ + expectedResults.passedSnapshots > 0 + ? `\u001b[1m\u001b[32m${expectedResults.passedSnapshots} passed\u001b[39m\u001b[22m, ` + : "" + }${expectedResults.totalSnapshots} total` + ); + // None of the snapshots should be obsolete. + expect(stderrLines).not.toContainEqual( + expect.stringMatching(new RegExp("[0-9]+ snapshot(:?s)? obsolete")) + ); + + // The duration here is based on the mocked time, so it should be deterministic. + expect(stderrLines).toContainEqual( + expect.stringMatching( + new RegExp( + `${escapeStringRegexp("\u001b[1mTime:\u001b[22m ")}[0-9.]+ s` + ) + ) + ); + + expect(stderrLines).toContain( + `\u001b[2mRan all test suites\u001b[22m\u001b[2m${ + testNamePattern !== undefined + ? ` with tests matching \u001b[22m"${testNamePattern}"\u001b[2m` + : "" + }.\u001b[22m` + ); + + (expectPluginToBeEnabled && expectResultsToBeUploaded && !failToUploadResults + ? expect(stderrLines).toContain + : expect(stderrLines).not.toContain)( + `Unflakable report: http://localhost:${apiServerPort}/test-suites/${expectedSuiteId}/runs/${MOCK_RUN_ID}` + ); +}; diff --git a/packages/plugins-common/src/config.ts b/packages/plugins-common/src/config.ts index 6606b5a..9d4dd8e 100644 --- a/packages/plugins-common/src/config.ts +++ b/packages/plugins-common/src/config.ts @@ -223,6 +223,14 @@ export const setCosmiconfig = (config: typeof cosmiconfig): void => { ).__unflakableCosmiconfig = config; }; +export const setCosmiconfigSync = (config: typeof cosmiconfigSync): void => { + ( + globalThis as { + __unflakableCosmiconfigSync?: typeof cosmiconfigSync; + } + ).__unflakableCosmiconfigSync = config; +}; + const loadConfigFile = async ( searchFrom: string ): Promise => { @@ -254,7 +262,13 @@ export const loadConfig = ( ); const loadConfigFileSync = (searchFrom: string): UnflakableConfigFile => { - const configExplorer = cosmiconfigSync("unflakable", { + const configExplorer = ( + ( + globalThis as { + __unflakableCosmiconfigSync?: typeof cosmiconfigSync; + } + ).__unflakableCosmiconfigSync ?? cosmiconfigSync + )("unflakable", { searchPlaces: SEARCH_PLACES, }); debug(`Searching for config from directory \`${searchFrom}\` upward`); diff --git a/packages/plugins-common/src/index.ts b/packages/plugins-common/src/index.ts index 32218ac..9ea836e 100644 --- a/packages/plugins-common/src/index.ts +++ b/packages/plugins-common/src/index.ts @@ -9,6 +9,7 @@ export { loadConfig, loadConfigSync, setCosmiconfig, + setCosmiconfigSync, } from "./config"; export { EnvVar, diff --git a/packages/plugins-common/src/manifest.ts b/packages/plugins-common/src/manifest.ts index f118f69..e4799f1 100644 --- a/packages/plugins-common/src/manifest.ts +++ b/packages/plugins-common/src/manifest.ts @@ -30,9 +30,10 @@ export const getTestSuiteManifest = ({ }) .catch((e: Error) => { log( - chalk.red( - `ERROR: Failed to get Unflakable manifest: ${e.toString()}\n` - ) + chalk.yellow.bold("Test failures will NOT be quarantined.\n") + chalk.red(`ERROR: Failed to get Unflakable manifest: ${e.toString()}`) + + "\n" + + chalk.yellow.bold("Test failures will NOT be quarantined.") + + "\n" ); return undefined; }) diff --git a/packages/cypress-plugin/test/integration-common/.eslintrc.js b/packages/test-common/.eslintrc.js similarity index 62% rename from packages/cypress-plugin/test/integration-common/.eslintrc.js rename to packages/test-common/.eslintrc.js index b6febf5..02409c8 100644 --- a/packages/cypress-plugin/test/integration-common/.eslintrc.js +++ b/packages/test-common/.eslintrc.js @@ -1,5 +1,5 @@ // Copyright (c) 2023 Developer Innovations, LLC module.exports = { - extends: ["../../../../.eslintrc-ts.js"], + extends: ["../../.eslintrc-ts.js"], }; diff --git a/packages/cypress-plugin/test/integration-common/package.json b/packages/test-common/package.json similarity index 68% rename from packages/cypress-plugin/test/integration-common/package.json rename to packages/test-common/package.json index a852526..f1b4508 100644 --- a/packages/cypress-plugin/test/integration-common/package.json +++ b/packages/test-common/package.json @@ -1,15 +1,20 @@ { - "name": "cypress-integration-common", + "name": "unflakable-test-common", "private": true, "dependencies": { + "cosmiconfig": "^7.0.1", "debug": "^4.3.3", - "expect": "^29.5.0", - "simple-git": "^3.16.0" + "deep-equal": "^2.0.5", + "expect": "25.1.0 - 29", + "mockttp": "^3.7.5", + "simple-git": "^3.16.0", + "tree-kill": "^1.2.2" }, "devDependencies": { "@rollup/plugin-commonjs": "^24.1.0", "@rollup/plugin-node-resolve": "^15.0.2", "@rollup/plugin-typescript": "^11.1.1", + "@types/jest": "25.1.0 - 29", "@unflakable/plugins-common": "workspace:^", "rimraf": "^5.0.1", "rollup": "^3.21.1", diff --git a/packages/cypress-plugin/test/integration-common/rollup.config.mjs b/packages/test-common/rollup.config.mjs similarity index 80% rename from packages/cypress-plugin/test/integration-common/rollup.config.mjs rename to packages/test-common/rollup.config.mjs index 1474adc..0e63ae9 100644 --- a/packages/cypress-plugin/test/integration-common/rollup.config.mjs +++ b/packages/test-common/rollup.config.mjs @@ -24,10 +24,21 @@ const isExternal = (id) => */ export default [ { - input: ["src/config.ts", "src/git.ts", "src/mock-cosmiconfig.ts"], + input: [ + "src/config.ts", + "src/git.ts", + "src/mock-backend.ts", + "src/mock-cosmiconfig.ts", + "src/mock-git.ts", + "src/spawn.ts", + ], output: { dir: "dist", format: "cjs", + // Jest 28+ provides a .default export, while Jest < 28 directly exports the expect() function + // as its top-level module.exports value. Using "compat" here lets us + // `import { default as expect } from "expect"` with both versions. + interop: (id) => (id === "expect" ? "compat" : "default"), }, external: isExternal, // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment diff --git a/packages/cypress-plugin/test/integration-common/src/config.ts b/packages/test-common/src/config.ts similarity index 60% rename from packages/cypress-plugin/test/integration-common/src/config.ts rename to packages/test-common/src/config.ts index 81ce31b..485556f 100644 --- a/packages/cypress-plugin/test/integration-common/src/config.ts +++ b/packages/test-common/src/config.ts @@ -1,9 +1,13 @@ // Copyright (c) 2023 Developer Innovations, LLC import _debug from "debug"; -import { UnflakableConfig, setCosmiconfig } from "@unflakable/plugins-common"; -import type { cosmiconfig, Options } from "cosmiconfig"; -import { expect } from "expect"; +import { + UnflakableConfig, + setCosmiconfig, + setCosmiconfigSync, +} from "@unflakable/plugins-common"; +import type { cosmiconfig, cosmiconfigSync, Options } from "cosmiconfig"; +import { default as expect } from "expect"; const debug = _debug("unflakable:integration-common:config"); @@ -57,4 +61,30 @@ export const registerCosmiconfigMock = (): void => { }; } ); + + setCosmiconfigSync( + ( + moduleName: string, + options?: Options + ): ReturnType => { + expect(moduleName).toBe("unflakable"); + expect(options?.searchPlaces).toContain("package.json"); + expect(options?.searchPlaces).toContain("unflakable.json"); + expect(options?.searchPlaces).toContain("unflakable.js"); + expect(options?.searchPlaces).toContain("unflakable.yaml"); + expect(options?.searchPlaces).toContain("unflakable.yml"); + return { + clearCaches: throwUnimplemented, + clearLoadCache: throwUnimplemented, + clearSearchCache: throwUnimplemented, + load: throwUnimplemented, + search: ( + searchFrom?: string + ): ReturnType["search"]> => { + expect(searchFrom).toBe(params.searchFrom); + return params.searchResult; + }, + }; + } + ); }; diff --git a/packages/cypress-plugin/test/integration-common/src/git.ts b/packages/test-common/src/git.ts similarity index 100% rename from packages/cypress-plugin/test/integration-common/src/git.ts rename to packages/test-common/src/git.ts diff --git a/packages/test-common/src/mock-backend.ts b/packages/test-common/src/mock-backend.ts new file mode 100644 index 0000000..2479a87 --- /dev/null +++ b/packages/test-common/src/mock-backend.ts @@ -0,0 +1,256 @@ +// Copyright (c) 2023 Developer Innovations, LLC + +import { + CompletedRequest, + getLocal as getLocalHttpServer, + MockedEndpoint, + Mockttp, +} from "mockttp"; +import type { + CallbackResponseMessageResult, + CallbackResponseResult, +} from "mockttp/dist/rules/requests/request-handler-definitions"; +import { gunzipSync } from "zlib"; +import _debug from "debug"; +import { + CreateTestSuiteRunFromUploadRequest, + CreateTestSuiteRunInlineRequest, + TestSuiteManifest, + TestSuiteRunPendingSummary, +} from "@unflakable/js-api"; + +const debug = _debug("unflakable:test-common:mock-backend"); + +export type UnmatchedEndpoints = { + unmatchedApiRequestEndpoint: MockedEndpoint; + unmatchedObjectStoreRequestEndpoint: MockedEndpoint; +}; + +export class MockBackend { + private readonly apiServer: Mockttp; + private readonly objectStoreServer: Mockttp; + + constructor() { + this.apiServer = getLocalHttpServer({ + // debug: true, + suggestChanges: false, + }); + this.objectStoreServer = getLocalHttpServer({ + // debug: true, + suggestChanges: false, + }); + } + + public get apiServerPort(): number { + return this.apiServer.port; + } + + public addExpectations = async ( + onError: (e: unknown) => void, + manifest: TestSuiteManifest | null, + verifyUploadResults: (request: CompletedRequest) => void, + runSummary: TestSuiteRunPendingSummary | null, + userAgentRegex: RegExp, + { + expectPluginToBeEnabled, + expectResultsToBeUploaded, + expectedApiKey, + expectedSuiteId, + }: { + expectPluginToBeEnabled: boolean; + expectResultsToBeUploaded: boolean; + expectedApiKey: string; + expectedSuiteId: string; + } + ): Promise => { + const onUnmatchedRequest = ( + request: CompletedRequest + ): CallbackResponseResult => { + onError( + new Error(`Unexpected request ${request.method} ${request.path}`) + ); + return { statusCode: 500 }; + }; + + const unmatchedApiRequestEndpoint = await this.apiServer + .forUnmatchedRequest() + .thenCallback(onUnmatchedRequest); + const unmatchedObjectStoreRequestEndpoint = await this.objectStoreServer + .forUnmatchedRequest() + .thenCallback(onUnmatchedRequest); + + if (!expectPluginToBeEnabled) { + return { + unmatchedApiRequestEndpoint, + unmatchedObjectStoreRequestEndpoint, + }; + } + + await this.apiServer + .forGet(`/api/v1/test-suites/${expectedSuiteId}/manifest`) + .times(manifest === null ? 3 : 1) + .withHeaders({ + Authorization: `Bearer ${expectedApiKey}`, + }) + .thenCallback((request): CallbackResponseResult => { + try { + expect(request.headers["user-agent"]).toMatch(userAgentRegex); + + if (manifest === null) { + return "reset"; + } + + return { + statusCode: 200, + json: manifest, + }; + } catch (e: unknown) { + onError(e); + return { statusCode: 500 }; + } + }); + + if (expectResultsToBeUploaded) { + const uploadPath = + `/unflakable-backend-mock-test-uploads/teams/MOCK_TEAM_ID/suites/${expectedSuiteId}/runs/` + + `upload/MOCK_UPLOAD_ID`; + const uploadQuery = "?X-Amz-Signature=MOCK_SIGNATURE"; + + await this.apiServer + .forPost(`/api/v1/test-suites/${expectedSuiteId}/runs/upload`) + .once() + .withHeaders({ + Authorization: `Bearer ${expectedApiKey}`, + "Content-Type": "application/json", + }) + .thenCallback(async (request) => { + try { + expect(await request.body.getText()).toBe(""); + return { + statusCode: 201, + headers: { + Location: `http://localhost:${this.objectStoreServer.port}${uploadPath}${uploadQuery}`, + }, + json: { + upload_id: "MOCK_UPLOAD_ID", + }, + }; + } catch (e) { + onError(e); + return { + statusCode: 500, + }; + } + }); + + let runRequest: CreateTestSuiteRunInlineRequest | null = null; + await this.objectStoreServer + .forPut(uploadPath) + .once() + .withExactQuery(uploadQuery) + .withHeaders({ + "Content-Encoding": "gzip", + "Content-Type": "application/json", + }) + .thenCallback((request): CallbackResponseMessageResult => { + try { + runRequest = JSON.parse( + gunzipSync(request.body.buffer).toString() + ) as CreateTestSuiteRunInlineRequest; + + verifyUploadResults(request); + + return { + statusCode: 200, + }; + } catch (e) { + onError(e); + return { statusCode: 500 }; + } + }); + + await this.apiServer + .forPost(`/api/v1/test-suites/${expectedSuiteId}/runs`) + .times(runSummary === null ? 3 : 1) + .withHeaders({ + Authorization: `Bearer ${expectedApiKey}`, + "Content-Type": "application/json", + }) + .thenCallback(async (request): Promise => { + try { + const body = await request.body.getText(); + expect(body).not.toBeNull(); + + const parsedBody = ((): CreateTestSuiteRunFromUploadRequest => { + try { + return JSON.parse( + body as string + ) as CreateTestSuiteRunFromUploadRequest; + } catch (e) { + throw new Error( + `Invalid request body: ${JSON.stringify(body)}`, + { + cause: e, + } + ); + } + })(); + + expect(parsedBody.upload_id).toBe("MOCK_UPLOAD_ID"); + expect(runRequest).not.toBeNull(); + + if (runSummary === null) { + return "reset"; + } + + return { + json: runSummary, + statusCode: 201, + }; + } catch (e) { + onError(e); + return { + statusCode: 500, + }; + } + }); + } + + return { + unmatchedApiRequestEndpoint, + unmatchedObjectStoreRequestEndpoint, + }; + }; + + public checkExpectations = async ({ + unmatchedApiRequestEndpoint, + unmatchedObjectStoreRequestEndpoint, + }: UnmatchedEndpoints): Promise => { + expect(await this.apiServer.getPendingEndpoints()).toStrictEqual([ + unmatchedApiRequestEndpoint, + ]); + expect(await this.objectStoreServer.getPendingEndpoints()).toStrictEqual([ + unmatchedObjectStoreRequestEndpoint, + ]); + }; + + public start = async (): Promise => { + await this.apiServer.start(); + debug( + `Listening for mock API requests on http://localhost:${this.apiServer.port}` + ); + + await this.objectStoreServer.start(); + debug( + `Listening for mock S3 requests on http://localhost:${this.objectStoreServer.port}` + ); + }; + + public stop = async (): Promise => { + debug(`Stopping mock API server`); + await this.apiServer.stop(); + + debug(`Stopping mock S3 server`); + return this.objectStoreServer.stop(); + }; +} diff --git a/packages/cypress-plugin/test/integration-common/src/mock-cosmiconfig.ts b/packages/test-common/src/mock-cosmiconfig.ts similarity index 77% rename from packages/cypress-plugin/test/integration-common/src/mock-cosmiconfig.ts rename to packages/test-common/src/mock-cosmiconfig.ts index 09da367..1f6936e 100644 --- a/packages/cypress-plugin/test/integration-common/src/mock-cosmiconfig.ts +++ b/packages/test-common/src/mock-cosmiconfig.ts @@ -1,7 +1,6 @@ // Copyright (c) 2023 Developer Innovations, LLC -// Script loaded by Node.JS via --require that mocks cosmiconfig within the cypress-unflakable bin -// script for testing. +// Script loaded by Node.JS via --require that mocks cosmiconfig for testing. import { registerCosmiconfigMock } from "./config"; diff --git a/packages/test-common/src/mock-git.ts b/packages/test-common/src/mock-git.ts new file mode 100644 index 0000000..fc84fea --- /dev/null +++ b/packages/test-common/src/mock-git.ts @@ -0,0 +1,7 @@ +// Copyright (c) 2023 Developer Innovations, LLC + +// Script loaded by Node.JS via --require that mocks simple-git for testing. + +import { registerSimpleGitMock } from "./git"; + +registerSimpleGitMock(); diff --git a/packages/test-common/src/spawn.ts b/packages/test-common/src/spawn.ts new file mode 100644 index 0000000..f693c3c --- /dev/null +++ b/packages/test-common/src/spawn.ts @@ -0,0 +1,155 @@ +// Copyright (c) 2023 Developer Innovations, LLC + +import { TextDecoder } from "util"; +import treeKill from "tree-kill"; +import _debug from "debug"; +import { spawn } from "child_process"; + +const debug = _debug("unflakable:test-common:spawn"); + +// Async callbacks (e.g., mock API routes) can set this when an occurs during the test. +export type AsyncTestError = { error: unknown | undefined }; + +export const spawnTestWithTimeout = async ( + nodeArgs: string[], + env: { [key in string]: string | undefined }, + cwd: string, + timeout_ms: number, + verifyOutput: (stdoutLines: string[], stderrLines: string[]) => Promise, + expectedExitCode: number, + escapeStderrDebugOutput: boolean, + asyncTestError: AsyncTestError +): Promise => { + debug( + `Spawning test:\n args = %o\n environment = %o\n cwd = %s`, + nodeArgs, + env, + cwd + ); + + const child = spawn("node", nodeArgs, { + cwd, + env, + }); + + const onOutput = ( + name: string, + onLine: (line: string, now: Date) => void, + escapeDebugOutput: boolean + ): ((data: Buffer) => void) => { + const debugExt = debug.extend(name); + const decoder = new TextDecoder("utf-8", { fatal: true }); + + const pending = { s: "" }; + + // Don't eat the last line of output. + child.on("exit", () => { + if (pending.s !== "") { + onLine(pending.s, new Date()); + debugExt(escapeDebugOutput ? JSON.stringify(pending.s) : pending.s); + } + }); + + return (data: Buffer): void => { + const now = new Date(); + // In case data terminates in the middle of a Unicode sequence, we need to use a stateful + // TextDecoder with `stream: true`. Otherwise, invalid UTF-8 sequences at the end get + // converted to 0xFFFD, which breaks the tests non-deterministically (i.e., makes them flaky). + const lines = decoder.decode(data, { stream: true }).split("\n"); + + // If the last line is empty, then `dataStr` ends in a linebreak. Otherwise, we have a + // partial line that we want to defer until the next call. + lines.slice(0, lines.length - 1).forEach((line, idx) => { + const lineWithPending = idx === 0 ? pending.s + line : line; + onLine(lineWithPending, now); + debugExt( + escapeDebugOutput ? JSON.stringify(lineWithPending) : lineWithPending + ); + }); + + pending.s = lines[lines.length - 1]; + }; + }; + + const stdoutLines = [] as string[]; + const stderrLines = [] as string[]; + const combinedLines = [] as string[]; + + child.stderr.on( + "data", + onOutput( + "stderr", + (line, now) => { + stderrLines.push(line); + combinedLines.push(`${now.toISOString()} ${line}`); + }, + // Don't escape stderr output since it likely comes from debug output in the subprocess, which + // is intended for human consumption and not for verifying test results. + escapeStderrDebugOutput + ) + ); + child.stdout.on( + "data", + onOutput( + "stdout", + (line, now) => { + stdoutLines.push(line); + combinedLines.push(`${now.toISOString()} ${line}`); + }, + // Escape special characters in debug output so that we can more easily understand test + // failures related to unexpected output. + true + ) + ); + + type ChildResult = { + code: number | null; + signal: NodeJS.Signals | null; + }; + + try { + const { code, signal } = await new Promise( + (resolve, reject) => { + const watchdog = setTimeout(() => { + console.error( + `Test timed out after ${timeout_ms}ms; killing child process tree` + ); + const timeoutError = new Error( + `Test timed out after ${timeout_ms}ms` + ); + if (asyncTestError.error === undefined) { + asyncTestError.error = timeoutError; + } + treeKill(child.pid, "SIGKILL", () => reject(timeoutError)); + }, timeout_ms); + + child.on("error", (err) => { + clearTimeout(watchdog); + reject(err); + }); + child.on("exit", (code, signal) => { + clearTimeout(watchdog); + resolve({ code, signal }); + }); + } + ); + + if (asyncTestError.error !== undefined) { + throw asyncTestError.error; + } + + await verifyOutput(stdoutLines, stderrLines); + + expect(signal).toBe(null); + expect(code).toBe(expectedExitCode); + } catch (e: unknown) { + // Jest doesn't have a built-in setting for printing console logs only for failed tests, so we + // just defer the output until this catch block and attach it to the error. See + // https://github.com/jestjs/jest/issues/4156. We don't call console.log() directly here because + // that output gets printed before the failed test, whereas the error gets printed immediately + // after, which makes it easy to associate with the corresponding test. + throw new Error(`Test failed with output:\n\n${combinedLines.join("\n")}`, { + cause: e, + }); + } +}; diff --git a/packages/cypress-plugin/test/integration-common/src/tsconfig.json b/packages/test-common/src/tsconfig.json similarity index 79% rename from packages/cypress-plugin/test/integration-common/src/tsconfig.json rename to packages/test-common/src/tsconfig.json index 3aa8530..b51fb3a 100644 --- a/packages/cypress-plugin/test/integration-common/src/tsconfig.json +++ b/packages/test-common/src/tsconfig.json @@ -4,7 +4,8 @@ "declaration": true, "declarationDir": "../dist", // Required by Rollup (consumed by Rollup in the *-plugin packages). - "module": "esnext" + "module": "esnext", + "types": ["jest", "node"] }, "include": ["."] } diff --git a/packages/test-common/tsconfig.json b/packages/test-common/tsconfig.json new file mode 100644 index 0000000..00ed11b --- /dev/null +++ b/packages/test-common/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + // Remove DOM types and support 2-argument Error constructor that takes a cause. + "lib": ["ES2022"] + }, + "include": [".eslintrc.js", "rollup.config.mjs"] +} diff --git a/yarn.lock b/yarn.lock index ca4db0f..7ac906b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -40,7 +40,7 @@ __metadata: languageName: node linkType: hard -"@babel/core@npm:^7.0.0, @babel/core@npm:^7.11.6, @babel/core@npm:^7.12.3, @babel/core@npm:^7.21.5": +"@babel/core@npm:^7.11.6, @babel/core@npm:^7.12.3, @babel/core@npm:^7.21.5": version: 7.22.1 resolution: "@babel/core@npm:7.22.1" dependencies: @@ -1307,7 +1307,7 @@ __metadata: languageName: node linkType: hard -"@babel/runtime@npm:^7.0.0, @babel/runtime@npm:^7.8.4": +"@babel/runtime@npm:^7.8.4": version: 7.17.2 resolution: "@babel/runtime@npm:7.17.2" dependencies: @@ -2829,12 +2829,12 @@ __metadata: linkType: hard "@types/jest@npm:25.1.0 - 29, @types/jest@npm:^29.5.2": - version: 29.5.2 - resolution: "@types/jest@npm:29.5.2" + version: 29.5.3 + resolution: "@types/jest@npm:29.5.3" dependencies: expect: ^29.0.0 pretty-format: ^29.0.0 - checksum: 7d205599ea3cccc262bad5cc173d3242d6bf8138c99458509230e4ecef07a52d6ddcde5a1dbd49ace655c0af51d2dbadef3748697292ea4d86da19d9e03e19c0 + checksum: e36bb92e0b9e5ea7d6f8832baa42f087fc1697f6cd30ec309a07ea4c268e06ec460f1f0cfd2581daf5eff5763475190ec1ad8ac6520c49ccfe4f5c0a48bfa676 languageName: node linkType: hard @@ -4616,13 +4616,6 @@ __metadata: languageName: node linkType: hard -"core-js@npm:^3.0.0": - version: 3.21.1 - resolution: "core-js@npm:3.21.1" - checksum: d68eddd831340ad5b24ac29c72fda022a43b17f194c4278b6b875a843283d316502cb4abd07f28631d6ebc4387f66aa06e2b1b3c8fd7e08096a751b5c63f6889 - languageName: node - linkType: hard - "core-util-is@npm:1.0.2": version: 1.0.2 resolution: "core-util-is@npm:1.0.2" @@ -4720,24 +4713,6 @@ __metadata: languageName: node linkType: hard -"cypress-integration-common@workspace:^, cypress-integration-common@workspace:packages/cypress-plugin/test/integration-common": - version: 0.0.0-use.local - resolution: "cypress-integration-common@workspace:packages/cypress-plugin/test/integration-common" - dependencies: - "@rollup/plugin-commonjs": ^24.1.0 - "@rollup/plugin-node-resolve": ^15.0.2 - "@rollup/plugin-typescript": ^11.1.1 - "@unflakable/plugins-common": "workspace:^" - debug: ^4.3.3 - expect: ^29.5.0 - rimraf: ^5.0.1 - rollup: ^3.21.1 - rollup-plugin-dts: ^5.3.0 - simple-git: ^3.16.0 - typescript: ^4.9.5 - languageName: unknown - linkType: soft - "cypress-integration-input-esm@workspace:packages/cypress-plugin/test/integration-input-esm": version: 0.0.0-use.local resolution: "cypress-integration-input-esm@workspace:packages/cypress-plugin/test/integration-input-esm" @@ -4746,7 +4721,6 @@ __metadata: "@types/react-dom": ^18.2.4 "@unflakable/cypress-plugin": "workspace:^" cypress: 10 - 12 - cypress-integration-common: "workspace:^" mocha: =7.0.1 mocha-junit-reporter: ^2.2.0 process: ^0.11.10 @@ -4754,6 +4728,7 @@ __metadata: react-dom: ^18.2.0 ts-loader: ^9.4.3 typescript: ^4.9.5 + unflakable-test-common: "workspace:^" webpack: ^5.84.1 languageName: unknown linkType: soft @@ -4766,7 +4741,6 @@ __metadata: "@types/react-dom": ^18.2.4 "@unflakable/cypress-plugin": "workspace:^" cypress: 10 - 12 - cypress-integration-common: "workspace:^" cypress-multi-reporters: ^1.6.3 mocha: =7.0.1 mocha-junit-reporter: ^2.2.0 @@ -4775,6 +4749,7 @@ __metadata: react-dom: ^18.2.0 ts-loader: ^9.4.3 typescript: ^4.9.5 + unflakable-test-common: "workspace:^" webpack: ^5.84.1 languageName: unknown linkType: soft @@ -4787,7 +4762,6 @@ __metadata: "@types/react-dom": ^18.2.4 "@unflakable/cypress-plugin": "workspace:^" cypress: 10 - 12 - cypress-integration-common: "workspace:^" mocha: =7.0.1 mocha-junit-reporter: ^2.2.0 process: ^0.11.10 @@ -4795,6 +4769,7 @@ __metadata: react-dom: ^18.2.0 ts-loader: ^9.4.3 typescript: ^4.9.5 + unflakable-test-common: "workspace:^" webpack: ^5.84.1 languageName: unknown linkType: soft @@ -4808,16 +4783,15 @@ __metadata: "@unflakable/js-api": "workspace:^" "@unflakable/plugins-common": "workspace:^" cypress: 10 - 12 - cypress-integration-common: "workspace:^" debug: ^4.3.3 escape-string-regexp: ^4.0.0 jest: ^29.5.0 jest-environment-node: ^29.5.0 jest-expect-message: ^1.1.3 mockttp: ^3.7.5 - tree-kill: ^1.2.2 ts-jest: ^29.1.0 typescript: ^4.9.5 + unflakable-test-common: "workspace:^" languageName: unknown linkType: soft @@ -5745,7 +5719,7 @@ __metadata: languageName: node linkType: hard -"expect@npm:^29.0.0, expect@npm:^29.5.0": +"expect@npm:25.1.0 - 29, expect@npm:^29.0.0, expect@npm:^29.5.0": version: 29.5.0 resolution: "expect@npm:29.5.0" dependencies: @@ -5910,43 +5884,6 @@ __metadata: languageName: node linkType: hard -"fetch-mock-jest@npm:^1.5.1": - version: 1.5.1 - resolution: "fetch-mock-jest@npm:1.5.1" - dependencies: - fetch-mock: ^9.11.0 - peerDependencies: - node-fetch: "*" - peerDependenciesMeta: - node-fetch: - optional: true - checksum: 371b1c4a1fb1cf507ace0bf1a505ae0a9b0184ef0edfa6225c3d93866ba80fa89b47d412eb077315007e0a73028e2ed5be4b88b85e992e53a834ecbd44c4a3be - languageName: node - linkType: hard - -"fetch-mock@npm:^9.11.0": - version: 9.11.0 - resolution: "fetch-mock@npm:9.11.0" - dependencies: - "@babel/core": ^7.0.0 - "@babel/runtime": ^7.0.0 - core-js: ^3.0.0 - debug: ^4.1.1 - glob-to-regexp: ^0.4.0 - is-subset: ^0.1.1 - lodash.isequal: ^4.5.0 - path-to-regexp: ^2.2.1 - querystring: ^0.2.0 - whatwg-url: ^6.5.0 - peerDependencies: - node-fetch: "*" - peerDependenciesMeta: - node-fetch: - optional: true - checksum: debc4dd83bcda79b0aa71c38d08da6036906cdc49393343eb3426112314a7e57557255664f745d2e3f0b9b2a6e852bd3a564ae3f08332c27e422d3441bb865bd - languageName: node - linkType: hard - "figures@npm:^3.2.0": version: 3.2.0 resolution: "figures@npm:3.2.0" @@ -6375,7 +6312,7 @@ __metadata: languageName: node linkType: hard -"glob-to-regexp@npm:^0.4.0, glob-to-regexp@npm:^0.4.1": +"glob-to-regexp@npm:^0.4.1": version: 0.4.1 resolution: "glob-to-regexp@npm:0.4.1" checksum: e795f4e8f06d2a15e86f76e4d92751cf8bbfcf0157cea5c2f0f35678a8195a750b34096b1256e436f0cebc1883b5ff0888c47348443e69546a5a87f9e1eb1167 @@ -7181,13 +7118,6 @@ __metadata: languageName: node linkType: hard -"is-subset@npm:^0.1.1": - version: 0.1.1 - resolution: "is-subset@npm:0.1.1" - checksum: 97b8d7852af165269b7495095691a6ce6cf20bdfa1f846f97b4560ee190069686107af4e277fbd93aa0845c4d5db704391460ff6e9014aeb73264ba87893df44 - languageName: node - linkType: hard - "is-symbol@npm:^1.0.2, is-symbol@npm:^1.0.3": version: 1.0.4 resolution: "is-symbol@npm:1.0.4" @@ -7413,7 +7343,7 @@ __metadata: languageName: node linkType: hard -"jest-cli@npm:25.1.0 - 29, jest-cli@npm:^29.5.0": +"jest-cli@npm:^29.5.0": version: 29.5.0 resolution: "jest-cli@npm:29.5.0" dependencies: @@ -7582,13 +7512,12 @@ __metadata: "@types/temp": ^0.9.1 "@unflakable/jest-plugin": "workspace:^" "@unflakable/js-api": "workspace:^" - cross-env: ^7.0.3 - deep-equal: ^2.0.5 - fetch-mock-jest: ^1.5.1 + escape-string-regexp: ^4.0.0 jest: 25.1.0 - 29 - jest-cli: 25.1.0 - 29 jest-environment-node: 25.1.0 - 29 + mockttp: ^3.7.5 temp: ^0.9.4 + unflakable-test-common: "workspace:^" languageName: unknown linkType: soft @@ -8141,13 +8070,6 @@ __metadata: languageName: node linkType: hard -"lodash.isequal@npm:^4.5.0": - version: 4.5.0 - resolution: "lodash.isequal@npm:4.5.0" - checksum: da27515dc5230eb1140ba65ff8de3613649620e8656b19a6270afe4866b7bd461d9ba2ac8a48dcc57f7adac4ee80e1de9f965d89d4d81a0ad52bb3eec2609644 - languageName: node - linkType: hard - "lodash.memoize@npm:4.x": version: 4.1.2 resolution: "lodash.memoize@npm:4.1.2" @@ -8169,13 +8091,6 @@ __metadata: languageName: node linkType: hard -"lodash.sortby@npm:^4.7.0": - version: 4.7.0 - resolution: "lodash.sortby@npm:4.7.0" - checksum: db170c9396d29d11fe9a9f25668c4993e0c1331bcb941ddbd48fb76f492e732add7f2a47cfdf8e9d740fa59ac41bbfaf931d268bc72aab3ab49e9f89354d718c - languageName: node - linkType: hard - "lodash@npm:^4.16.4, lodash@npm:^4.17.14, lodash@npm:^4.17.15, lodash@npm:^4.17.21": version: 4.17.21 resolution: "lodash@npm:4.17.21" @@ -9187,13 +9102,6 @@ __metadata: languageName: node linkType: hard -"path-to-regexp@npm:^2.2.1": - version: 2.4.0 - resolution: "path-to-regexp@npm:2.4.0" - checksum: 581175bf2968e51452f2b8c71f10e75c995693668b4ecf7d0b48962fbe0c56830661ca5dd5fd6d8e2f0cc9a045ce07e89af504ab133e1d21887c2712df85b1f4 - languageName: node - linkType: hard - "path-type@npm:^4.0.0": version: 4.0.0 resolution: "path-type@npm:4.0.0" @@ -9418,13 +9326,6 @@ __metadata: languageName: node linkType: hard -"querystring@npm:^0.2.0": - version: 0.2.1 - resolution: "querystring@npm:0.2.1" - checksum: 7b83b45d641e75fd39cd6625ddfd44e7618e741c61e95281b57bbae8fde0afcc12cf851924559e5cc1ef9baa3b1e06e22b164ea1397d65dd94b801f678d9c8ce - languageName: node - linkType: hard - "queue-microtask@npm:^1.2.2": version: 1.2.3 resolution: "queue-microtask@npm:1.2.3" @@ -10665,15 +10566,6 @@ __metadata: languageName: node linkType: hard -"tr46@npm:^1.0.1": - version: 1.0.1 - resolution: "tr46@npm:1.0.1" - dependencies: - punycode: ^2.1.0 - checksum: 96d4ed46bc161db75dbf9247a236ea0bfcaf5758baae6749e92afab0bc5a09cb59af21788ede7e55080f2bf02dce3e4a8f2a484cc45164e29f4b5e68f7cbcc1a - languageName: node - linkType: hard - "tr46@npm:~0.0.3": version: 0.0.3 resolution: "tr46@npm:0.0.3" @@ -10928,6 +10820,29 @@ __metadata: languageName: node linkType: hard +"unflakable-test-common@workspace:^, unflakable-test-common@workspace:packages/test-common": + version: 0.0.0-use.local + resolution: "unflakable-test-common@workspace:packages/test-common" + dependencies: + "@rollup/plugin-commonjs": ^24.1.0 + "@rollup/plugin-node-resolve": ^15.0.2 + "@rollup/plugin-typescript": ^11.1.1 + "@types/jest": 25.1.0 - 29 + "@unflakable/plugins-common": "workspace:^" + cosmiconfig: ^7.0.1 + debug: ^4.3.3 + deep-equal: ^2.0.5 + expect: 25.1.0 - 29 + mockttp: ^3.7.5 + rimraf: ^5.0.1 + rollup: ^3.21.1 + rollup-plugin-dts: ^5.3.0 + simple-git: ^3.16.0 + tree-kill: ^1.2.2 + typescript: ^4.9.5 + languageName: unknown + linkType: soft + "unicode-canonical-property-names-ecmascript@npm:^2.0.0": version: 2.0.0 resolution: "unicode-canonical-property-names-ecmascript@npm:2.0.0" @@ -11132,13 +11047,6 @@ __metadata: languageName: node linkType: hard -"webidl-conversions@npm:^4.0.2": - version: 4.0.2 - resolution: "webidl-conversions@npm:4.0.2" - checksum: c93d8dfe908a0140a4ae9c0ebc87a33805b416a33ee638a605b551523eec94a9632165e54632f6d57a39c5f948c4bab10e0e066525e9a4b87a79f0d04fbca374 - languageName: node - linkType: hard - "webpack-sources@npm:^3.2.3": version: 3.2.3 resolution: "webpack-sources@npm:3.2.3" @@ -11193,17 +11101,6 @@ __metadata: languageName: node linkType: hard -"whatwg-url@npm:^6.5.0": - version: 6.5.0 - resolution: "whatwg-url@npm:6.5.0" - dependencies: - lodash.sortby: ^4.7.0 - tr46: ^1.0.1 - webidl-conversions: ^4.0.2 - checksum: a10bd5e29f4382cd19789c2a7bbce25416e606b6fefc241c7fe34a2449de5bc5709c165bd13634eda433942d917ca7386a52841780b82dc37afa8141c31a8ebd - languageName: node - linkType: hard - "which-boxed-primitive@npm:^1.0.1, which-boxed-primitive@npm:^1.0.2": version: 1.0.2 resolution: "which-boxed-primitive@npm:1.0.2" From 0b2224d89fc476e499b81de464823c13a8d9b432 Mon Sep 17 00:00:00 2001 From: "David A. Ramos" Date: Sat, 8 Jul 2023 22:03:34 -0700 Subject: [PATCH 06/53] [jest] Rename integration test files to snake-case --- .../src/{disablePlugin.test.ts => disable-plugin.test.ts} | 0 .../src/{disableUpload.test.ts => disable-upload.test.ts} | 0 .../src/{ignoreFailures.test.ts => ignore-failures.test.ts} | 0 .../integration/src/{longNames.test.ts => long-names.test.ts} | 0 .../src/{noQuarantine.test.ts => no-quarantine.test.ts} | 0 .../src/{pluginFailures.test.ts => plugin-failures.test.ts} | 0 .../test/integration/src/{runTestCase.ts => run-test-case.ts} | 0 .../integration/src/{skipTests.test.ts => skip-tests.test.ts} | 0 .../src/{testNamePattern.test.ts => test-name-pattern.test.ts} | 0 packages/jest-plugin/test/integration/src/test-wrappers.ts | 2 +- packages/jest-plugin/test/integration/src/verify-output.ts | 2 +- 11 files changed, 2 insertions(+), 2 deletions(-) rename packages/jest-plugin/test/integration/src/{disablePlugin.test.ts => disable-plugin.test.ts} (100%) rename packages/jest-plugin/test/integration/src/{disableUpload.test.ts => disable-upload.test.ts} (100%) rename packages/jest-plugin/test/integration/src/{ignoreFailures.test.ts => ignore-failures.test.ts} (100%) rename packages/jest-plugin/test/integration/src/{longNames.test.ts => long-names.test.ts} (100%) rename packages/jest-plugin/test/integration/src/{noQuarantine.test.ts => no-quarantine.test.ts} (100%) rename packages/jest-plugin/test/integration/src/{pluginFailures.test.ts => plugin-failures.test.ts} (100%) rename packages/jest-plugin/test/integration/src/{runTestCase.ts => run-test-case.ts} (100%) rename packages/jest-plugin/test/integration/src/{skipTests.test.ts => skip-tests.test.ts} (100%) rename packages/jest-plugin/test/integration/src/{testNamePattern.test.ts => test-name-pattern.test.ts} (100%) diff --git a/packages/jest-plugin/test/integration/src/disablePlugin.test.ts b/packages/jest-plugin/test/integration/src/disable-plugin.test.ts similarity index 100% rename from packages/jest-plugin/test/integration/src/disablePlugin.test.ts rename to packages/jest-plugin/test/integration/src/disable-plugin.test.ts diff --git a/packages/jest-plugin/test/integration/src/disableUpload.test.ts b/packages/jest-plugin/test/integration/src/disable-upload.test.ts similarity index 100% rename from packages/jest-plugin/test/integration/src/disableUpload.test.ts rename to packages/jest-plugin/test/integration/src/disable-upload.test.ts diff --git a/packages/jest-plugin/test/integration/src/ignoreFailures.test.ts b/packages/jest-plugin/test/integration/src/ignore-failures.test.ts similarity index 100% rename from packages/jest-plugin/test/integration/src/ignoreFailures.test.ts rename to packages/jest-plugin/test/integration/src/ignore-failures.test.ts diff --git a/packages/jest-plugin/test/integration/src/longNames.test.ts b/packages/jest-plugin/test/integration/src/long-names.test.ts similarity index 100% rename from packages/jest-plugin/test/integration/src/longNames.test.ts rename to packages/jest-plugin/test/integration/src/long-names.test.ts diff --git a/packages/jest-plugin/test/integration/src/noQuarantine.test.ts b/packages/jest-plugin/test/integration/src/no-quarantine.test.ts similarity index 100% rename from packages/jest-plugin/test/integration/src/noQuarantine.test.ts rename to packages/jest-plugin/test/integration/src/no-quarantine.test.ts diff --git a/packages/jest-plugin/test/integration/src/pluginFailures.test.ts b/packages/jest-plugin/test/integration/src/plugin-failures.test.ts similarity index 100% rename from packages/jest-plugin/test/integration/src/pluginFailures.test.ts rename to packages/jest-plugin/test/integration/src/plugin-failures.test.ts diff --git a/packages/jest-plugin/test/integration/src/runTestCase.ts b/packages/jest-plugin/test/integration/src/run-test-case.ts similarity index 100% rename from packages/jest-plugin/test/integration/src/runTestCase.ts rename to packages/jest-plugin/test/integration/src/run-test-case.ts diff --git a/packages/jest-plugin/test/integration/src/skipTests.test.ts b/packages/jest-plugin/test/integration/src/skip-tests.test.ts similarity index 100% rename from packages/jest-plugin/test/integration/src/skipTests.test.ts rename to packages/jest-plugin/test/integration/src/skip-tests.test.ts diff --git a/packages/jest-plugin/test/integration/src/testNamePattern.test.ts b/packages/jest-plugin/test/integration/src/test-name-pattern.test.ts similarity index 100% rename from packages/jest-plugin/test/integration/src/testNamePattern.test.ts rename to packages/jest-plugin/test/integration/src/test-name-pattern.test.ts diff --git a/packages/jest-plugin/test/integration/src/test-wrappers.ts b/packages/jest-plugin/test/integration/src/test-wrappers.ts index 243e4a9..b5394e0 100644 --- a/packages/jest-plugin/test/integration/src/test-wrappers.ts +++ b/packages/jest-plugin/test/integration/src/test-wrappers.ts @@ -2,7 +2,7 @@ import jestPackage from "jest/package.json"; import path from "path"; -import { ResultCounts, runTestCase, TestCaseParams } from "./runTestCase"; +import { ResultCounts, runTestCase, TestCaseParams } from "./run-test-case"; import { MockBackend } from "unflakable-test-common/dist/mock-backend"; import * as util from "util"; diff --git a/packages/jest-plugin/test/integration/src/verify-output.ts b/packages/jest-plugin/test/integration/src/verify-output.ts index bbcb461..a5fc020 100644 --- a/packages/jest-plugin/test/integration/src/verify-output.ts +++ b/packages/jest-plugin/test/integration/src/verify-output.ts @@ -2,7 +2,7 @@ // These are the chalk-formatted strings that include console color codes. import escapeStringRegexp from "escape-string-regexp"; -import { MOCK_RUN_ID, ResultCounts, TestCaseParams } from "./runTestCase"; +import { MOCK_RUN_ID, ResultCounts, TestCaseParams } from "./run-test-case"; import { TestAttemptResult } from "@unflakable/js-api"; const FAIL = From 0275c28f2045cdc0d7ce7dbe970281a87ce60ae2 Mon Sep 17 00:00:00 2001 From: "David A. Ramos" Date: Sat, 8 Jul 2023 22:16:01 -0700 Subject: [PATCH 07/53] [cypress] Avoid mockBackend singleton in integration tests --- .../test/integration/src/basic-matrix.test.ts | 3 ++- .../test/integration/src/basic.test.ts | 5 ++++- .../test/integration/src/config.test.ts | 6 +++++- .../test/integration/src/disable-plugin.test.ts | 5 ++++- .../test/integration/src/disable-upload.test.ts | 6 +++++- .../test/integration/src/git.test.ts | 8 +++++++- .../test/integration/src/hook-failures.test.ts | 17 ++++++++++++++++- .../test/integration/src/long-names.test.ts | 4 +++- .../test/integration/src/no-quarantine.test.ts | 3 ++- .../integration/src/plugin-failures.test.ts | 5 ++++- .../test/integration/src/retries.test.ts | 5 ++++- .../test/integration/src/run-test-case.ts | 7 ++++--- .../test/integration/src/test-wrappers.ts | 15 +++++++++++---- .../test/integration/src/unicode.test.ts | 4 +++- 14 files changed, 74 insertions(+), 19 deletions(-) diff --git a/packages/cypress-plugin/test/integration/src/basic-matrix.test.ts b/packages/cypress-plugin/test/integration/src/basic-matrix.test.ts index e8b29ca..60afb14 100644 --- a/packages/cypress-plugin/test/integration/src/basic-matrix.test.ts +++ b/packages/cypress-plugin/test/integration/src/basic-matrix.test.ts @@ -10,7 +10,7 @@ import { QuarantineMode } from "@unflakable/plugins-common"; import { afterEach, beforeEach } from "@jest/globals"; import * as fs from "fs/promises"; -integrationTestSuite(() => { +integrationTestSuite((mockBackend) => { Object.entries(TEST_PROJECTS).forEach(([projectName, project]) => { describe( projectName === "integration-input" @@ -84,6 +84,7 @@ integrationTestSuite(() => { } : defaultSummaryTotals, }, + mockBackend, done ) ); diff --git a/packages/cypress-plugin/test/integration/src/basic.test.ts b/packages/cypress-plugin/test/integration/src/basic.test.ts index 7809d01..9435b3c 100644 --- a/packages/cypress-plugin/test/integration/src/basic.test.ts +++ b/packages/cypress-plugin/test/integration/src/basic.test.ts @@ -7,7 +7,7 @@ import { } from "./test-wrappers"; import { QuarantineMode } from "@unflakable/plugins-common"; -integrationTestSuite(() => { +integrationTestSuite((mockBackend) => { it("quarantine flaky test", (done) => integrationTest( { @@ -21,6 +21,7 @@ integrationTestSuite(() => { numQuarantined: 5, }, }, + mockBackend, done )); @@ -43,6 +44,7 @@ integrationTestSuite(() => { numTests: 19, }, }, + mockBackend, done )); @@ -76,6 +78,7 @@ integrationTestSuite(() => { numTests: 19, }, }, + mockBackend, done ) ); diff --git a/packages/cypress-plugin/test/integration/src/config.test.ts b/packages/cypress-plugin/test/integration/src/config.test.ts index 2f285d8..fb40b62 100644 --- a/packages/cypress-plugin/test/integration/src/config.test.ts +++ b/packages/cypress-plugin/test/integration/src/config.test.ts @@ -2,7 +2,7 @@ import { integrationTestSuite, integrationTest } from "./test-wrappers"; -integrationTestSuite(() => { +integrationTestSuite((mockBackend) => { it("set test suite ID via config", (done) => integrationTest( { @@ -16,6 +16,7 @@ integrationTestSuite(() => { expectedSuiteId: "MOCK_SUITE_ID_CONFIG", }, }, + mockBackend, done )); @@ -33,6 +34,7 @@ integrationTestSuite(() => { expectedSuiteId: "MOCK_SUITE_ID_ENV", }, }, + mockBackend, done )); @@ -47,6 +49,7 @@ integrationTestSuite(() => { expectedSuiteId: "MOCK_SUITE_ID_CLI", }, }, + mockBackend, done )); @@ -65,6 +68,7 @@ integrationTestSuite(() => { expectedSuiteId: "MOCK_SUITE_ID_CLI", }, }, + mockBackend, done )); }); diff --git a/packages/cypress-plugin/test/integration/src/disable-plugin.test.ts b/packages/cypress-plugin/test/integration/src/disable-plugin.test.ts index 463c9dc..59eb9e1 100644 --- a/packages/cypress-plugin/test/integration/src/disable-plugin.test.ts +++ b/packages/cypress-plugin/test/integration/src/disable-plugin.test.ts @@ -6,7 +6,7 @@ import { integrationTestSuite, } from "./test-wrappers"; -integrationTestSuite(() => { +integrationTestSuite((mockBackend) => { const expectedExitCodeWithPluginDisabled = 11; const summaryTotalsWithPluginDisabled = { ...defaultSummaryTotals, @@ -27,6 +27,7 @@ integrationTestSuite(() => { expectedExitCode: expectedExitCodeWithPluginDisabled, summaryTotals: summaryTotalsWithPluginDisabled, }, + mockBackend, done )); @@ -42,6 +43,7 @@ integrationTestSuite(() => { expectedExitCode: expectedExitCodeWithPluginDisabled, summaryTotals: summaryTotalsWithPluginDisabled, }, + mockBackend, done )); @@ -57,6 +59,7 @@ integrationTestSuite(() => { }, }, }, + mockBackend, done )); }); diff --git a/packages/cypress-plugin/test/integration/src/disable-upload.test.ts b/packages/cypress-plugin/test/integration/src/disable-upload.test.ts index f9171e3..14b1b3a 100644 --- a/packages/cypress-plugin/test/integration/src/disable-upload.test.ts +++ b/packages/cypress-plugin/test/integration/src/disable-upload.test.ts @@ -2,7 +2,7 @@ import { integrationTest, integrationTestSuite } from "./test-wrappers"; -integrationTestSuite(() => { +integrationTestSuite((mockBackend) => { it("disable upload via config", (done) => integrationTest( { @@ -13,6 +13,7 @@ integrationTestSuite(() => { expectResultsToBeUploaded: false, }, }, + mockBackend, done )); @@ -26,6 +27,7 @@ integrationTestSuite(() => { expectResultsToBeUploaded: false, }, }, + mockBackend, done )); @@ -41,6 +43,7 @@ integrationTestSuite(() => { expectResultsToBeUploaded: false, }, }, + mockBackend, done )); @@ -56,6 +59,7 @@ integrationTestSuite(() => { }, }, }, + mockBackend, done )); }); diff --git a/packages/cypress-plugin/test/integration/src/git.test.ts b/packages/cypress-plugin/test/integration/src/git.test.ts index 26290db..5546179 100644 --- a/packages/cypress-plugin/test/integration/src/git.test.ts +++ b/packages/cypress-plugin/test/integration/src/git.test.ts @@ -3,7 +3,7 @@ import { integrationTest, integrationTestSuite } from "./test-wrappers"; import path from "path"; -integrationTestSuite(() => { +integrationTestSuite((mockBackend) => { it("no git repo", (done) => integrationTest( { @@ -17,6 +17,7 @@ integrationTestSuite(() => { }, }, }, + mockBackend, done )); @@ -47,6 +48,7 @@ integrationTestSuite(() => { expectedBranch: "pull/MOCK_PR_NUMBER/merge", }, }, + mockBackend, done )); @@ -62,6 +64,7 @@ integrationTestSuite(() => { expectedCommit: "MOCK_COMMIT_ENV", }, }, + mockBackend, done )); @@ -84,6 +87,7 @@ integrationTestSuite(() => { expectedCommit: "MOCK_COMMIT_CLI", }, }, + mockBackend, done )); @@ -100,6 +104,7 @@ integrationTestSuite(() => { expectedRepoRelativePathPrefix: "", }, }, + mockBackend, done )); @@ -114,6 +119,7 @@ integrationTestSuite(() => { expectedRepoRelativePathPrefix: "", }, }, + mockBackend, done )); }); diff --git a/packages/cypress-plugin/test/integration/src/hook-failures.test.ts b/packages/cypress-plugin/test/integration/src/hook-failures.test.ts index 224384e..013f65b 100644 --- a/packages/cypress-plugin/test/integration/src/hook-failures.test.ts +++ b/packages/cypress-plugin/test/integration/src/hook-failures.test.ts @@ -2,7 +2,7 @@ import { integrationTest, integrationTestSuite } from "./test-wrappers"; -integrationTestSuite(() => { +integrationTestSuite((mockBackend) => { it("run should succeed when before() fails both tests are quarantined", (done) => integrationTest( { @@ -23,6 +23,7 @@ integrationTestSuite(() => { numTests: 2, }, }, + mockBackend, done )); @@ -48,6 +49,7 @@ integrationTestSuite(() => { numTests: 2, }, }, + mockBackend, done )); @@ -73,6 +75,7 @@ integrationTestSuite(() => { numTests: 2, }, }, + mockBackend, done )); @@ -95,6 +98,7 @@ integrationTestSuite(() => { numTests: 2, }, }, + mockBackend, done )); @@ -118,6 +122,7 @@ integrationTestSuite(() => { numTests: 2, }, }, + mockBackend, done )); @@ -141,6 +146,7 @@ integrationTestSuite(() => { numTests: 2, }, }, + mockBackend, done )); @@ -164,6 +170,7 @@ integrationTestSuite(() => { numTests: 2, }, }, + mockBackend, done )); @@ -188,6 +195,7 @@ integrationTestSuite(() => { numTests: 2, }, }, + mockBackend, done )); @@ -212,6 +220,7 @@ integrationTestSuite(() => { numTests: 2, }, }, + mockBackend, done )); @@ -238,6 +247,7 @@ integrationTestSuite(() => { numTests: 2, }, }, + mockBackend, done )); @@ -262,6 +272,7 @@ integrationTestSuite(() => { numTests: 2, }, }, + mockBackend, done )); @@ -288,6 +299,7 @@ integrationTestSuite(() => { numTests: 2, }, }, + mockBackend, done )); @@ -310,6 +322,7 @@ integrationTestSuite(() => { numTests: 2, }, }, + mockBackend, done )); @@ -334,6 +347,7 @@ integrationTestSuite(() => { numTests: 2, }, }, + mockBackend, done )); @@ -358,6 +372,7 @@ integrationTestSuite(() => { numTests: 2, }, }, + mockBackend, done )); }); diff --git a/packages/cypress-plugin/test/integration/src/long-names.test.ts b/packages/cypress-plugin/test/integration/src/long-names.test.ts index 1a10435..9d18150 100644 --- a/packages/cypress-plugin/test/integration/src/long-names.test.ts +++ b/packages/cypress-plugin/test/integration/src/long-names.test.ts @@ -7,7 +7,7 @@ import { } from "./test-wrappers"; import { QuarantineMode } from "@unflakable/plugins-common"; -integrationTestSuite(() => { +integrationTestSuite((mockBackend) => { it.each(["ignore_failures", "skip_tests"] as QuarantineMode[])( "test names longer than 4096 chars should be truncated w/ quarantineMode = %s", (quarantineMode, done) => @@ -35,6 +35,7 @@ integrationTestSuite(() => { } : defaultSummaryTotals, }, + mockBackend, done ) ); @@ -66,6 +67,7 @@ integrationTestSuite(() => { numQuarantined: quarantineMode === "skip_tests" ? 6 : 5, }, }, + mockBackend, done ) ); diff --git a/packages/cypress-plugin/test/integration/src/no-quarantine.test.ts b/packages/cypress-plugin/test/integration/src/no-quarantine.test.ts index 6bd9b90..3710a9c 100644 --- a/packages/cypress-plugin/test/integration/src/no-quarantine.test.ts +++ b/packages/cypress-plugin/test/integration/src/no-quarantine.test.ts @@ -6,7 +6,7 @@ import { integrationTestSuite, } from "./test-wrappers"; -integrationTestSuite(() => { +integrationTestSuite((mockBackend) => { it.each(["CLI", "config"] as ("CLI" | "config")[])( "set quarantineMode to no_quarantine via %s", (mode, done) => @@ -35,6 +35,7 @@ integrationTestSuite(() => { numFlaky: 3, }, }, + mockBackend, done ) ); diff --git a/packages/cypress-plugin/test/integration/src/plugin-failures.test.ts b/packages/cypress-plugin/test/integration/src/plugin-failures.test.ts index 823d9b6..b6434d7 100644 --- a/packages/cypress-plugin/test/integration/src/plugin-failures.test.ts +++ b/packages/cypress-plugin/test/integration/src/plugin-failures.test.ts @@ -6,7 +6,7 @@ import { integrationTestSuite, } from "./test-wrappers"; -integrationTestSuite(() => { +integrationTestSuite((mockBackend) => { it("run should not fail due to error fetching manifest", (done) => integrationTest( { @@ -31,6 +31,7 @@ integrationTestSuite(() => { numTests: 19, }, }, + mockBackend, done )); @@ -42,6 +43,7 @@ integrationTestSuite(() => { }, expectedExitCode: 1, }, + mockBackend, done )); @@ -61,6 +63,7 @@ integrationTestSuite(() => { numQuarantined: 0, }, }, + mockBackend, done )); }); diff --git a/packages/cypress-plugin/test/integration/src/retries.test.ts b/packages/cypress-plugin/test/integration/src/retries.test.ts index 317a5a2..bc99335 100644 --- a/packages/cypress-plugin/test/integration/src/retries.test.ts +++ b/packages/cypress-plugin/test/integration/src/retries.test.ts @@ -6,7 +6,7 @@ import { integrationTestSuite, } from "./test-wrappers"; -integrationTestSuite(() => { +integrationTestSuite((mockBackend) => { describe.each(["CLI", "config"] as ("CLI" | "config")[])( "set failureRetries via %s", (mode) => { @@ -32,6 +32,7 @@ integrationTestSuite(() => { numFailing: 8, }, }, + mockBackend, done )); @@ -51,6 +52,7 @@ integrationTestSuite(() => { expectedRetries: 1, }, }, + mockBackend, done )); @@ -70,6 +72,7 @@ integrationTestSuite(() => { expectedRetries: 3, }, }, + mockBackend, done )); } diff --git a/packages/cypress-plugin/test/integration/src/run-test-case.ts b/packages/cypress-plugin/test/integration/src/run-test-case.ts index 5bf324d..ad3399b 100644 --- a/packages/cypress-plugin/test/integration/src/run-test-case.ts +++ b/packages/cypress-plugin/test/integration/src/run-test-case.ts @@ -166,8 +166,6 @@ export const specPattern = (params: TestCaseParams): string => { const specRepoPath = (params: TestCaseParams, specNameStub: string): string => params.expectedRepoRelativePathPrefix + specProjectPath(params, specNameStub); -export const mockBackend = new MockBackend(); - export const MOCK_RUN_ID = "MOCK_RUN_ID"; const TIMESTAMP_REGEX = /^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{3}Z$/; @@ -445,6 +443,7 @@ const verifyUploadResults = ( const addBackendExpectations = async ( params: TestCaseParams, summaryTotals: SummaryTotals, + mockBackend: MockBackend, onError: (e: unknown) => void ): Promise => { const { @@ -585,7 +584,8 @@ _debug.formatArgs = formatDebugArgsWithTimestamp; export const runTestCase = async ( params: TestCaseParams, expectedExitCode: number, - summaryTotals: SummaryTotals + summaryTotals: SummaryTotals, + mockBackend: MockBackend ): Promise => { const { skipFailures, @@ -604,6 +604,7 @@ export const runTestCase = async ( const unmatchedRequestEndpoints = await addBackendExpectations( params, summaryTotals, + mockBackend, (error) => { if (asyncTestError.error === undefined) { asyncTestError.error = error ?? new Error("undefined error"); diff --git a/packages/cypress-plugin/test/integration/src/test-wrappers.ts b/packages/cypress-plugin/test/integration/src/test-wrappers.ts index f0f420d..090ecaa 100644 --- a/packages/cypress-plugin/test/integration/src/test-wrappers.ts +++ b/packages/cypress-plugin/test/integration/src/test-wrappers.ts @@ -1,11 +1,12 @@ // Copyright (c) 2023 Developer Innovations, LLC -import { mockBackend, runTestCase, TestCaseParams } from "./run-test-case"; +import { runTestCase, TestCaseParams } from "./run-test-case"; import path from "path"; import cypressPackage from "cypress/package.json"; import { SummaryTotals } from "./parse-output"; import * as os from "os"; import * as util from "util"; +import { MockBackend } from "unflakable-test-common/dist/mock-backend"; declare global { // eslint-disable-next-line @typescript-eslint/no-namespace @@ -39,6 +40,7 @@ export const defaultSummaryTotals: SummaryTotals = { export const integrationTest = ( testCase: TestCase, + mockBackend: MockBackend, done: jest.DoneCallback ): void => { void runTestCase( @@ -99,7 +101,8 @@ export const integrationTest = ( }, }, testCase.expectedExitCode ?? defaultExitCode, - testCase.summaryTotals ?? defaultSummaryTotals + testCase.summaryTotals ?? defaultSummaryTotals, + mockBackend ) .then(done) .catch((e) => { @@ -108,7 +111,11 @@ export const integrationTest = ( }); }; -export const integrationTestSuite = (runTests: () => void): void => { +export const integrationTestSuite = ( + runTests: (mockBackend: MockBackend) => void +): void => { + const mockBackend = new MockBackend(); + beforeEach(() => mockBackend.start()); afterEach(() => mockBackend.stop()); @@ -134,7 +141,7 @@ export const integrationTestSuite = (runTests: () => void): void => { describe(`Node ${ nodeMajorVersion !== null ? nodeMajorVersion[0] : process.version }`, () => { - runTests(); + runTests(mockBackend); }); } ); diff --git a/packages/cypress-plugin/test/integration/src/unicode.test.ts b/packages/cypress-plugin/test/integration/src/unicode.test.ts index be9a10a..a400392 100644 --- a/packages/cypress-plugin/test/integration/src/unicode.test.ts +++ b/packages/cypress-plugin/test/integration/src/unicode.test.ts @@ -6,7 +6,7 @@ import { integrationTestSuite, } from "./test-wrappers"; -integrationTestSuite(() => { +integrationTestSuite((mockBackend) => { it("emoji test names should be allowed", (done) => integrationTest( { @@ -17,6 +17,7 @@ integrationTestSuite(() => { }, }, }, + mockBackend, done )); @@ -37,6 +38,7 @@ integrationTestSuite(() => { numQuarantined: 5, }, }, + mockBackend, done )); }); From 7bcefbee5d796f86d81fa570dcb69636f27b4ef7 Mon Sep 17 00:00:00 2001 From: "David A. Ramos" Date: Sun, 9 Jul 2023 14:13:36 -0700 Subject: [PATCH 08/53] [jest] Update Jest peerDependencies Previously, the peerDependencies listed specific Jest sub-packages required by the plugin, but this led to yarn warnings caused by those packages being transitive dependencies rather than top-level dependencies. Since those dependencies are implied by having Jest as a dependency, we simply include `jest` as the sole peer dependency. --- packages/jest-plugin/package.json | 5 +---- yarn.lock | 5 +---- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/packages/jest-plugin/package.json b/packages/jest-plugin/package.json index 98f312d..a7bd454 100644 --- a/packages/jest-plugin/package.json +++ b/packages/jest-plugin/package.json @@ -57,10 +57,7 @@ "typescript": "^4.9.5" }, "peerDependencies": { - "@jest/console": "25.1.0 - 29", - "@jest/reporters": "25.1.0 - 29", - "jest-runner": "25.1.0 - 29", - "jest-util": "25.1.0 - 29" + "jest": "25.1.0 - 29" }, "scripts": { "build": "yarn clean && tsc --noEmit && rollup --config", diff --git a/yarn.lock b/yarn.lock index 7ac906b..4d35e41 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3249,10 +3249,7 @@ __metadata: simple-git: ^3.16.0 typescript: ^4.9.5 peerDependencies: - "@jest/console": 25.1.0 - 29 - "@jest/reporters": 25.1.0 - 29 - jest-runner: 25.1.0 - 29 - jest-util: 25.1.0 - 29 + jest: 25.1.0 - 29 languageName: unknown linkType: soft From 6cdfce156e6cfb69523ff8b7452ade8149ea7443 Mon Sep 17 00:00:00 2001 From: "David A. Ramos" Date: Sun, 9 Jul 2023 14:31:48 -0700 Subject: [PATCH 09/53] Rename CI build step to Build Now that it's producing artifacts consumed by the tests and used for releases, this is a more appropriate name. --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 307d146..73332d2 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -16,7 +16,7 @@ on: jobs: check: - name: Typecheck, lint, and audit + name: Build runs-on: ubuntu-latest timeout-minutes: 10 steps: From de105123ad5c6d5b672afc0f06008a3272680757 Mon Sep 17 00:00:00 2001 From: "David A. Ramos" Date: Sun, 9 Jul 2023 01:12:18 -0700 Subject: [PATCH 10/53] [jest] Add Windows support --- .github/workflows/ci.yaml | 126 +++++++++++++++++- packages/jest-plugin/src/reporter.ts | 3 +- packages/jest-plugin/src/runner.ts | 10 +- .../test/integration/jest.config.js | 2 +- .../test/integration/src/run-test-case.ts | 13 +- .../test/integration/src/test-wrappers.ts | 25 +++- .../test/integration/src/verify-output.ts | 9 +- scripts/set-jest-version.ts | 27 +++- 8 files changed, 188 insertions(+), 27 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 73332d2..ef0019f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -288,12 +288,16 @@ jobs: # Make chalk emit TTY colors. FORCE_COLOR: "1" - # FIXME: use Jest plugin once it works on Windows - run: yarn workspace cypress-integration test - jest_integration_tests: + UNFLAKABLE_API_KEY: ${{ secrets.UNFLAKABLE_API_KEY }} + UNFLAKABLE_SUITE_ID: ${{ github.repository == 'unflakable/unflakable-javascript' && '2QwtGckRudLNUGBsdkVEoSknck1' || '2Qwt9RyPIbOI95C6qjXCzcTelni' }} + run: | + yarn workspace cypress-integration test ` + --reporters @unflakable/jest-plugin/dist/reporter ` + --runner @unflakable/jest-plugin/dist/runner + + jest_linux_integration_tests: name: "Jest ${{ matrix.jest }} Linux Node ${{ matrix.node }} Integration Tests" - # FIXME: also test on Windows runs-on: ubuntu-latest timeout-minutes: 20 needs: @@ -391,6 +395,12 @@ jobs: # Enable debug logs within the Jest plugin. TEST_DEBUG: "unflakable:*" + + # Enable terminal colors for debug() output. + DEBUG_COLORS: "1" + + # Make chalk emit TTY colors. + FORCE_COLOR: "1" run: | if [ "${{ github.repository }}" == "unflakable/unflakable-javascript" ]; then export UNFLAKABLE_SUITE_ID=29KWCuK12VnU7pkpvWgrGS0woAX @@ -401,3 +411,111 @@ jobs: yarn workspace jest-integration test \ --reporters @unflakable/jest-plugin/dist/reporter \ --runner @unflakable/jest-plugin/dist/runner + + jest_windows_integration_tests: + name: "Jest ${{ matrix.jest }} Windows Node ${{ matrix.node }} Integration Tests" + runs-on: windows-2019 + timeout-minutes: 30 + needs: + # Don't incur the cost of the test matrix if the basic build fails. + - check + if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || needs.check.outputs.affects_jest == 'true' + strategy: + fail-fast: false + matrix: + node: + - 16 + - 18 + - 20 + jest: + - "29.6" + - "29.5" + - "29.4" + - "29.3" + - "29.2" + - "29.1" + - "29.0" + - "28.1" + - "28.0" + - "27.5" + - "27.4" + - "27.3" + - "27.2" + - "27.1" + - "27.0" + - "26.6" + - "26.5" + - "26.4" + - "26.3" + - "26.2" + - "26.1" + - "26.0" + - "25.5" + - "25.4" + - "25.3" + - "25.2" + - "25.1" + env: + CYPRESS_INSTALL_BINARY: "0" + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.node }} + cache: yarn + + - id: install + run: yarn install --immutable + + - uses: actions/download-artifact@v3 + with: + path: .artifacts + + - name: Install pre-built plugin packages + shell: bash + run: | + curl -Lo jq.exe https://github.com/jqlang/jq/releases/download/jq-1.6/jq-win64.exe + cat package.json \ + | ./jq.exe '. + {"resolutions": (.resolutions + { + "@unflakable/jest-plugin": "file:./.artifacts/jest-plugin/package.tgz", + "@unflakable/js-api": "file:./.artifacts/js-api/package.tgz" + })}' > package-new.json + mv package-new.json package.json + yarn install --no-immutable + + - name: Set Jest version + run: | + yarn set-jest-version ${{ matrix.jest }} + Select-String -Pattern '^".*jest.*' -Path yarn.lock -Context 0,1 + + - name: Set resolution for chalk to 3.0.0 + if: ${{ startsWith(matrix.jest, '25.') }} + run: yarn set resolution "chalk@npm:^3.0.0 || ^4.0.0" 3.0 + + - name: Build test dependencies + run: yarn build:plugins-common && yarn build:test-common + + - name: Test + env: + # Enable debug logs within the jest-plugin/test/integration Jest tests that invoke Jest + # on the jest-plugin/test/integration-input test cases. WARNING: these are very verbose + # but are useful for seeing the raw chalk terminal codes. + # DEBUG: unflakable:* + + # Enable debug logs within the Jest plugin. + TEST_DEBUG: "unflakable:*" + + # Enable terminal colors for debug() output. + DEBUG_COLORS: "1" + + # Make chalk emit TTY colors. + FORCE_COLOR: "1" + + UNFLAKABLE_API_KEY: ${{ secrets.UNFLAKABLE_API_KEY }} + UNFLAKABLE_SUITE_ID: ${{ github.repository == 'unflakable/unflakable-javascript' && '29KWCuK12VnU7pkpvWgrGS0woAX' || '28UidZ8cSKjRe4g1xkd9EE8noDF' }} + run: | + yarn workspace jest-integration test ` + --reporters @unflakable/jest-plugin/dist/reporter ` + --runner @unflakable/jest-plugin/dist/runner diff --git a/packages/jest-plugin/src/reporter.ts b/packages/jest-plugin/src/reporter.ts index d3e47cd..d10bfa1 100644 --- a/packages/jest-plugin/src/reporter.ts +++ b/packages/jest-plugin/src/reporter.ts @@ -42,6 +42,7 @@ import { loadApiKey, loadConfigSync, loadGitRepo, + toPosix, UnflakableConfig, } from "@unflakable/plugins-common"; @@ -434,7 +435,7 @@ export default class UnflakableReporter extends BaseReporter { ) .map( ([, assertionResults]): TestRunRecord => ({ - filename: path.relative(repoRoot, testFilePath), + filename: toPosix(path.relative(repoRoot, testFilePath)), name: testKey(assertionResults[0]), attempts: assertionResults .map((testResult: UnflakableAssertionResult) => ({ diff --git a/packages/jest-plugin/src/runner.ts b/packages/jest-plugin/src/runner.ts index 75d3e18..b87ff3d 100644 --- a/packages/jest-plugin/src/runner.ts +++ b/packages/jest-plugin/src/runner.ts @@ -32,6 +32,7 @@ import { loadApiKey, loadConfigSync, loadGitRepo, + toPosix, QuarantineMode, UnflakableConfig, } from "@unflakable/plugins-common"; @@ -59,7 +60,7 @@ const wrapOnResult = async (test: Test, testResult: TestResult): Promise => { const testResults = testResult.testResults.map( (assertionResult: AssertionResult): UnflakableAssertionResult => { - const testFilename = path.relative(repoRoot, test.path); + const testFilename = toPosix(path.relative(repoRoot, test.path)); if (assertionResult.status === FAILED) { if (manifest === undefined) { debug( @@ -411,7 +412,7 @@ class UnflakableRunner { // Then, we run the remaining test files normally below. await tests .reduce((promise, test) => { - const relPath = path.relative(repoRoot, test.path); + const relPath = toPosix(path.relative(repoRoot, test.path)); const quarantinedTestsInFile = quarantinedTestsByFile[relPath]; if ( quarantinedTestsInFile !== undefined && @@ -462,7 +463,10 @@ class UnflakableRunner { .then(() => { const normalTestFiles = tests.filter( (test) => - !(path.relative(repoRoot, test.path) in quarantinedTestsByFile) + !( + toPosix(path.relative(repoRoot, test.path)) in + quarantinedTestsByFile + ) ); if (normalTestFiles.length > 0) { debug( diff --git a/packages/jest-plugin/test/integration/jest.config.js b/packages/jest-plugin/test/integration/jest.config.js index 33e1278..1e3441c 100644 --- a/packages/jest-plugin/test/integration/jest.config.js +++ b/packages/jest-plugin/test/integration/jest.config.js @@ -13,7 +13,7 @@ module.exports = { }, // NB: This should be greater than TEST_TIMEOUT_MS used by the watchdog in runTestCase(). - testTimeout: 40000, + testTimeout: 120000, verbose: true, }; diff --git a/packages/jest-plugin/test/integration/src/run-test-case.ts b/packages/jest-plugin/test/integration/src/run-test-case.ts index ac9fc49..0e9cb36 100644 --- a/packages/jest-plugin/test/integration/src/run-test-case.ts +++ b/packages/jest-plugin/test/integration/src/run-test-case.ts @@ -29,9 +29,9 @@ import { spawnTestWithTimeout, } from "unflakable-test-common/dist/spawn"; -// Jest times out after 40 seconds, so we bail early here to allow time to print the +// Jest times out after 120 seconds, so we bail early here to allow time to print the // captured output before Jest kills the test. -const TEST_TIMEOUT_MS = 30000; +const TEST_TIMEOUT_MS = 110000; const userAgentRegex = new RegExp( "unflakable-js-api/(?:[-0-9.]|alpha|beta)+ unflakable-jest-plugin/(?:[-0-9.]|alpha|beta)+ \\(Jest [0-9]+\\.[0-9]+\\.[0-9]+; Node v[0-9]+\\.[0-9]+\\.[0-9]\\)" @@ -481,7 +481,14 @@ export const runTestCase = async ( params.config !== null ? { config: params.config, - filepath: "MOCK_BASE/packages/jest-plugin/test/unflakable.yml", + filepath: path.join( + "MOCK_BASE", + "packages", + "jest-plugin", + "test", + "integration-input", + "package.json" + ), } : null, }; diff --git a/packages/jest-plugin/test/integration/src/test-wrappers.ts b/packages/jest-plugin/test/integration/src/test-wrappers.ts index b5394e0..baf2dd2 100644 --- a/packages/jest-plugin/test/integration/src/test-wrappers.ts +++ b/packages/jest-plugin/test/integration/src/test-wrappers.ts @@ -4,6 +4,7 @@ import jestPackage from "jest/package.json"; import path from "path"; import { ResultCounts, runTestCase, TestCaseParams } from "./run-test-case"; import { MockBackend } from "unflakable-test-common/dist/mock-backend"; +import * as os from "os"; import * as util from "util"; export type TestCase = { @@ -98,11 +99,23 @@ export const integrationTestSuite = ( describe(`Jest ${ jestMinorVersion !== null ? jestMinorVersion[0] : jestPackage.version }`, () => { - // Only use Node major version for test name. - describe(`Node ${ - nodeMajorVersion !== null ? nodeMajorVersion[0] : process.version - }`, () => { - runTests(mockBackend); - }); + const platform = os.platform(); + describe( + platform === "darwin" + ? `MacOS` + : platform === "linux" + ? "Linux" + : platform === "win32" + ? "Windows" + : platform, + () => { + // Only use Node major version for test name. + describe(`Node ${ + nodeMajorVersion !== null ? nodeMajorVersion[0] : process.version + }`, () => { + runTests(mockBackend); + }); + } + ); }); }; diff --git a/packages/jest-plugin/test/integration/src/verify-output.ts b/packages/jest-plugin/test/integration/src/verify-output.ts index a5fc020..a664ee4 100644 --- a/packages/jest-plugin/test/integration/src/verify-output.ts +++ b/packages/jest-plugin/test/integration/src/verify-output.ts @@ -5,6 +5,9 @@ import escapeStringRegexp from "escape-string-regexp"; import { MOCK_RUN_ID, ResultCounts, TestCaseParams } from "./run-test-case"; import { TestAttemptResult } from "@unflakable/js-api"; +const FAIL_SYMBOL = process.platform === "win32" ? "×" : "✕"; +const PASS_SYMBOL = process.platform === "win32" ? "√" : "✓"; + const FAIL = "\u001b[0m\u001b[7m\u001b[1m\u001b[31m FAIL \u001b[39m\u001b[22m\u001b[27m\u001b[0m"; const PASS = @@ -23,13 +26,13 @@ const testResultRegexMatch = ( `^${" ".repeat(indent ?? 4)}${escapeStringRegexp( result === "pass" ? // Green - "\u001b[32m✓\u001b[39m" + `\u001b[32m${PASS_SYMBOL}\u001b[39m` : result === "fail" ? // Red - "\u001b[31m✕\u001b[39m" + `\u001b[31m${FAIL_SYMBOL}\u001b[39m` : result === "quarantined" ? // Yellow - "\u001b[33m✕\u001b[39m" + `\u001b[33m${FAIL_SYMBOL}\u001b[39m` : result === "skipped" ? // Yellow "\u001b[33m○\u001b[39m" diff --git a/scripts/set-jest-version.ts b/scripts/set-jest-version.ts index 56a614d..b12f92b 100644 --- a/scripts/set-jest-version.ts +++ b/scripts/set-jest-version.ts @@ -1,6 +1,6 @@ // Copyright (c) 2022-2023 Developer Innovations, LLC -import { spawnSync } from "child_process"; +import { execSync, spawnSync } from "child_process"; import * as fs from "fs"; import * as yaml from "js-yaml"; import * as semver from "semver"; @@ -20,16 +20,28 @@ type PackageSpec = { }; type Lockfile = { [key in string]: PackageSpec }; -const setYarnResolution = (descriptor: string, resolution: string): void => { +const getYarnPath = (): string => + execSync("yarn config get yarnPath").toString().trimEnd(); + +const setYarnResolution = ( + yarnPath: string, + descriptor: string, + resolution: string +): void => { debug(`Setting yarn resolution \`${descriptor}\` to ${resolution}`); const outcome = spawnSync( - "yarn", - ["set", "resolution", descriptor, resolution], + "node", + // For some reason, yarn.cmd doesn't update the lockfile on Windows, but running node explicitly + // with the path to yarn does. + [yarnPath, "set", "resolution", descriptor, resolution], { stdio: "inherit" } ); - if (outcome.status !== 0) { + if (outcome.error !== undefined) { + console.error("ERROR: Failed to run yarn: %o", outcome.error); + process.exit(1); + } else if (outcome.status !== 0) { console.error("ERROR: Exiting due to yarn error"); process.exit(1); } @@ -222,6 +234,8 @@ const main = (): never => { ); } + const yarnPath = getYarnPath(); + const maxIterations = 10; // The jest TS types aren't released very frequently and mostly correspond to major versions, so @@ -231,6 +245,7 @@ const main = (): never => { targetSemVerMinVersion.major <= 29 ) { setYarnResolution( + yarnPath, "@types/jest@npm:25.1.0 - 29", targetSemVerMinVersion.major.toString() ); @@ -297,7 +312,7 @@ const main = (): never => { done = false; - setYarnResolution(descriptor, packageTargetSemVerRange.raw); + setYarnResolution(yarnPath, descriptor, packageTargetSemVerRange.raw); }); } }); From 82c88926e9cc56f185dda80d3a702acc29ffec2e Mon Sep 17 00:00:00 2001 From: "David A. Ramos" Date: Mon, 10 Jul 2023 23:31:16 -0700 Subject: [PATCH 11/53] Check integration test exit code before output Certain Windows test runs appear to end abruptly with truncated output. Hopefully this will help identify the underlying cause. --- packages/test-common/src/spawn.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/test-common/src/spawn.ts b/packages/test-common/src/spawn.ts index f693c3c..099042c 100644 --- a/packages/test-common/src/spawn.ts +++ b/packages/test-common/src/spawn.ts @@ -138,10 +138,10 @@ export const spawnTestWithTimeout = async ( throw asyncTestError.error; } - await verifyOutput(stdoutLines, stderrLines); - expect(signal).toBe(null); expect(code).toBe(expectedExitCode); + + await verifyOutput(stdoutLines, stderrLines); } catch (e: unknown) { // Jest doesn't have a built-in setting for printing console logs only for failed tests, so we // just defer the output until this catch block and attach it to the error. See From 7083586da133542d9fef9ab2971a4d5e2b6503a6 Mon Sep 17 00:00:00 2001 From: "David A. Ramos" Date: Mon, 10 Jul 2023 23:32:25 -0700 Subject: [PATCH 12/53] Don't exit with missing suite ID when plugin is disabled --- packages/cypress-plugin/src/plugin.ts | 4 +++- packages/jest-plugin/src/reporter.ts | 4 +++- packages/plugins-common/src/config.ts | 30 ++++++++++++++++++++------- packages/plugins-common/src/index.ts | 1 + packages/test-common/src/config.ts | 2 +- packages/test-common/src/git.ts | 2 +- 6 files changed, 32 insertions(+), 11 deletions(-) diff --git a/packages/cypress-plugin/src/plugin.ts b/packages/cypress-plugin/src/plugin.ts index a28057e..b244d07 100644 --- a/packages/cypress-plugin/src/plugin.ts +++ b/packages/cypress-plugin/src/plugin.ts @@ -19,6 +19,7 @@ import { normalizeTestName, toPosix, UnflakableConfig, + UnflakableConfigEnabled, } from "@unflakable/plugins-common"; import { printWarning, require, userAgent } from "./utils"; import { configureMochaReporter } from "./reporter-config"; @@ -505,7 +506,8 @@ ${ end_time: new Date(results.endedTestsAt).toISOString(), test_runs: testRuns, }, - testSuiteId: this.unflakableConfig.testSuiteId, + testSuiteId: (this.unflakableConfig as UnflakableConfigEnabled) + .testSuiteId, apiKey: this.apiKey, baseUrl: this.unflakableConfig.apiBaseUrl, clientDescription: userAgentStr, diff --git a/packages/jest-plugin/src/reporter.ts b/packages/jest-plugin/src/reporter.ts index d10bfa1..c612fa5 100644 --- a/packages/jest-plugin/src/reporter.ts +++ b/packages/jest-plugin/src/reporter.ts @@ -44,6 +44,7 @@ import { loadGitRepo, toPosix, UnflakableConfig, + UnflakableConfigEnabled, } from "@unflakable/plugins-common"; const debug = _debug("unflakable:reporter"); @@ -414,7 +415,8 @@ export default class UnflakableReporter extends BaseReporter { aggregatedResults: AggregatedResult, unflakableConfig: UnflakableConfig ): Promise { - const testSuiteId = unflakableConfig.testSuiteId; + const testSuiteId = (unflakableConfig as UnflakableConfigEnabled) + .testSuiteId; const git = unflakableConfig.gitAutoDetect ? await loadGitRepo() : null; const repoRoot = git !== null ? await getRepoRoot(git) : this.rootDir; diff --git a/packages/plugins-common/src/config.ts b/packages/plugins-common/src/config.ts index 9d4dd8e..db11559 100644 --- a/packages/plugins-common/src/config.ts +++ b/packages/plugins-common/src/config.ts @@ -9,19 +9,30 @@ const debug = _debug("unflakable:plugins-common:config"); export type QuarantineMode = "no_quarantine" | "skip_tests" | "ignore_failures"; -export type UnflakableConfig = { +type UnflakableConfigInner = { apiBaseUrl: string | undefined; - enabled: boolean; failureRetries: number; gitAutoDetect: boolean; quarantineMode: QuarantineMode; - testSuiteId: string; uploadResults: boolean; }; -type UnflakableConfigFile = Omit & { +export type UnflakableConfigEnabled = { + enabled: true; + testSuiteId: string; +} & UnflakableConfigInner; + +export type UnflakableConfig = + | UnflakableConfigEnabled + | ({ + enabled: false; + testSuiteId: string | undefined; + } & UnflakableConfigInner); + +type UnflakableConfigFile = { + enabled: boolean; testSuiteId: string | undefined; -}; +} & UnflakableConfigInner; const defaultConfig: UnflakableConfigFile = { apiBaseUrl: undefined, @@ -196,12 +207,17 @@ const mergeConfigWithEnv = ( config.testSuiteId.length > 0 ) { debug(`Using suite ID \`${config.testSuiteId}\` from config file`); - return config as UnflakableConfig; - } else { + return config.enabled + ? // TypeScript has trouble inferring that these types are correct otherwise. + { ...config, enabled: true, testSuiteId: config.testSuiteId } + : { ...config, enabled: false }; + } else if (config.enabled) { throw new Error( `Unflakable test suite ID not found in config file or ${suiteIdOverride.name} environment ` + "variable" ); + } else { + return { ...config, enabled: false }; } }; diff --git a/packages/plugins-common/src/index.ts b/packages/plugins-common/src/index.ts index 9ea836e..851f379 100644 --- a/packages/plugins-common/src/index.ts +++ b/packages/plugins-common/src/index.ts @@ -5,6 +5,7 @@ import path from "path"; export { QuarantineMode, UnflakableConfig, + UnflakableConfigEnabled, loadApiKey, loadConfig, loadConfigSync, diff --git a/packages/test-common/src/config.ts b/packages/test-common/src/config.ts index 485556f..4ac20ed 100644 --- a/packages/test-common/src/config.ts +++ b/packages/test-common/src/config.ts @@ -9,7 +9,7 @@ import { import type { cosmiconfig, cosmiconfigSync, Options } from "cosmiconfig"; import { default as expect } from "expect"; -const debug = _debug("unflakable:integration-common:config"); +const debug = _debug("unflakable:test-common:config"); const throwUnimplemented = (): never => { throw new Error("unimplemented"); diff --git a/packages/test-common/src/git.ts b/packages/test-common/src/git.ts index a35a569..bbbadd7 100644 --- a/packages/test-common/src/git.ts +++ b/packages/test-common/src/git.ts @@ -5,7 +5,7 @@ import { setSimpleGitFactory } from "@unflakable/plugins-common"; import { SimpleGit, TaskOptions, Response as GitResponse } from "simple-git"; import deepEqual from "deep-equal"; -const debug = _debug("unflakable:integration-common:git"); +const debug = _debug("unflakable:test-common:git"); export type SimpleGitMockRef = { sha: string; From 9600152c4a73b4be1e31c86a32d8d31153c979b2 Mon Sep 17 00:00:00 2001 From: "David A. Ramos" Date: Tue, 11 Jul 2023 15:33:08 -0700 Subject: [PATCH 13/53] [jest] Tolerate slow integration tests Jest prints a duration after the spec file path when the spec takes longer than some duration. This change avoids failing integration tests when that extra output appears. --- .../test/integration/src/verify-output.ts | 105 ++++++++++++------ 1 file changed, 73 insertions(+), 32 deletions(-) diff --git a/packages/jest-plugin/test/integration/src/verify-output.ts b/packages/jest-plugin/test/integration/src/verify-output.ts index a664ee4..e4d06c8 100644 --- a/packages/jest-plugin/test/integration/src/verify-output.ts +++ b/packages/jest-plugin/test/integration/src/verify-output.ts @@ -14,8 +14,25 @@ const PASS = "\u001b[0m\u001b[7m\u001b[1m\u001b[32m PASS \u001b[39m\u001b[22m\u001b[27m\u001b[0m"; const QUARANTINED = "\u001b[0m\u001b[7m\u001b[1m\u001b[33m QUARANTINED \u001b[39m\u001b[22m\u001b[27m\u001b[0m"; -const formatTestFilename = (path: string, filename: string): string => - `\u001b[2m${path}\u001b[22m\u001b[1m${filename}\u001b[22m`; +const SLOW_TEST_DETAIL_REGEX = + "\u001b\\[0m\u001b\\[1m\u001b\\[41m[0-9.]+ ?.?s\u001b\\[49m\u001b\\[22m\u001b\\[0m"; +const formatTestFilename = (dir: string, filename: string): string => + `\u001b[2m${dir}\u001b[22m\u001b[1m${filename}\u001b[22m`; + +const specResultRegexMatch = ( + result: "fail" | "pass" | "quarantined", + dir: string, + filename: string +): RegExp => + // NB: Includes possible slow test duration: + // https://github.com/jestjs/jest/blob/6d2632adae0f0fa1fe116d3b475fd9783d0de1b5/packages/jest-reporters/src/getResultHeader.ts#L43 + new RegExp( + `^${escapeStringRegexp( + `${result === "quarantined" ? QUARANTINED + " " : ""}${ + result === "pass" ? PASS : FAIL + } ${formatTestFilename(dir, filename)}` + )}(?: \\(${SLOW_TEST_DETAIL_REGEX}\\))?$` + ); const testResultRegexMatch = ( result: TestAttemptResult | "skipped", @@ -76,9 +93,9 @@ export const verifyOutput = ( // Test our VerboseReporter customization. (testNamePattern === undefined || "should pass".match(testNamePattern) !== null - ? expect(stderrLines).toContain - : expect(stderrLines).not.toContain)( - `${PASS} ${formatTestFilename("src/", "pass.test.ts")}` + ? expect(stderrLines).toContainEqual + : expect(stderrLines).not.toContainEqual)( + expect.stringMatching(specResultRegexMatch("pass", "src/", "pass.test.ts")) ); (testNamePattern === undefined || "should pass".match(testNamePattern) !== null @@ -93,9 +110,9 @@ export const verifyOutput = ( "describe block should ([escape regex]?.*$ fail".match( testNamePattern ) !== null) - ? expect(stderrLines).toContain - : expect(stderrLines).not.toContain)( - `${FAIL} ${formatTestFilename("src/", "fail.test.ts")}` + ? expect(stderrLines).toContainEqual + : expect(stderrLines).not.toContainEqual)( + expect.stringMatching(specResultRegexMatch("fail", "src/", "fail.test.ts")) ); (!skipFailures && (testNamePattern === undefined || @@ -118,15 +135,19 @@ export const verifyOutput = ( (testNamePattern === undefined || flakyTest1Name.match(testNamePattern) !== null); (flakyTest1ShouldRun - ? expect(stderrLines).toContain - : expect(stderrLines).not.toContain)( - `${ - quarantineFlake && - !failToFetchManifest && - expectQuarantinedTestsToBeQuarantined - ? `${QUARANTINED} ` - : "" - }${FAIL} ${formatTestFilename("src/", "flake.test.ts")}` + ? expect(stderrLines).toContainEqual + : expect(stderrLines).not.toContainEqual)( + expect.stringMatching( + specResultRegexMatch( + quarantineFlake && + !failToFetchManifest && + expectQuarantinedTestsToBeQuarantined + ? "quarantined" + : "fail", + "src/", + "flake.test.ts" + ) + ) ); // This test should fail then pass (though we're not verifying the order here). (flakyTest1ShouldRun @@ -134,7 +155,11 @@ export const verifyOutput = ( : expect(stderrLines).not.toContainEqual)( expect.stringMatching( testResultRegexMatch( - quarantineFlake && !failToFetchManifest ? "quarantined" : "fail", + quarantineFlake && + !failToFetchManifest && + expectQuarantinedTestsToBeQuarantined + ? "quarantined" + : "fail", flakyTest1Name, 2 ) @@ -159,7 +184,11 @@ export const verifyOutput = ( : expect(stderrLines).not.toContainEqual)( expect.stringMatching( testResultRegexMatch( - quarantineFlake && !failToFetchManifest ? "quarantined" : "fail", + quarantineFlake && + !failToFetchManifest && + expectQuarantinedTestsToBeQuarantined + ? "quarantined" + : "fail", flakyTest2Name, 2 ) @@ -171,20 +200,32 @@ export const verifyOutput = ( expect.stringMatching(testResultRegexMatch("pass", flakyTest2Name, 2)) ); + (!skipFailures + ? expect(stderrLines).toContainEqual + : expect(stderrLines).not.toContainEqual)( + expect.stringMatching( + specResultRegexMatch("fail", "src/", "invalid.test.ts") + ) + ); + (!skipQuarantined && (!expectQuarantinedTestsToBeSkipped || failToFetchManifest) && (testNamePattern === undefined || "describe block should be quarantined".match(testNamePattern) !== null) - ? expect(stderrLines).toContain - : expect(stderrLines).not.toContain)( - `${ - expectPluginToBeEnabled && - !failToFetchManifest && - expectQuarantinedTestsToBeQuarantined && - !expectQuarantinedTestsToBeSkipped - ? `${QUARANTINED} ` - : "" - }${FAIL} ${formatTestFilename("src/", "quarantined.test.ts")}` + ? expect(stderrLines).toContainEqual + : expect(stderrLines).not.toContainEqual)( + expect.stringMatching( + specResultRegexMatch( + expectPluginToBeEnabled && + !failToFetchManifest && + expectQuarantinedTestsToBeQuarantined && + !expectQuarantinedTestsToBeSkipped + ? "quarantined" + : "fail", + "src/", + "quarantined.test.ts" + ) + ) ); (!skipQuarantined && (testNamePattern === undefined || @@ -223,9 +264,9 @@ export const verifyOutput = ( !expectQuarantinedTestsToBeQuarantined) && mixedQuarantinedTestShouldRun) || mixedFailTestShouldRun - ? expect(stderrLines).toContain - : expect(stderrLines).not.toContain)( - `${FAIL} ${formatTestFilename("src/", "mixed.test.ts")}` + ? expect(stderrLines).toContainEqual + : expect(stderrLines).not.toContainEqual)( + expect.stringMatching(specResultRegexMatch("fail", "src/", "mixed.test.ts")) ); (mixedQuarantinedTestShouldRun ? expect(stderrLines).toContainEqual From ac6cac3c0648ef5c22e4d9f0c22dbab9b64699d0 Mon Sep 17 00:00:00 2001 From: "David A. Ramos" Date: Wed, 12 Jul 2023 12:55:16 -0700 Subject: [PATCH 14/53] [cypress] Enable cypress:server logs for Linux integration tests --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index ef0019f..a1f235e 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -173,7 +173,7 @@ jobs: # DEBUG: unflakable:* # Enable debug logs within the Cypress plugin. - TEST_DEBUG: unflakable:* + TEST_DEBUG: "unflakable:*,cypress:server:*" # Enable terminal colors for debug() output. DEBUG_COLORS: "1" From ddd8276a03f677ad09fbd9cecccfae93e50bd8c1 Mon Sep 17 00:00:00 2001 From: "David A. Ramos" Date: Sun, 16 Jul 2023 14:24:55 -0700 Subject: [PATCH 15/53] [cypress] Add verbose logging in CI for Chrome remote interface client CI runs are regularly timing out waiting for Chrome tabs to close. Hopefully this logging will pinpoint exactly where the issue is happening. --- .github/workflows/ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a1f235e..2dfabd9 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -173,7 +173,7 @@ jobs: # DEBUG: unflakable:* # Enable debug logs within the Cypress plugin. - TEST_DEBUG: "unflakable:*,cypress:server:*" + TEST_DEBUG: "unflakable:*,cypress:server:*,cypress-verbose:server:browsers:cri-client:*" # Enable terminal colors for debug() output. DEBUG_COLORS: "1" @@ -281,7 +281,7 @@ jobs: # DEBUG: unflakable:* # Enable debug logs within the Cypress plugin. - TEST_DEBUG: "unflakable:*,cypress:server:*" + TEST_DEBUG: "unflakable:*,cypress:server:*,cypress-verbose:server:browsers:cri-client:*" # Enable terminal colors for debug() output. DEBUG_COLORS: "1" From 253cb346b09b91523b7165cc17524c6306ba5828 Mon Sep 17 00:00:00 2001 From: "David A. Ramos" Date: Tue, 18 Jul 2023 23:18:37 -0700 Subject: [PATCH 16/53] Refactor and simplify test counts --- packages/jest-plugin/rollup.config.mjs | 1 + packages/jest-plugin/src/reporter.ts | 322 ++++++++----- packages/jest-plugin/src/runner.ts | 456 +++++++++++------- packages/jest-plugin/src/types.ts | 24 +- packages/jest-plugin/src/utils.ts | 2 +- .../src/vendored/SummaryReporter.ts | 4 +- .../jest-plugin/src/vendored/getSummary.ts | 135 +----- .../test/integration/src/run-test-case.ts | 2 + packages/plugins-common/src/tsconfig.json | 1 + 9 files changed, 524 insertions(+), 423 deletions(-) diff --git a/packages/jest-plugin/rollup.config.mjs b/packages/jest-plugin/rollup.config.mjs index d9474d2..e234e90 100644 --- a/packages/jest-plugin/rollup.config.mjs +++ b/packages/jest-plugin/rollup.config.mjs @@ -36,6 +36,7 @@ export default [ // Mimicks TypeScript `esModuleInterop` (see // https://rollupjs.org/configuration-options/#output-format). interop: "auto", + sourcemap: true, }, // Bundle the internal @unflakable/plugins-common package in dist/, but leave most other // imported packages as an external. Internal modules begin with `.` or `/`. diff --git a/packages/jest-plugin/src/reporter.ts b/packages/jest-plugin/src/reporter.ts index c612fa5..8d4d8de 100644 --- a/packages/jest-plugin/src/reporter.ts +++ b/packages/jest-plugin/src/reporter.ts @@ -5,7 +5,6 @@ import type { AggregatedResult, AssertionResult, Status, - TestResult, } from "@jest/test-result"; import { BaseReporter, @@ -14,7 +13,7 @@ import { Test, VerboseReporter, } from "@jest/reporters"; -import { FAILED, groupBy, testKey, USER_AGENT } from "./utils"; +import { groupBy, testKey, USER_AGENT } from "./utils"; import { TestAttemptResult, TestRunRecord, @@ -23,8 +22,10 @@ import { } from "@unflakable/js-api"; import { UnflakableAggregatedResult, + UnflakableAggregatedResultWithCounts, UnflakableAssertionResult, UnflakableTestResult, + UnflakableTestResultWithCounts, } from "./types"; import { specialChars } from "jest-util"; import type { Config } from "@jest/types"; @@ -46,6 +47,7 @@ import { UnflakableConfig, UnflakableConfigEnabled, } from "@unflakable/plugins-common"; +import { addResult, makeEmptyAggregatedTestResult } from "@jest/test-result"; const debug = _debug("unflakable:reporter"); @@ -97,116 +99,153 @@ const getIcon = (test: UnflakableAssertionResult): string => { } }; -// Removes stats attributed to retries (which shouldn't affect the overall stats) and counts flaky -// tests. -const processedResults = ( - aggregatedResults: AggregatedResult -): UnflakableAggregatedResult => { +// Recomputes the aggregated stats after taking into account retries, flakes, and quarantined +// failures. This function returns new objects and does NOT modify its input. +const computeResultsForReporter = ( + origAggregatedResults: UnflakableAggregatedResult +): UnflakableAggregatedResultWithCounts => { const resultsByFilenameAndName = Object.fromEntries( Object.entries( groupBy( - aggregatedResults.testResults, + origAggregatedResults.testResults, (testFileResult: UnflakableTestResult) => testFileResult.testFilePath ) ).map(([testFilePath, testFileResults]) => [ testFilePath, groupBy( testFileResults.flatMap((testFileResult) => testFileResult.testResults), - (assertionResult) => JSON.stringify(testKey(assertionResult)) + (assertionResult) => JSON.stringify(testKey(assertionResult, false)) ), ]) ); - return aggregatedResults.testResults.reduce( - (filteredResults, testResult: UnflakableTestResult) => { - if ((testResult._unflakableAttempt ?? 0) > 0) { - return { - ...filteredResults, - numFailedTestSuites: - testResult.numFailingTests > 0 || - testResult.testExecError !== undefined - ? filteredResults.numFailedTestSuites - 1 - : filteredResults.numFailedTestSuites, - numPassedTestSuites: - !testResult.skipped && - !( - testResult.numFailingTests > 0 || - testResult.testExecError !== undefined - ) - ? filteredResults.numPassedTestSuites - 1 - : filteredResults.numPassedTestSuites, - numPendingTestSuites: testResult.skipped - ? filteredResults.numPendingTestSuites - 1 - : filteredResults.numPendingTestSuites, - snapshot: { - ...filteredResults.snapshot, - matched: - filteredResults.snapshot.matched - testResult.snapshot.matched, - total: - filteredResults.snapshot.total - - testResult.snapshot.added - - testResult.snapshot.matched - - testResult.snapshot.unmatched - - testResult.snapshot.updated, - // When we retry failed tests, Jest incorrectly counts named snapshots as obsolete. - unchecked: - filteredResults.snapshot.unchecked - - testResult.snapshot.unchecked, - uncheckedKeysByFile: filteredResults.snapshot.uncheckedKeysByFile - .map((uncheckedSnapshot) => { - uncheckedSnapshot.keys = uncheckedSnapshot.keys.filter((key) => - testResult.snapshot.uncheckedKeys.includes(key) - ); - return uncheckedSnapshot; - }) - .filter((uncheckedSnapshot) => uncheckedSnapshot.keys.length > 0), - unmatched: - filteredResults.snapshot.unmatched - - testResult.snapshot.unmatched, - filesUnmatched: - filteredResults.snapshot.filesUnmatched - - (testResult.snapshot.unmatched > 0 ? 1 : 0), - }, - }; - } else { + // Only includes the first attempt of each test file, but the stats take into account subsequent + // attempts to determine flakiness. + const updatedTestResults: UnflakableTestResultWithCounts[] = + origAggregatedResults.testResults + .filter((testResult) => (testResult._unflakableAttempt ?? 0) === 0) + .map((testResult): UnflakableTestResultWithCounts => { const attemptsByTestName = resultsByFilenameAndName[testResult.testFilePath]; - const numFlakyTests = testResult.testResults.reduce( - (numFlakyTests, assertionResult) => { + + const { + numFailingTests, + numFlakyTests, + numPassingTests, + numQuarantinedTests, + } = testResult.testResults.reduce( + ( + { + numFailingTests, + numFlakyTests, + numPassingTests, + numQuarantinedTests, + }, + assertionResult + ) => { const attempts = - attemptsByTestName[JSON.stringify(testKey(assertionResult))]; - return attempts.some((attempt) => attempt.status === "passed") && + attemptsByTestName[ + JSON.stringify(testKey(assertionResult, false)) + ]; + const isPassing = + attempts.some((attempt) => attempt.status === "passed") && + attempts.every( + (attempt) => + attempt.status === "passed" || attempt.status === "pending" + ); + const isQuarantined = + !isPassing && + attempts.some( + (attempt) => + attempt.status === "failed" && + attempt._unflakableIsQuarantined === true + ); + const isFlaky = + !isQuarantined && + attempts.some((attempt) => attempt.status === "passed") && attempts.some( - (attempt: UnflakableAssertionResult) => + (attempt) => attempt.status === "failed" && attempt._unflakableIsQuarantined !== true - ) - ? numFlakyTests + 1 - : numFlakyTests; + ); + const isFailing = + !isQuarantined && + attempts.some((attempt) => attempt.status === "failed") && + attempts.every( + (attempt) => + attempt.status === "failed" || attempt.status === "pending" + ); + return { + numFailingTests: numFailingTests + (isFailing ? 1 : 0), + numFlakyTests: numFlakyTests + (isFlaky ? 1 : 0), + numPassingTests: numPassingTests + (isPassing ? 1 : 0), + numQuarantinedTests: + numQuarantinedTests + (isQuarantined ? 1 : 0), + }; }, - 0 + { + numFailingTests: 0, + numFlakyTests: 0, + numPassingTests: 0, + numQuarantinedTests: 0, + } ); + return { - ...filteredResults, - testResults: [ - ...filteredResults.testResults, - { - ...testResult, - numFailingTests: Math.max( - testResult.numFailingTests - numFlakyTests, - 0 - ), - _unflakableNumFlakyTests: numFlakyTests, - }, - ], + ...testResult, + numFailingTests, + numPassingTests, + _unflakableNumFlakyTests: numFlakyTests, + _unflakableNumQuarantinedTests: numQuarantinedTests, }; + }); + + const emptyAggregatedResults: UnflakableAggregatedResultWithCounts = { + ...makeEmptyAggregatedTestResult(), + + // Jest sets these fields separately in its TestScheduler: + // https://github.com/jestjs/jest/blob/7cf50065ace0f0fffeb695a7980e404a17d3b761/packages/jest-core/src/TestScheduler.ts#L429 + numTotalTestSuites: origAggregatedResults.numTotalTestSuites, + startTime: origAggregatedResults.startTime, + success: origAggregatedResults.success, + + testResults: updatedTestResults, + _unflakableNumFlakyTests: 0, + _unflakableNumQuarantinedTests: 0, + _unflakableNumQuarantinedSuites: 0, + }; + + return updatedTestResults.reduce((aggregatedResults, testResult) => { + addResult(aggregatedResults, testResult); + + aggregatedResults.numTotalTests += + testResult._unflakableNumFlakyTests + + testResult._unflakableNumQuarantinedTests; + + aggregatedResults._unflakableNumFlakyTests += + testResult._unflakableNumFlakyTests; + aggregatedResults._unflakableNumQuarantinedTests += + testResult._unflakableNumQuarantinedTests; + + // Handle edge cases that onResult() considers a suite pass but that should be failed or + // quarantined: + // https://github.com/jestjs/jest/blob/6d2632adae0f0fa1fe116d3b475fd9783d0de1b5/packages/jest-test-result/src/helpers.ts#L110 + if ( + !testResult.skipped && + testResult.testExecError === undefined && + testResult.numFailingTests === 0 + ) { + if (testResult._unflakableNumFlakyTests > 0) { + aggregatedResults.numFailedTestSuites += 1; + aggregatedResults.numPassedTestSuites -= 1; + } else if (testResult._unflakableNumQuarantinedTests > 0) { + aggregatedResults._unflakableNumQuarantinedSuites += 1; + aggregatedResults.numPassedTestSuites -= 1; } - }, - { - ...aggregatedResults, - testResults: [] as UnflakableTestResult[], } - ); + + return aggregatedResults; + }, emptyAggregatedResults); }; export default class UnflakableReporter extends BaseReporter { @@ -221,6 +260,7 @@ export default class UnflakableReporter extends BaseReporter { private readonly summaryReporter: SummaryReporter; constructor(globalConfig: Config.GlobalConfig) { + debug("constructor"); super(); this.rootDir = globalConfig.rootDir; this.unflakableConfig = loadConfigSync(globalConfig.rootDir); @@ -246,13 +286,14 @@ export default class UnflakableReporter extends BaseReporter { // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-ignore: monkey patch to avoid full re-implementation of VerboseReporter. verboseReporter._logTest = ( - test: UnflakableAssertionResult, + assertionResult: UnflakableAssertionResult, indentLevel: number ): void => { - const status = getIcon(test); - const duration = test.duration ?? 0; + const status = getIcon(assertionResult); + const duration = assertionResult.duration ?? 0; const time = duration > 0 ? ` (${formatTime(Math.round(duration))})` : ""; + // prettier-ignore ( // eslint-disable-next-line @typescript-eslint/ban-ts-comment @@ -265,8 +306,8 @@ export default class UnflakableReporter extends BaseReporter { status + " " + chalk.dim( - test.title + - (test._unflakableIsQuarantined === true + assertionResult.title + + (assertionResult._unflakableIsQuarantined === true ? chalk.yellow(" [quarantined]") : "") + time @@ -282,7 +323,7 @@ export default class UnflakableReporter extends BaseReporter { this.defaultReporter.printTestFileHeader = ( _testPath: unknown, config: Config.ProjectConfig, - result: UnflakableTestResult + result: UnflakableTestResultWithCounts ): void => { const resultHeader = getResultHeader(result, globalConfig, config); @@ -346,50 +387,76 @@ export default class UnflakableReporter extends BaseReporter { this.defaultReporter.onTestStart(test); } - onTestCaseResult(test: Test, testCaseResult: AssertionResult): void { - debug("onTestCaseResult"); + onTestCaseResult( + test: Test, + assertionResult: UnflakableAssertionResult + ): void { + debug( + `onTestCaseResult path=\`${test.path}\` title=%o status=%o`, + [...assertionResult.ancestorTitles, assertionResult.title], + assertionResult.status + ); + // Not defined in Jest < 26.2. if (this.defaultReporter.onTestCaseResult !== undefined) { - this.defaultReporter.onTestCaseResult(test, testCaseResult); + this.defaultReporter.onTestCaseResult(test, assertionResult); } } + // NB: This is called once per test *file* attempt, but the aggregatedResults include every test + // attempt so far, including those from other files. onTestResult( test: Test, - testResult: TestResult, - aggregatedResult: AggregatedResult + testResult: UnflakableTestResult, + aggregatedResults: UnflakableAggregatedResult ): void { - debug("onTestResult"); + debug(`onTestResult path=\`${test.path}\``); + + const testResultForReporter: UnflakableTestResultWithCounts = { + ...testResult, + // Undo the sanitization of quarantined tests that the runner performs in order to keep + // Jest from exiting with non-zero status if all the failed tests are quarantined. This needs + // to be non-zero if there are any quarantined tests so that + // DefaultReporter.printTestFileHeader() prints FAIL for this file (preceded by QUARANTINED + // if all of the failures are quarantined). + numFailingTests: testResult.testResults.filter( + (assertionResult) => assertionResult.status === "failed" + ).length, + // We don't know when tests are flaky until the end. + _unflakableNumFlakyTests: 0, + _unflakableNumQuarantinedTests: testResult.testResults.filter( + (attempt) => + attempt.status === "failed" && + attempt._unflakableIsQuarantined === true + ).length, + snapshot: { + ...testResult.snapshot, + // When we retry failed tests, Jest incorrectly counts named snapshots as obsolete. Filter + // out obsolete tests during retries. + unchecked: + (testResult._unflakableAttempt ?? 0) > 0 + ? 0 + : testResult.snapshot.unchecked, + uncheckedKeys: + (testResult._unflakableAttempt ?? 0) > 0 + ? [] + : testResult.snapshot.uncheckedKeys, + }, + }; + this.defaultReporter.onTestResult( test, - { - ...testResult, - // Undo the sanitization of quarantined tests that the runner performs in order to keep - // Jest from exiting with non-zero status if all the failed tests are quarantined. - numFailingTests: testResult.testResults.filter( - (assertionResult) => assertionResult.status === FAILED - ).length, - snapshot: { - ...testResult.snapshot, - // When we retry failed tests, Jest incorrectly counts named snapshots as obsolete. Filter - // out obsolete tests during retries. - unchecked: - ((testResult as UnflakableTestResult)._unflakableAttempt ?? 0) > 0 - ? 0 - : testResult.snapshot.unchecked, - uncheckedKeys: - ((testResult as UnflakableTestResult)._unflakableAttempt ?? 0) > 0 - ? [] - : testResult.snapshot.uncheckedKeys, - }, - }, - processedResults(aggregatedResult) + testResultForReporter, + computeResultsForReporter(aggregatedResults) ); } + // NB: This is called only once, after UnflakableRunner.runTests() returns (i.e., after all + // retries have been exhausted or all tests passed). The aggregatedResults include every test + // attempt. async onRunComplete( contexts: Set | undefined, - aggregatedResults: AggregatedResult + aggregatedResults: UnflakableAggregatedResult ): Promise { debug("onRunComplete"); this.defaultReporter.onRunComplete(); @@ -397,7 +464,7 @@ export default class UnflakableReporter extends BaseReporter { // Don't double-count tests that were retried. this.summaryReporter.onRunComplete( contexts, - processedResults(aggregatedResults) + computeResultsForReporter(aggregatedResults) ); if (this.unflakableConfig.enabled && this.unflakableConfig.uploadResults) { @@ -413,10 +480,9 @@ export default class UnflakableReporter extends BaseReporter { private async uploadResults( aggregatedResults: AggregatedResult, - unflakableConfig: UnflakableConfig + unflakableConfig: UnflakableConfigEnabled ): Promise { - const testSuiteId = (unflakableConfig as UnflakableConfigEnabled) - .testSuiteId; + const testSuiteId = unflakableConfig.testSuiteId; const git = unflakableConfig.gitAutoDetect ? await loadGitRepo() : null; const repoRoot = git !== null ? await getRepoRoot(git) : this.rootDir; @@ -432,13 +498,13 @@ export default class UnflakableReporter extends BaseReporter { testFileResults.flatMap( (testFileResult) => testFileResult.testResults ), - (assertionResult) => JSON.stringify(testKey(assertionResult)) + (assertionResult) => JSON.stringify(testKey(assertionResult, true)) ) ) .map( ([, assertionResults]): TestRunRecord => ({ filename: toPosix(path.relative(repoRoot, testFilePath)), - name: testKey(assertionResults[0]), + name: testKey(assertionResults[0], true), attempts: assertionResults .map((testResult: UnflakableAssertionResult) => ({ testResult, diff --git a/packages/jest-plugin/src/runner.ts b/packages/jest-plugin/src/runner.ts index b87ff3d..0aac0eb 100644 --- a/packages/jest-plugin/src/runner.ts +++ b/packages/jest-plugin/src/runner.ts @@ -33,40 +33,120 @@ import { loadConfigSync, loadGitRepo, toPosix, - QuarantineMode, UnflakableConfig, } from "@unflakable/plugins-common"; const debug = _debug("unflakable:runner"); +// Exported by newer Jest versions but not older ones prior to 26.2.0. +export declare type TestEvents = { + "test-file-start": [Test]; + "test-file-success": [Test, TestResult]; + "test-file-failure": [Test, SerializableError]; + "test-case-result": [string, AssertionResult]; +}; + +export declare type UnsubscribeFn = () => void; + type TestFailure = { test: Test; testResult: TestResult }; -const wrapOnResult = - ({ - attempt, - manifest, - onResult, - quarantineMode, - repoRoot, - testFailures, - }: { - attempt: number; - manifest: TestSuiteManifest | undefined; - onResult: OnTestSuccess; - quarantineMode: QuarantineMode; - repoRoot: string; - testFailures: TestFailure[]; - }) => - async (test: Test, testResult: TestResult): Promise => { - const testResults = testResult.testResults.map( - (assertionResult: AssertionResult): UnflakableAssertionResult => { +class UnflakableRunner { + readonly supportsEventEmitters = true; + + private readonly context?: TestRunnerContext; + private readonly globalConfig: Config.GlobalConfig; + private readonly manifest: Promise; + private readonly unflakableConfig: UnflakableConfig; + + private testEventHandlers: { + [key in keyof TestEvents]?: (( + eventData: TestEvents[key] + ) => void | Promise)[]; + } = {}; + + constructor(globalConfig: Config.GlobalConfig, context?: TestRunnerContext) { + debug("constructor"); + this.unflakableConfig = loadConfigSync(globalConfig.rootDir); + + const testSuiteId = this.unflakableConfig.enabled + ? this.unflakableConfig.testSuiteId + : ""; + + if (this.unflakableConfig.enabled) { + const apiKey = loadApiKey(); + this.manifest = getTestSuiteManifest({ + testSuiteId, + apiKey, + baseUrl: this.unflakableConfig.apiBaseUrl, + clientDescription: USER_AGENT, + log: process.stderr.write.bind(process.stderr), + }); + } else { + debug("Not fetching manifest because plugin is disabled"); + this.manifest = Promise.resolve(undefined); + } + + this.context = context; + this.globalConfig = globalConfig; + } + + // We expose an on() method that TestScheduler can call to register its event callbacks: + // https://github.com/jestjs/jest/blob/7cf50065ace0f0fffeb695a7980e404a17d3b761/packages/jest-core/src/TestScheduler.ts#L264. + // We also return an unsubscribe function from each call, although unsubscribing doesn't seem to + // serve much of a purpose since the test runner goes out of scope immediately after Jest calls + // the unsubscribe functions. We just do the same thing Jest does and unregister our own handlers + // before the inner TestRunner goes out of scope, and then clear the TestScheduler's registered + // callbacks from UnflakableTestRunner when TestScheduler calls its unsubscribe functions. This + // may be needed to break circular references and ensure that everything gets GCed. + on( + eventName: Name, + listener: (eventData: TestEvents[Name]) => void | Promise + ): UnsubscribeFn { + debug(`subscribing to \`${eventName}\` listener`); + if (this.testEventHandlers[eventName] === undefined) { + this.testEventHandlers[eventName] = []; + } + + type EventListener = (eventData: TestEvents[Name]) => void | Promise; + + (this.testEventHandlers[eventName] as EventListener[]).push(listener); + + return () => { + debug(`unsubscribing from \`${eventName}\` listener`); + const idx = ( + this.testEventHandlers[eventName] as EventListener[] + ).indexOf(listener); + if (idx !== -1) { + (this.testEventHandlers[eventName] as EventListener[]).splice(idx, 1); + } + }; + } + + // Called after each test *file* runs successfully (which may include failed tests, but the test + // file itself didn't throw any errors when it was loaded). This function modifies + // `testResult.testResults` by adding our own fields, and returns an updated + // `UnflakableTestResult` that also includes some of our own fields. In the case of retries, we + // also clear stats that would otherwise result in double-counted tests being emitted by the + // SummaryReporter. + private onResult( + attempt: number, + manifest: TestSuiteManifest | undefined, + repoRoot: string, + testsToRetry: TestFailure[], + test: Test, + testResult: TestResult + ): void { + debug(`onResult attempt=${attempt} path=\`${test.path}\``); + + testResult.testResults.forEach( + (assertionResult: UnflakableAssertionResult): void => { const testFilename = toPosix(path.relative(repoRoot, test.path)); if (assertionResult.status === FAILED) { if (manifest === undefined) { debug( "Not quarantining test failure due to failure to fetch manifest" ); - } else if (quarantineMode === "no_quarantine") { + } else if (this.unflakableConfig.quarantineMode === "no_quarantine") { debug( "Not quarantining test failure because quarantineMode is set to `no_quarantine`" ); @@ -74,96 +154,64 @@ const wrapOnResult = const isQuarantined = isTestQuarantined( manifest, testFilename, - testKey(assertionResult) + testKey(assertionResult, true) ); debug( `Test is ${ isQuarantined ? "" : "NOT " }quarantined: ${JSON.stringify( - testKey(assertionResult) + testKey(assertionResult, false) )} in file ${testFilename}` ); - return { - ...assertionResult, - // Use a separate field instead of adding a new `status` to avoid confusing third- - // party code that consumes the `Status` enum. - _unflakableIsQuarantined: isQuarantined, - }; + + // Use a separate field instead of adding a new `status` to avoid confusing third- + // party code that consumes the `Status` enum. + assertionResult._unflakableIsQuarantined = isQuarantined; } } - return assertionResult; } ); - const numFailingTests = testResults.filter( - (assertionResult) => + const numFailingTests = testResult.testResults.filter( + (assertionResult: UnflakableAssertionResult) => assertionResult.status === FAILED && assertionResult._unflakableIsQuarantined !== true ).length; - const numQuarantinedTests = testResults.filter( - (assertionResult) => assertionResult._unflakableIsQuarantined === true - ).length; - const processedTestResult: UnflakableTestResult = - attempt === 0 - ? { - ...testResult, - // NB: If this value is non-zero, the whole Jest run will terminate with a non-zero exit - // code. - numFailingTests, - _unflakableAttempt: attempt, - _unflakableNumQuarantinedTests: numQuarantinedTests, - testResults, - } - : // Don't double-count retried tests or SummaryReporter will produce confusing results. - { - ...testResult, - numFailingTests: 0, - numPassingTests: 0, - numPendingTests: 0, - numTodoTests: 0, - _unflakableAttempt: attempt, - _unflakableNumQuarantinedTests: numQuarantinedTests, - testResults, - }; - - if (numFailingTests > 0 || numQuarantinedTests > 0) { - testFailures.push({ test, testResult }); - } - - await onResult(test, processedTestResult); - }; - -class UnflakableRunner { - private readonly context?: TestRunnerContext; - private readonly globalConfig: Config.GlobalConfig; - private readonly manifest: Promise; - private readonly unflakableConfig: UnflakableConfig; - constructor(globalConfig: Config.GlobalConfig, context?: TestRunnerContext) { - this.unflakableConfig = loadConfigSync(globalConfig.rootDir); - - const testSuiteId = this.unflakableConfig.enabled - ? this.unflakableConfig.testSuiteId - : ""; + // We retry any type of failure, including quarantined failures. + if ( + testResult.testResults.some( + (assertionResult) => assertionResult.status === FAILED + ) + ) { + testsToRetry.push({ test, testResult }); + } - if (this.unflakableConfig.enabled) { - const apiKey = loadApiKey(); - this.manifest = getTestSuiteManifest({ - testSuiteId, - apiKey, - baseUrl: this.unflakableConfig.apiBaseUrl, - clientDescription: USER_AGENT, - log: process.stderr.write.bind(process.stderr), - }); + (testResult as UnflakableTestResult)._unflakableAttempt = attempt; + if (attempt === 0) { + // NB: If this value is non-zero, the whole Jest run will terminate with a non-zero exit + // code. + testResult.numFailingTests = numFailingTests; } else { - debug("Not fetching manifest because plugin is disabled"); - this.manifest = Promise.resolve(undefined); + // Don't double-count retried tests or SummaryReporter will produce confusing results. + testResult.numFailingTests = 0; + testResult.numPassingTests = 0; + testResult.numPendingTests = 0; + testResult.numTodoTests = 0; } - - this.context = context; - this.globalConfig = globalConfig; } + // FIXME: test what happens when running Jest with multiple --projects arguments. Jest seems to + // create a separate "context" per project and associate that context with each test in the + // project. + + // EmittingTestRunnerInterface in Jest 28+. + async runTests( + tests: Array, + watcher: TestWatcher, + options: TestRunnerOptions + ): Promise; + // CallbackTestRunnerInterface in Jest 28+, and any version < Jest 28. async runTests( tests: Array, watcher: TestWatcher, @@ -171,7 +219,16 @@ class UnflakableRunner { onResult: OnTestSuccess, onFailure: OnTestFailure, options: TestRunnerOptions + ): Promise; + async runTests( + tests: Array, + watcher: TestWatcher, + onStartOrOptions: OnTestStart | TestRunnerOptions, + onResult?: OnTestSuccess, + onFailure?: OnTestFailure, + options?: TestRunnerOptions ): Promise { + debug("runTests"); const repoRoot = this.unflakableConfig.enabled && this.unflakableConfig.gitAutoDetect ? await (async (): Promise => { @@ -182,42 +239,37 @@ class UnflakableRunner { })() : this.globalConfig.rootDir; - let testFailures = await this.runTestsImpl( + let testsToRetry = await this.runTestsImpl( tests, watcher, - onStart, + onResult !== undefined ? (onStartOrOptions as OnTestStart) : undefined, onResult, onFailure, - options, + onResult !== undefined + ? (options as TestRunnerOptions) + : (onStartOrOptions as TestRunnerOptions), repoRoot, this.globalConfig, - this.context, - this.unflakableConfig, - await this.manifest, 0 // attempt ); - if (!this.unflakableConfig.enabled || testFailures.length === 0) { - return; - } - const attempts = - this.unflakableConfig.failureRetries > 0 + this.unflakableConfig.enabled && this.unflakableConfig.failureRetries > 0 ? this.unflakableConfig.failureRetries + 1 : 1; // NB: jest-circus also supports failure retries, but it's configured via the `jest` object // in each test file's environment, not via the global config that we have control over. for ( let attempt = 1; - testFailures.length !== 0 && attempt < attempts; + testsToRetry.length !== 0 && attempt < attempts; attempt++ ) { process.stderr.write( chalk.stderr.yellow.bold( - `Retrying ${testFailures.reduce( + `Retrying ${testsToRetry.reduce( (count, { testResult }) => count + testResult.numFailingTests, 0 - )} failed test(s) from ${testFailures.length} file(s) -- ${ + )} failed test(s) from ${testsToRetry.length} file(s) -- ${ attempts - attempt - 1 } ${attempts - attempt - 1 === 1 ? "retry" : "retries"} remaining` ) + "\n" @@ -226,14 +278,13 @@ class UnflakableRunner { // Similar to how we skip quarantined tests when quarantineMode is "skip_tests", we need to // re-run each failed file separately so that we can pass a custom testNamePattern regex to // each. This ensures that we only rerun the failed tests in each file. - testFailures = await testFailures.reduce( + testsToRetry = await testsToRetry.reduce( (promise, { test, testResult }) => - promise.then(async (newTestFailures) => { + promise.then(async (newTestsToRetry) => { const failedTestPattern = testResult.testResults .filter( (assertionResult: UnflakableAssertionResult) => - assertionResult.status === FAILED && - assertionResult._unflakableIsQuarantined !== true + assertionResult.status === FAILED ) .map((failedTest) => { const testId = testKey(failedTest, false); @@ -252,20 +303,21 @@ class UnflakableRunner { ...this.globalConfig, testNamePattern, }); - return newTestFailures.concat( + return newTestsToRetry.concat( await this.runTestsImpl( [test], watcher, - onStart, + onResult !== undefined + ? (onStartOrOptions as OnTestStart) + : undefined, // We re-wrap it in the next iteration. onResult, onFailure, - options, + onResult !== undefined + ? (options as TestRunnerOptions) + : (onStartOrOptions as TestRunnerOptions), repoRoot, filteredGlobalConfig, - this.context, - this.unflakableConfig, - await this.manifest, attempt ) ); @@ -275,70 +327,143 @@ class UnflakableRunner { } } + // Returns an array of tests that should be retried, which includes quarantined failures. private async runTestsImpl( tests: Test[], watcher: TestWatcher, - onStart: OnTestStart, - onResult: OnTestSuccess, - onFailure: OnTestFailure, + onStart: OnTestStart | undefined, + onResult: OnTestSuccess | undefined, + onFailure: OnTestFailure | undefined, options: TestRunnerOptions, repoRoot: string, globalConfig: Config.GlobalConfig, - context: TestRunnerContext | undefined, - unflakableConfig: UnflakableConfig, - manifest: TestSuiteManifest | undefined, attempt: number ): Promise { - const testFailures: TestFailure[] = []; - - const onResultImpl = this.unflakableConfig.enabled - ? wrapOnResult({ - attempt, - manifest, - onResult, - quarantineMode: unflakableConfig.quarantineMode, - repoRoot, - testFailures, - }) - : onResult; - - const runTests = ( + debug("runTestsImpl"); + const testsToRetry: TestFailure[] = []; + const manifest = await this.manifest; + + const runTests = async ( globalConfig: Config.GlobalConfig, tests: Array, watcher: TestWatcher, options: TestRunnerOptions ): Promise => { - const testRunner = new TestRunner(globalConfig, context ?? {}); + const testRunner = new TestRunner(globalConfig, this.context ?? {}); - // We have to give up on per-event type safety here to maintain compatibility with versions - // prior to 26.2.0. - type EventListener = (eventData: unknown) => void | Promise; - type EventSubscriber = ( - eventName: string, - listener: EventListener - ) => () => void; + const onResultImpl = + onResult !== undefined && this.unflakableConfig.enabled + ? async (test: Test, result: TestResult): Promise => { + // NB: We call this first because it modifies `result`. + this.onResult( + attempt, + manifest, + repoRoot, + testsToRetry, + test, + result + ); + await onResult(test, result); + } + : onResult; // The event emitter interface was introduced in Jest 26.2.0 (see // https://github.com/facebook/jest/pull/10227). if ((testRunner as unknown as { on?: unknown }).on !== undefined) { const eventEmittingTestRunner = testRunner as unknown as { - on: EventSubscriber; + on: ( + eventName: Name, + listener: (eventData: TestEvents[Name]) => void | Promise + ) => UnsubscribeFn; }; - eventEmittingTestRunner.on("test-file-start", (([test]: [Test]) => - onStart(test)) as EventListener); - eventEmittingTestRunner.on("test-file-success", (([test, result]: [ - Test, - TestResult - ]) => onResultImpl(test, result)) as EventListener); - eventEmittingTestRunner.on("test-file-failure", (([test, error]: [ - Test, - SerializableError - ]) => onFailure(test, error)) as EventListener); - - return testRunner.runTests.length === 6 - ? // Jest prior to 28.0.0 expects the callback arguments (see - // https://github.com/facebook/jest/pull/12641). + + // EmittingTestRunnerInterface in Jest 28+. + const supportsEventEmitters = ( + testRunner as { + supportsEventEmitters?: boolean; + } + ).supportsEventEmitters; + + const unsubscribes = [ + ...(onStart !== undefined + ? [ + eventEmittingTestRunner.on( + "test-file-start", + ([test]: [Test]) => onStart(test) + ), + ] + : []), + ...(onResultImpl !== undefined + ? [ + eventEmittingTestRunner.on( + "test-file-success", + ([test, result]: [Test, TestResult]) => + onResultImpl(test, result) + ), + ] + : []), + ...(onFailure !== undefined + ? [ + eventEmittingTestRunner.on( + "test-file-failure", + ([test, error]: [Test, SerializableError]) => + onFailure(test, error) + ), + ] + : []), + ...Object.entries(this.testEventHandlers).flatMap( + ([eventName, listeners]) => + listeners.map((listener) => { + if ( + eventName === "test-file-success" && + this.unflakableConfig.enabled + ) { + return eventEmittingTestRunner.on( + "test-file-success", + ([ + test, + result, + ]: TestEvents["test-file-success"]): void | Promise => { + // NB: We call this first because it modifies `result`. + this.onResult( + attempt, + manifest, + repoRoot, + testsToRetry, + test, + result + ); + // NB: This triggers Jest's ReporterDispatcher to call onTestResult() for each + // reporter. + return ( + listener as ( + eventData: TestEvents["test-file-success"] + ) => void | Promise + )([test, result]); + } + ); + } else { + return eventEmittingTestRunner.on( + eventName as keyof TestEvents, + listener as ( + eventData: TestEvents[keyof TestEvents] + ) => void | Promise + ); + } + }) + ), + ]; + + await (supportsEventEmitters === true + ? // EmittingTestRunnerInterface in Jest 28+. ( + testRunner.runTests as unknown as ( + tests: Array, + watcher: TestWatcher, + options: TestRunnerOptions + ) => Promise + )(tests, watcher, options) + : ( testRunner.runTests as unknown as ( tests: Array, watcher: TestWatcher, @@ -356,15 +481,9 @@ class UnflakableRunner { undefined, undefined, options - ) - : // Jest >= 28.0.0 no longer expects the callback arguments. - ( - testRunner.runTests as unknown as ( - tests: Array, - watcher: TestWatcher, - options: TestRunnerOptions - ) => Promise - )(tests, watcher, options); + )); + + unsubscribes.forEach((unsubscribe) => unsubscribe()); } else { // Prior to Jest 26.2.0, use the legacy callback interface. return ( @@ -394,7 +513,8 @@ class UnflakableRunner { if ( manifest !== undefined && manifest.quarantined_tests.length > 0 && - unflakableConfig.quarantineMode === "skip_tests" + this.unflakableConfig.enabled && + this.unflakableConfig.quarantineMode === "skip_tests" ) { debug( `Skipping ${manifest.quarantined_tests.length} quarantined test(s)` @@ -480,7 +600,7 @@ class UnflakableRunner { } else { await runTests(globalConfig, tests, watcher, options); } - return testFailures; + return testsToRetry; } } diff --git a/packages/jest-plugin/src/types.ts b/packages/jest-plugin/src/types.ts index 8c73927..a1d007b 100644 --- a/packages/jest-plugin/src/types.ts +++ b/packages/jest-plugin/src/types.ts @@ -10,12 +10,15 @@ export type UnflakableAssertionResult = AssertionResult & { _unflakableIsQuarantined?: boolean; }; -export type UnflakableTestResult = TestResult & { +export type UnflakableTestResult = Omit & { _unflakableAttempt?: number; - // Added by runner. - _unflakableNumQuarantinedTests?: number; - // Added by reporter. - _unflakableNumFlakyTests?: number; + testResults: UnflakableAssertionResult[]; +}; + +// Counts added by reporter. +export type UnflakableTestResultWithCounts = UnflakableTestResult & { + _unflakableNumQuarantinedTests: number; + _unflakableNumFlakyTests: number; }; export type UnflakableAggregatedResult = Omit< @@ -24,3 +27,14 @@ export type UnflakableAggregatedResult = Omit< > & { testResults: UnflakableTestResult[]; }; + +export type UnflakableAggregatedResultWithCounts = Omit< + AggregatedResult, + "testResults" +> & { + _unflakableNumFlakyTests: number; + _unflakableNumQuarantinedTests: number; + _unflakableNumQuarantinedSuites: number; + + testResults: UnflakableTestResultWithCounts[]; +}; diff --git a/packages/jest-plugin/src/utils.ts b/packages/jest-plugin/src/utils.ts index 2bdc0d7..9a4e0b9 100644 --- a/packages/jest-plugin/src/utils.ts +++ b/packages/jest-plugin/src/utils.ts @@ -14,7 +14,7 @@ export const PASSED: Status = "passed"; export const testKey = ( assertionResult: AssertionResult, - normalize = true + normalize: boolean ): string[] => { const fullKey = [...assertionResult.ancestorTitles, assertionResult.title]; return normalize ? normalizeTestName(fullKey) : fullKey; diff --git a/packages/jest-plugin/src/vendored/SummaryReporter.ts b/packages/jest-plugin/src/vendored/SummaryReporter.ts index 3298410..91c9b13 100644 --- a/packages/jest-plugin/src/vendored/SummaryReporter.ts +++ b/packages/jest-plugin/src/vendored/SummaryReporter.ts @@ -39,7 +39,7 @@ import { BaseReporter, ReporterOnStartOptions } from "@jest/reporters"; import { getSummary } from "./getSummary"; import getSnapshotSummary from "./getSnapshotSummary"; import { getResultHeader } from "./getResultHeader"; -import { UnflakableAggregatedResult } from "../types"; +import { UnflakableAggregatedResultWithCounts } from "../types"; const TEST_SUMMARY_THRESHOLD = 20; @@ -106,7 +106,7 @@ export default class SummaryReporter extends BaseReporter { onRunComplete( contexts: Set | undefined, - aggregatedResults: UnflakableAggregatedResult + aggregatedResults: UnflakableAggregatedResultWithCounts ): void { const { numTotalTestSuites, testResults, wasInterrupted } = aggregatedResults; diff --git a/packages/jest-plugin/src/vendored/getSummary.ts b/packages/jest-plugin/src/vendored/getSummary.ts index fda7bbf..51547af 100644 --- a/packages/jest-plugin/src/vendored/getSummary.ts +++ b/packages/jest-plugin/src/vendored/getSummary.ts @@ -31,72 +31,14 @@ All modifications to the above referenced file are copyrighted and licensed unde forth in the LICENSE file at the root of this repository. */ -import type { SummaryOptions, Test } from "@jest/reporters"; -import { - UnflakableAggregatedResult, - UnflakableAssertionResult, - UnflakableTestResult, -} from "../types"; +import type { SummaryOptions } from "@jest/reporters"; +import { UnflakableAggregatedResultWithCounts } from "../types"; import * as JestUtil from "jest-util"; -import type { AssertionResult } from "@jest/test-result"; import chalk from "chalk"; import { formatTime } from "./formatTime"; const PROGRESS_BAR_WIDTH = 40; -const getValuesCurrentTestCases = ( - currentTestCases: { - test: Test; - testCaseResult: UnflakableAssertionResult; - }[] = [] -): { - numFailingTests: number; - numPassingTests: number; - numPendingTests: number; - numQuarantinedTests: number; - numTodoTests: number; - numTotalTests: number; -} => { - let numFailingTests = 0; - let numPassingTests = 0; - let numPendingTests = 0; - let numQuarantinedTests = 0; - let numTodoTests = 0; - let numTotalTests = 0; - currentTestCases.forEach((testCase) => { - if (testCase.testCaseResult._unflakableIsQuarantined === true) { - numQuarantinedTests++; - } else { - switch (testCase.testCaseResult.status) { - case "failed": - numFailingTests++; - break; - case "passed": - numPassingTests++; - break; - case "skipped": - numPendingTests++; - break; - case "todo": - numTodoTests++; - break; - default: - break; - } - } - numTotalTests++; - }); - - return { - numFailingTests, - numPassingTests, - numPendingTests, - numQuarantinedTests, - numTodoTests, - numTotalTests, - }; -}; - const renderTime = ( runTime: number, estimatedTime: number, @@ -131,7 +73,7 @@ const renderTime = ( }; export const getSummary = ( - aggregatedResults: UnflakableAggregatedResult, + aggregatedResults: UnflakableAggregatedResultWithCounts, options?: SummaryOptions ): string => { let runTime = (Date.now() - aggregatedResults.startTime) / 1000; @@ -139,33 +81,6 @@ export const getSummary = ( runTime = Math.floor(runTime); } - const valuesForCurrentTestCases = getValuesCurrentTestCases( - ( - options as { - // Not defined in Jest < 26.2. - currentTestCases?: - | { test: Test; testCaseResult: AssertionResult }[] - | undefined; - } - )?.currentTestCases ?? [] - ); - - let suitesQuarantined = 0, - testsFlaky = 0, - testsQuarantined = 0; - aggregatedResults.testResults.forEach((testResult: UnflakableTestResult) => { - testsFlaky += testResult._unflakableNumFlakyTests ?? 0; - testsQuarantined += testResult._unflakableNumQuarantinedTests ?? 0; - if ( - !testResult.skipped && - testResult.numFailingTests === 0 && - testResult.testExecError === undefined && - (testResult._unflakableNumQuarantinedTests ?? 0) > 0 - ) { - suitesQuarantined += 1; - } - }); - const estimatedTime = options?.estimatedTime ?? 0; const snapshotResults = aggregatedResults.snapshot; const snapshotsAdded = snapshotResults.added; @@ -177,16 +92,18 @@ export const getSummary = ( const snapshotsTotal = snapshotResults.total; const snapshotsUpdated = snapshotResults.updated; const suitesFailed = aggregatedResults.numFailedTestSuites; - const suitesPassed = - aggregatedResults.numPassedTestSuites - suitesQuarantined; + const suitesPassed = aggregatedResults.numPassedTestSuites; const suitesPending = aggregatedResults.numPendingTestSuites; + const suitesQuarantined = aggregatedResults._unflakableNumQuarantinedSuites; const suitesRun = suitesFailed + suitesPassed + suitesQuarantined; const suitesTotal = aggregatedResults.numTotalTestSuites; const testsFailed = aggregatedResults.numFailedTests; + const testsFlaky = aggregatedResults._unflakableNumFlakyTests; const testsPassed = aggregatedResults.numPassedTests; const testsPending = aggregatedResults.numPendingTests; + const testsQuarantined = aggregatedResults._unflakableNumQuarantinedTests; const testsTodo = aggregatedResults.numTodoTests; - const testsTotal = aggregatedResults.numTotalTests + testsQuarantined; + const testsTotal = aggregatedResults.numTotalTests; const width = options?.width ?? 0; const suites = `${chalk.bold("Test Suites: ")}${ @@ -205,41 +122,21 @@ export const getSummary = ( suitesRun !== suitesTotal ? `${suitesRun} of ${suitesTotal}` : suitesTotal } total`; - const updatedTestsFailed = Math.max( - testsFailed + valuesForCurrentTestCases.numFailingTests - testsFlaky, - 0 - ); - const updatedTestsQuarantined = - testsQuarantined + valuesForCurrentTestCases.numQuarantinedTests; - const updatedTestsPending = - testsPending + valuesForCurrentTestCases.numPendingTests; - const updatedTestsTodo = testsTodo + valuesForCurrentTestCases.numTodoTests; - const updatedTestsPassed = - testsPassed + valuesForCurrentTestCases.numPassingTests; - const updatedTestsTotal = - testsTotal + valuesForCurrentTestCases.numTotalTests; - const tests = chalk.bold("Tests: ") + - (updatedTestsFailed > 0 - ? chalk.bold.red(`${updatedTestsFailed} failed`) + ", " - : "") + + (testsFailed > 0 ? chalk.bold.red(`${testsFailed} failed`) + ", " : "") + (testsFlaky > 0 ? chalk.bold.magentaBright(`${testsFlaky} flaky`) + ", " : "") + - (updatedTestsQuarantined > 0 - ? chalk.bold.yellow(`${updatedTestsQuarantined} quarantined`) + ", " - : "") + - (updatedTestsPending > 0 - ? chalk.bold.yellow(`${updatedTestsPending} skipped`) + ", " - : "") + - (updatedTestsTodo > 0 - ? chalk.bold.magenta(`${updatedTestsTodo} todo`) + ", " + (testsQuarantined > 0 + ? chalk.bold.yellow(`${testsQuarantined} quarantined`) + ", " : "") + - (updatedTestsPassed > 0 - ? chalk.bold.green(`${updatedTestsPassed} passed`) + ", " + (testsPending > 0 + ? chalk.bold.yellow(`${testsPending} skipped`) + ", " : "") + - `${updatedTestsTotal} total`; + (testsTodo > 0 ? chalk.bold.magenta(`${testsTodo} todo`) + ", " : "") + + (testsPassed > 0 ? chalk.bold.green(`${testsPassed} passed`) + ", " : "") + + `${testsTotal} total`; const snapshots = chalk.bold("Snapshots: ") + diff --git a/packages/jest-plugin/test/integration/src/run-test-case.ts b/packages/jest-plugin/test/integration/src/run-test-case.ts index 0e9cb36..6b6f112 100644 --- a/packages/jest-plugin/test/integration/src/run-test-case.ts +++ b/packages/jest-plugin/test/integration/src/run-test-case.ts @@ -511,6 +511,8 @@ export const runTestCase = async ( require.resolve("unflakable-test-common/dist/mock-cosmiconfig"), "--require", require.resolve("unflakable-test-common/dist/mock-git"), + // Uncomment to enable debugger. + // "--inspect", jestBin, // --no-cache disables the cache that stores the past timings, which makes the output // non-deterministic since it gets bolded if it exceeds the expected time. diff --git a/packages/plugins-common/src/tsconfig.json b/packages/plugins-common/src/tsconfig.json index b2d7ce7..4810377 100644 --- a/packages/plugins-common/src/tsconfig.json +++ b/packages/plugins-common/src/tsconfig.json @@ -7,6 +7,7 @@ "lib": ["ES2019"], // Required by Rollup (consumed by Rollup in the *-plugin packages). "module": "esnext", + "sourceMap": true, // Avoids conflicting global definitions from, e.g., jasmine. "types": ["node"] }, From 05f810b5671dd290a0000a37cd1611af78fd24a0 Mon Sep 17 00:00:00 2001 From: "David A. Ramos" Date: Wed, 26 Jul 2023 02:43:14 -0700 Subject: [PATCH 17/53] [jest] Verify numbers of test/spec runs in integration tests --- package.json | 3 - packages/cypress-plugin/package.json | 1 + .../jest-plugin/test/babel.config.js | 2 + .../test/integration-input/jest.config.js | 6 +- .../test/integration-input/package.json | 3 +- .../test/integration/jest.config.js | 3 +- .../jest-plugin/test/integration/package.json | 8 + .../test/integration/src/matchers.ts | 101 + .../test/integration/src/verify-output.ts | 216 ++- packages/jest-plugin/tsconfig.json | 1 + yarn.lock | 1637 +++++++++-------- 11 files changed, 1072 insertions(+), 909 deletions(-) rename babel.config.js => packages/jest-plugin/test/babel.config.js (91%) create mode 100644 packages/jest-plugin/test/integration/src/matchers.ts diff --git a/package.json b/package.json index 39f540e..7cba66c 100644 --- a/package.json +++ b/package.json @@ -2,9 +2,6 @@ "packageManager": "yarn@3.5.1", "private": true, "devDependencies": { - "@babel/core": "^7.21.5", - "@babel/preset-env": "^7.21.5", - "@babel/preset-typescript": "^7.21.5", "@types/debug": "^4.1.7", "@types/js-yaml": "^4.0.5", "@types/node": "^14.18.43", diff --git a/packages/cypress-plugin/package.json b/packages/cypress-plugin/package.json index 1dba670..dc62879 100644 --- a/packages/cypress-plugin/package.json +++ b/packages/cypress-plugin/package.json @@ -86,6 +86,7 @@ "rimraf": "^5.0.1", "rollup": "^3.21.1", "rollup-plugin-dts": "^5.3.0", + "ts-jest": "^29.1.0", "typescript": "^4.9.5", "widest-line": "3.1.0" }, diff --git a/babel.config.js b/packages/jest-plugin/test/babel.config.js similarity index 91% rename from babel.config.js rename to packages/jest-plugin/test/babel.config.js index 18869fd..67d5947 100644 --- a/babel.config.js +++ b/packages/jest-plugin/test/babel.config.js @@ -1,6 +1,8 @@ // Copyright (c) 2022-2023 Developer Innovations, LLC /* eslint-env node */ + +/** @type {import('@babel/core').ConfigFunction} */ module.exports = (api) => { api.cache.using(() => process.env.NODE_ENV); return { diff --git a/packages/jest-plugin/test/integration-input/jest.config.js b/packages/jest-plugin/test/integration-input/jest.config.js index 3fb908f..190d8e6 100644 --- a/packages/jest-plugin/test/integration-input/jest.config.js +++ b/packages/jest-plugin/test/integration-input/jest.config.js @@ -3,10 +3,6 @@ module.exports = { clearMocks: true, maxWorkers: 2, - // The /dist path is required until https://github.com/facebook/jest/pull/11961 is fixed, which - // appears not to be until Jest 28.x. - reporters: ["@unflakable/jest-plugin/dist/reporter"], - runner: "@unflakable/jest-plugin/dist/runner", // Default changed in Jest 29 (see // https://github.com/facebook/jest/blob/94c06ef0aa9b327f3c400610b861e7308b29ee0d/docs/UpgradingToJest29.md). @@ -21,7 +17,7 @@ module.exports = { "^.+\\.[jt]sx?$": [ "babel-jest", { - configFile: "../../../../babel.config.js", + configFile: "../babel.config.js", }, ], }, diff --git a/packages/jest-plugin/test/integration-input/package.json b/packages/jest-plugin/test/integration-input/package.json index 73e67cf..f931f29 100644 --- a/packages/jest-plugin/test/integration-input/package.json +++ b/packages/jest-plugin/test/integration-input/package.json @@ -6,7 +6,8 @@ "@unflakable/js-api": "workspace:^", "jest": "25.1.0 - 29", "jest-each": "25.1.0 - 29", - "jest-environment-node": "25.1.0 - 29" + "jest-environment-node": "25.1.0 - 29", + "typescript": "^4.9.5" }, "scripts": { "test": "jest", diff --git a/packages/jest-plugin/test/integration/jest.config.js b/packages/jest-plugin/test/integration/jest.config.js index 1e3441c..ac2e971 100644 --- a/packages/jest-plugin/test/integration/jest.config.js +++ b/packages/jest-plugin/test/integration/jest.config.js @@ -1,13 +1,14 @@ // Copyright (c) 2022-2023 Developer Innovations, LLC module.exports = { + setupFilesAfterEnv: ["./src/matchers.ts"], testEnvironment: "node", transform: { "^.+\\.[jt]s$": [ "babel-jest", { - configFile: "../../../../babel.config.js", + configFile: "../babel.config.js", }, ], }, diff --git a/packages/jest-plugin/test/integration/package.json b/packages/jest-plugin/test/integration/package.json index 2d6b21b..494e9ae 100644 --- a/packages/jest-plugin/test/integration/package.json +++ b/packages/jest-plugin/test/integration/package.json @@ -2,14 +2,22 @@ "name": "jest-integration", "private": true, "devDependencies": { + "@babel/core": "^7.22.9", + "@babel/preset-env": "^7.22.9", + "@babel/preset-typescript": "^7.22.5", + "@jest/expect-utils": "25.1.0 - 29", "@types/temp": "^0.9.1", "@unflakable/jest-plugin": "workspace:^", "@unflakable/js-api": "workspace:^", "escape-string-regexp": "^4.0.0", + "expect": "25.1.0 - 29", "jest": "25.1.0 - 29", "jest-environment-node": "25.1.0 - 29", + "jest-get-type": "25.1.0 - 29", + "jest-matcher-utils": "25.1.0 - 29", "mockttp": "^3.7.5", "temp": "^0.9.4", + "typescript": "^4.9.5", "unflakable-test-common": "workspace:^" }, "scripts": { diff --git a/packages/jest-plugin/test/integration/src/matchers.ts b/packages/jest-plugin/test/integration/src/matchers.ts new file mode 100644 index 0000000..6919967 --- /dev/null +++ b/packages/jest-plugin/test/integration/src/matchers.ts @@ -0,0 +1,101 @@ +// Copyright (c) 2023 Developer Innovations, LLC + +import { + MatcherHintOptions, + matcherErrorMessage, + matcherHint, + RECEIVED_COLOR, + printWithType, + printReceived, + getLabelPrinter, + printExpected, + INVERTED_COLOR, + stringify, +} from "jest-matcher-utils"; +import type { SyncExpectationResult } from "expect"; +import { equals, iterableEquality } from "@jest/expect-utils"; +import * as getType from "jest-get-type"; + +function toContainEqualTimes( + this: jest.MatcherContext, + received: Array | Set, + expected: unknown, + times: number +): SyncExpectationResult { + const matcherName = "toContainEqualTimes"; + const isNot = this.isNot; + const options: MatcherHintOptions = { + comment: "deep equality", + isNot, + promise: this.promise, + }; + + if (received === null) { + throw new Error( + matcherErrorMessage( + matcherHint(matcherName, undefined, undefined, options), + `${RECEIVED_COLOR("received")} value must not be null nor undefined`, + printWithType("Received", received, printReceived) + ) + ); + } + + const matchIndices = Array.from(received).reduce( + (matchIndices: number[], item: unknown, index: number) => + equals(item, expected, [...(this.customTesters ?? []), iterableEquality]) + ? [...matchIndices, index] + : matchIndices, + [] + ); + + const pass = matchIndices.length === times; + + const message = (): string => { + const labelExpected = `Expected value ${times} time${ + times !== 1 ? "s" : "" + }`; + const labelReceived = `Received ${(getType.getType ?? getType)( + received + )} with ${matchIndices.length} match${ + matchIndices.length !== 1 ? "es" : "" + }`; + const printLabel = getLabelPrinter(labelExpected, labelReceived); + + return ( + matcherHint(matcherName, undefined, undefined, options) + + "\n\n" + + `${printLabel(labelExpected)}${ + isNot === true ? "not " : "" + }${printExpected(expected)}\n` + + `${printLabel(labelReceived)}${isNot === true ? " " : ""}${ + isNot === true && Array.isArray(received) + ? RECEIVED_COLOR( + `[${received + .map((item, i) => { + const stringified = stringify(item); + return matchIndices.includes(i) + ? INVERTED_COLOR(stringified) + : stringified; + }) + .join(", ")}]` + ) + : printReceived(received) + }` + ); + }; + + return { message, pass }; +} + +expect.extend({ + toContainEqualTimes, +}); + +declare global { + // eslint-disable-next-line @typescript-eslint/no-namespace + namespace jest { + interface Matchers { + toContainEqualTimes(expected: unknown, times: number): R; + } + } +} diff --git a/packages/jest-plugin/test/integration/src/verify-output.ts b/packages/jest-plugin/test/integration/src/verify-output.ts index e4d06c8..accc00d 100644 --- a/packages/jest-plugin/test/integration/src/verify-output.ts +++ b/packages/jest-plugin/test/integration/src/verify-output.ts @@ -91,39 +91,48 @@ export const verifyOutput = ( /* eslint-disable @typescript-eslint/unbound-method */ // Test our VerboseReporter customization. - (testNamePattern === undefined || - "should pass".match(testNamePattern) !== null - ? expect(stderrLines).toContainEqual - : expect(stderrLines).not.toContainEqual)( - expect.stringMatching(specResultRegexMatch("pass", "src/", "pass.test.ts")) + expect(stderrLines).toContainEqualTimes( + expect.stringMatching(specResultRegexMatch("pass", "src/", "pass.test.ts")), + testNamePattern === undefined || + "should pass".match(testNamePattern) !== null + ? 1 + : 0 ); - (testNamePattern === undefined || - "should pass".match(testNamePattern) !== null - ? expect(stderrLines).toContainEqual - : expect(stderrLines).not.toContainEqual)( + + expect(stderrLines).toContainEqualTimes( // This test doesn't have a describe() block, so it's only indented 2 spaces. - expect.stringMatching(testResultRegexMatch("pass", "should pass", 2)) + expect.stringMatching(testResultRegexMatch("pass", "should pass", 2)), + testNamePattern === undefined || + "should pass".match(testNamePattern) !== null + ? 1 + : 0 ); - (!skipFailures && - (testNamePattern === undefined || - "describe block should ([escape regex]?.*$ fail".match( - testNamePattern - ) !== null) - ? expect(stderrLines).toContainEqual - : expect(stderrLines).not.toContainEqual)( - expect.stringMatching(specResultRegexMatch("fail", "src/", "fail.test.ts")) + expect(stderrLines).toContainEqualTimes( + expect.stringMatching(specResultRegexMatch("fail", "src/", "fail.test.ts")), + !skipFailures && + (testNamePattern === undefined || + "describe block should ([escape regex]?.*$ fail".match( + testNamePattern + ) !== null) + ? expectPluginToBeEnabled + ? 1 + expectedFailureRetries + : 1 + : 0 ); - (!skipFailures && - (testNamePattern === undefined || - "describe block should ([escape regex]?.*$ fail".match( - testNamePattern - ) !== null) - ? expect(stderrLines).toContainEqual - : expect(stderrLines).not.toContainEqual)( + expect(stderrLines).toContainEqualTimes( expect.stringMatching( testResultRegexMatch("fail", "should ([escape regex]?.*$ fail") - ) + ), + !skipFailures && + (testNamePattern === undefined || + "describe block should ([escape regex]?.*$ fail".match( + testNamePattern + ) !== null) + ? expectPluginToBeEnabled + ? 1 + expectedFailureRetries + : 1 + : 0 ); const flakyTest1Name = `should be flaky 1${expectedFlakeTestNameSuffix}`; @@ -134,9 +143,7 @@ export const verifyOutput = ( !expectQuarantinedTestsToBeSkipped) && (testNamePattern === undefined || flakyTest1Name.match(testNamePattern) !== null); - (flakyTest1ShouldRun - ? expect(stderrLines).toContainEqual - : expect(stderrLines).not.toContainEqual)( + expect(stderrLines).toContainEqualTimes( expect.stringMatching( specResultRegexMatch( quarantineFlake && @@ -147,12 +154,11 @@ export const verifyOutput = ( "src/", "flake.test.ts" ) - ) + ), + flakyTest1ShouldRun ? 1 : 0 ); // This test should fail then pass (though we're not verifying the order here). - (flakyTest1ShouldRun - ? expect(stderrLines).toContainEqual - : expect(stderrLines).not.toContainEqual)( + expect(stderrLines).toContainEqualTimes( expect.stringMatching( testResultRegexMatch( quarantineFlake && @@ -163,12 +169,14 @@ export const verifyOutput = ( flakyTest1Name, 2 ) - ) + ), + flakyTest1ShouldRun ? 1 : 0 ); - (expectPluginToBeEnabled && expectedFailureRetries > 0 && flakyTest1ShouldRun - ? expect(stderrLines).toContainEqual - : expect(stderrLines).not.toContainEqual)( - expect.stringMatching(testResultRegexMatch("pass", flakyTest1Name, 2)) + expect(stderrLines).toContainEqualTimes( + expect.stringMatching(testResultRegexMatch("pass", flakyTest1Name, 2)), + expectPluginToBeEnabled && expectedFailureRetries > 0 && flakyTest1ShouldRun + ? 1 + : 0 ); const flakyTest2Name = `should be flaky 2${expectedFlakeTestNameSuffix}`; @@ -179,9 +187,7 @@ export const verifyOutput = ( !expectQuarantinedTestsToBeSkipped) && (testNamePattern === undefined || flakyTest2Name.match(testNamePattern) !== null); - (flakyTest2ShouldRun - ? expect(stderrLines).toContainEqual - : expect(stderrLines).not.toContainEqual)( + expect(stderrLines).toContainEqualTimes( expect.stringMatching( testResultRegexMatch( quarantineFlake && @@ -192,28 +198,25 @@ export const verifyOutput = ( flakyTest2Name, 2 ) - ) + ), + flakyTest2ShouldRun ? 1 : 0 ); - (expectPluginToBeEnabled && expectedFailureRetries > 0 && flakyTest2ShouldRun - ? expect(stderrLines).toContainEqual - : expect(stderrLines).not.toContainEqual)( - expect.stringMatching(testResultRegexMatch("pass", flakyTest2Name, 2)) + + expect(stderrLines).toContainEqualTimes( + expect.stringMatching(testResultRegexMatch("pass", flakyTest2Name, 2)), + expectPluginToBeEnabled && expectedFailureRetries > 0 && flakyTest2ShouldRun + ? 1 + : 0 ); - (!skipFailures - ? expect(stderrLines).toContainEqual - : expect(stderrLines).not.toContainEqual)( + expect(stderrLines).toContainEqualTimes( expect.stringMatching( specResultRegexMatch("fail", "src/", "invalid.test.ts") - ) + ), + !skipFailures ? 1 : 0 ); - (!skipQuarantined && - (!expectQuarantinedTestsToBeSkipped || failToFetchManifest) && - (testNamePattern === undefined || - "describe block should be quarantined".match(testNamePattern) !== null) - ? expect(stderrLines).toContainEqual - : expect(stderrLines).not.toContainEqual)( + expect(stderrLines).toContainEqualTimes( expect.stringMatching( specResultRegexMatch( expectPluginToBeEnabled && @@ -225,14 +228,17 @@ export const verifyOutput = ( "src/", "quarantined.test.ts" ) - ) + ), + !skipQuarantined && + (!expectQuarantinedTestsToBeSkipped || failToFetchManifest) && + (testNamePattern === undefined || + "describe block should be quarantined".match(testNamePattern) !== null) + ? expectPluginToBeEnabled + ? 1 + expectedFailureRetries + : 1 + : 0 ); - (!skipQuarantined && - (testNamePattern === undefined || - "describe block should be quarantined".match(testNamePattern) !== null) && - !expectQuarantinedTestsToBeSkipped - ? expect(stderrLines).toContainEqual - : expect(stderrLines).not.toContainEqual)( + expect(stderrLines).toContainEqualTimes( expect.stringMatching( testResultRegexMatch( expectPluginToBeEnabled && @@ -242,7 +248,16 @@ export const verifyOutput = ( : "fail", "should be quarantined" ) - ) + ), + !skipQuarantined && + (testNamePattern === undefined || + "describe block should be quarantined".match(testNamePattern) !== + null) && + !expectQuarantinedTestsToBeSkipped + ? expectPluginToBeEnabled + ? 1 + expectedFailureRetries + : 1 + : 0 ); const mixedFailTestShouldRun = @@ -259,18 +274,21 @@ export const verifyOutput = ( "mixed mixed: should pass".match(testNamePattern) !== null; // Mixed file containing both a failed test and a quarantined one. - (((!expectPluginToBeEnabled || - failToFetchManifest || - !expectQuarantinedTestsToBeQuarantined) && - mixedQuarantinedTestShouldRun) || - mixedFailTestShouldRun - ? expect(stderrLines).toContainEqual - : expect(stderrLines).not.toContainEqual)( - expect.stringMatching(specResultRegexMatch("fail", "src/", "mixed.test.ts")) + expect(stderrLines).toContainEqualTimes( + expect.stringMatching( + specResultRegexMatch("fail", "src/", "mixed.test.ts") + ), + ((!expectPluginToBeEnabled || + failToFetchManifest || + !expectQuarantinedTestsToBeQuarantined) && + mixedQuarantinedTestShouldRun) || + mixedFailTestShouldRun + ? expectPluginToBeEnabled + ? 1 + expectedFailureRetries + : 1 + : 0 ); - (mixedQuarantinedTestShouldRun - ? expect(stderrLines).toContainEqual - : expect(stderrLines).not.toContainEqual)( + expect(stderrLines).toContainEqualTimes( expect.stringMatching( testResultRegexMatch( expectPluginToBeEnabled && @@ -280,30 +298,32 @@ export const verifyOutput = ( : "fail", "mixed: should be quarantined" ) - ) + ), + mixedQuarantinedTestShouldRun + ? expectPluginToBeEnabled + ? 1 + expectedFailureRetries + : 1 + : 0 ); - (mixedFailTestShouldRun - ? expect(stderrLines).toContainEqual - : expect(stderrLines).not.toContainEqual)( - expect.stringMatching(testResultRegexMatch("fail", "mixed: should fail")) + expect(stderrLines).toContainEqualTimes( + expect.stringMatching(testResultRegexMatch("fail", "mixed: should fail")), + mixedFailTestShouldRun + ? expectPluginToBeEnabled + ? 1 + expectedFailureRetries + : 1 + : 0 ); - expect( - stderrLines.filter((line) => - testResultRegexMatch("pass", "mixed: should pass").test(line as string) - ) - ).toHaveLength(mixedPassTestShouldRun ? 1 : 0); - - // The passed test gets skipped during the retries. - if (mixedFailTestShouldRun || mixedQuarantinedTestShouldRun) { - expect( - stderrLines.filter((line) => - testResultRegexMatch("skipped", "mixed: should pass").test( - line as string - ) - ) - ).toHaveLength( - testNamePattern !== undefined && + expect(stderrLines).toContainEqualTimes( + expect.stringMatching(testResultRegexMatch("pass", "mixed: should pass")), + mixedPassTestShouldRun ? 1 : 0 + ); + expect(stderrLines).toContainEqualTimes( + expect.stringMatching( + testResultRegexMatch("skipped", "mixed: should pass") + ), + mixedFailTestShouldRun || mixedQuarantinedTestShouldRun + ? testNamePattern !== undefined && "mixed mixed: should pass".match(testNamePattern) === null && expectPluginToBeEnabled ? expectedFailureRetries + 1 @@ -313,8 +333,8 @@ export const verifyOutput = ( "mixed mixed: should pass".match(testNamePattern) === null ? 1 : 0 - ); - } + : 0 + ); // Test our SummaryReporter customization. expect(stderrLines).toContain( diff --git a/packages/jest-plugin/tsconfig.json b/packages/jest-plugin/tsconfig.json index d5ff9c4..ec572ba 100644 --- a/packages/jest-plugin/tsconfig.json +++ b/packages/jest-plugin/tsconfig.json @@ -18,6 +18,7 @@ ".eslintrc.js", "src", "test/.eslintrc.js", + "test/babel.config.js", "rollup.config.mjs", "window.d.ts" ] diff --git a/yarn.lock b/yarn.lock index 4d35e41..dc6274a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -15,16 +15,7 @@ __metadata: languageName: node linkType: hard -"@babel/code-frame@npm:^7.0.0, @babel/code-frame@npm:^7.12.13, @babel/code-frame@npm:^7.21.4": - version: 7.21.4 - resolution: "@babel/code-frame@npm:7.21.4" - dependencies: - "@babel/highlight": ^7.18.6 - checksum: e5390e6ec1ac58dcef01d4f18eaf1fd2f1325528661ff6d4a5de8979588b9f5a8e852a54a91b923846f7a5c681b217f0a45c2524eb9560553160cd963b7d592c - languageName: node - linkType: hard - -"@babel/code-frame@npm:^7.18.6": +"@babel/code-frame@npm:^7.0.0, @babel/code-frame@npm:^7.12.13, @babel/code-frame@npm:^7.18.6, @babel/code-frame@npm:^7.22.5": version: 7.22.5 resolution: "@babel/code-frame@npm:7.22.5" dependencies: @@ -33,271 +24,260 @@ __metadata: languageName: node linkType: hard -"@babel/compat-data@npm:^7.17.7, @babel/compat-data@npm:^7.20.5, @babel/compat-data@npm:^7.21.5, @babel/compat-data@npm:^7.22.0": - version: 7.22.3 - resolution: "@babel/compat-data@npm:7.22.3" - checksum: eb001646f41459f42ccb0d39ee8bb3c3c495bc297234817044c0002689c625e3159a6678c53fd31bd98cf21f31472b73506f350fc6906e3bdfa49cb706e2af8d +"@babel/compat-data@npm:^7.22.5, @babel/compat-data@npm:^7.22.6, @babel/compat-data@npm:^7.22.9": + version: 7.22.9 + resolution: "@babel/compat-data@npm:7.22.9" + checksum: bed77d9044ce948b4327b30dd0de0779fa9f3a7ed1f2d31638714ed00229fa71fc4d1617ae0eb1fad419338d3658d0e9a5a083297451e09e73e078d0347ff808 languageName: node linkType: hard -"@babel/core@npm:^7.11.6, @babel/core@npm:^7.12.3, @babel/core@npm:^7.21.5": - version: 7.22.1 - resolution: "@babel/core@npm:7.22.1" +"@babel/core@npm:^7.11.6, @babel/core@npm:^7.12.3, @babel/core@npm:^7.22.9": + version: 7.22.9 + resolution: "@babel/core@npm:7.22.9" dependencies: "@ampproject/remapping": ^2.2.0 - "@babel/code-frame": ^7.21.4 - "@babel/generator": ^7.22.0 - "@babel/helper-compilation-targets": ^7.22.1 - "@babel/helper-module-transforms": ^7.22.1 - "@babel/helpers": ^7.22.0 - "@babel/parser": ^7.22.0 - "@babel/template": ^7.21.9 - "@babel/traverse": ^7.22.1 - "@babel/types": ^7.22.0 + "@babel/code-frame": ^7.22.5 + "@babel/generator": ^7.22.9 + "@babel/helper-compilation-targets": ^7.22.9 + "@babel/helper-module-transforms": ^7.22.9 + "@babel/helpers": ^7.22.6 + "@babel/parser": ^7.22.7 + "@babel/template": ^7.22.5 + "@babel/traverse": ^7.22.8 + "@babel/types": ^7.22.5 convert-source-map: ^1.7.0 debug: ^4.1.0 gensync: ^1.0.0-beta.2 json5: ^2.2.2 - semver: ^6.3.0 - checksum: bbe45e791f223a7e692d2ea6597a73f48050abd24b119c85c48ac6504c30ce63343a2ea3f79b5847bf4b409ddd8a68b6cdc4f0272ded1d2ef6f6b1e9663432f0 + semver: ^6.3.1 + checksum: 7bf069aeceb417902c4efdaefab1f7b94adb7dea694a9aed1bda2edf4135348a080820529b1a300c6f8605740a00ca00c19b2d5e74b5dd489d99d8c11d5e56d1 languageName: node linkType: hard -"@babel/generator@npm:^7.22.0, @babel/generator@npm:^7.22.3, @babel/generator@npm:^7.7.2": - version: 7.22.3 - resolution: "@babel/generator@npm:7.22.3" +"@babel/generator@npm:^7.22.7, @babel/generator@npm:^7.22.9, @babel/generator@npm:^7.7.2": + version: 7.22.9 + resolution: "@babel/generator@npm:7.22.9" dependencies: - "@babel/types": ^7.22.3 + "@babel/types": ^7.22.5 "@jridgewell/gen-mapping": ^0.3.2 "@jridgewell/trace-mapping": ^0.3.17 jsesc: ^2.5.1 - checksum: ccb6426ca5b5a38f0d47a3ac9628e223d2aaaa489cbf90ffab41468795c22afe86855f68a58667f0f2673949f1810d4d5a57b826c17984eab3e28fdb34a909e6 + checksum: 7c9d2c58b8d5ac5e047421a6ab03ec2ff5d9a5ff2c2212130a0055e063ac349e0b19d435537d6886c999771aef394832e4f54cd9fc810100a7f23d982f6af06b languageName: node linkType: hard -"@babel/helper-annotate-as-pure@npm:^7.18.6": - version: 7.18.6 - resolution: "@babel/helper-annotate-as-pure@npm:7.18.6" +"@babel/helper-annotate-as-pure@npm:^7.22.5": + version: 7.22.5 + resolution: "@babel/helper-annotate-as-pure@npm:7.22.5" dependencies: - "@babel/types": ^7.18.6 - checksum: 88ccd15ced475ef2243fdd3b2916a29ea54c5db3cd0cfabf9d1d29ff6e63b7f7cd1c27264137d7a40ac2e978b9b9a542c332e78f40eb72abe737a7400788fc1b + "@babel/types": ^7.22.5 + checksum: 53da330f1835c46f26b7bf4da31f7a496dee9fd8696cca12366b94ba19d97421ce519a74a837f687749318f94d1a37f8d1abcbf35e8ed22c32d16373b2f6198d languageName: node linkType: hard -"@babel/helper-builder-binary-assignment-operator-visitor@npm:^7.18.6": - version: 7.21.5 - resolution: "@babel/helper-builder-binary-assignment-operator-visitor@npm:7.21.5" +"@babel/helper-builder-binary-assignment-operator-visitor@npm:^7.22.5": + version: 7.22.5 + resolution: "@babel/helper-builder-binary-assignment-operator-visitor@npm:7.22.5" dependencies: - "@babel/types": ^7.21.5 - checksum: 9a033d3d7a6409256272ea6fc03731511af9f936ee0b161ace05d171d7bd5adf455dc85f80437d92277462f6bd2af9af1f2d1967edc21ca4d5966ac0a09cf61d + "@babel/types": ^7.22.5 + checksum: d753acac62399fc6dd354cf1b9441bde0c331c2fe792a4c14904c5e5eafc3cac79478f6aa038e8a51c1148b0af6710a2e619855e4b5d54497ac972eaffed5884 languageName: node linkType: hard -"@babel/helper-compilation-targets@npm:^7.17.7, @babel/helper-compilation-targets@npm:^7.18.9, @babel/helper-compilation-targets@npm:^7.20.7, @babel/helper-compilation-targets@npm:^7.21.5, @babel/helper-compilation-targets@npm:^7.22.1": - version: 7.22.1 - resolution: "@babel/helper-compilation-targets@npm:7.22.1" +"@babel/helper-compilation-targets@npm:^7.22.5, @babel/helper-compilation-targets@npm:^7.22.6, @babel/helper-compilation-targets@npm:^7.22.9": + version: 7.22.9 + resolution: "@babel/helper-compilation-targets@npm:7.22.9" dependencies: - "@babel/compat-data": ^7.22.0 - "@babel/helper-validator-option": ^7.21.0 - browserslist: ^4.21.3 + "@babel/compat-data": ^7.22.9 + "@babel/helper-validator-option": ^7.22.5 + browserslist: ^4.21.9 lru-cache: ^5.1.1 - semver: ^6.3.0 + semver: ^6.3.1 peerDependencies: "@babel/core": ^7.0.0 - checksum: a686a01bd3288cf95ca26faa27958d34c04e2501c4b0858c3a6558776dec20317b5635f33d64c5a635b6fbdfe462a85c30d4bfa0ae7e7ffe3467e4d06442d7c8 + checksum: ea0006c6a93759025f4a35a25228ae260538c9f15023e8aac2a6d45ca68aef4cf86cfc429b19af9a402cbdd54d5de74ad3fbcf6baa7e48184dc079f1a791e178 languageName: node linkType: hard -"@babel/helper-create-class-features-plugin@npm:^7.18.6, @babel/helper-create-class-features-plugin@npm:^7.21.0": - version: 7.21.5 - resolution: "@babel/helper-create-class-features-plugin@npm:7.21.5" +"@babel/helper-create-class-features-plugin@npm:^7.22.5, @babel/helper-create-class-features-plugin@npm:^7.22.9": + version: 7.22.9 + resolution: "@babel/helper-create-class-features-plugin@npm:7.22.9" dependencies: - "@babel/helper-annotate-as-pure": ^7.18.6 - "@babel/helper-environment-visitor": ^7.21.5 - "@babel/helper-function-name": ^7.21.0 - "@babel/helper-member-expression-to-functions": ^7.21.5 - "@babel/helper-optimise-call-expression": ^7.18.6 - "@babel/helper-replace-supers": ^7.21.5 - "@babel/helper-skip-transparent-expression-wrappers": ^7.20.0 - "@babel/helper-split-export-declaration": ^7.18.6 - semver: ^6.3.0 + "@babel/helper-annotate-as-pure": ^7.22.5 + "@babel/helper-environment-visitor": ^7.22.5 + "@babel/helper-function-name": ^7.22.5 + "@babel/helper-member-expression-to-functions": ^7.22.5 + "@babel/helper-optimise-call-expression": ^7.22.5 + "@babel/helper-replace-supers": ^7.22.9 + "@babel/helper-skip-transparent-expression-wrappers": ^7.22.5 + "@babel/helper-split-export-declaration": ^7.22.6 + semver: ^6.3.1 peerDependencies: "@babel/core": ^7.0.0 - checksum: cf1bcdd5cf2949927ba63002381cc7db22d1c8ef12b85aacc5c6361ae538522f947e57c59a787f5ee44c5413cf881a3d76224f5583d2c0575282c7c1f68df797 + checksum: 6c2436d1a5a3f1ff24628d78fa8c6d3120c40285aa3eda7815b1adbf8c5951e0dd73d368cf845825888fa3dc2f207dadce53309825598d7c67953e5ed9dd51d2 languageName: node linkType: hard -"@babel/helper-create-regexp-features-plugin@npm:^7.18.6, @babel/helper-create-regexp-features-plugin@npm:^7.20.5": - version: 7.21.5 - resolution: "@babel/helper-create-regexp-features-plugin@npm:7.21.5" +"@babel/helper-create-regexp-features-plugin@npm:^7.18.6, @babel/helper-create-regexp-features-plugin@npm:^7.22.5": + version: 7.22.9 + resolution: "@babel/helper-create-regexp-features-plugin@npm:7.22.9" dependencies: - "@babel/helper-annotate-as-pure": ^7.18.6 + "@babel/helper-annotate-as-pure": ^7.22.5 regexpu-core: ^5.3.1 - semver: ^6.3.0 + semver: ^6.3.1 peerDependencies: "@babel/core": ^7.0.0 - checksum: c38cb01b242b0b2bb9783072e6ba4d4aa08c66ea39f9b74a45f31f95a6fe2ff3ba782d8ce09827c09939450d2d39a6db41c83051ef191482bfb67b63a5023e24 + checksum: 87cb48a7ee898ab205374274364c3adc70b87b08c7bd07f51019ae4562c0170d7148e654d591f825dee14b5fe11666a0e7966872dfdbfa0d1b94b861ecf0e4e1 languageName: node linkType: hard -"@babel/helper-define-polyfill-provider@npm:^0.3.3": - version: 0.3.3 - resolution: "@babel/helper-define-polyfill-provider@npm:0.3.3" +"@babel/helper-define-polyfill-provider@npm:^0.4.2": + version: 0.4.2 + resolution: "@babel/helper-define-polyfill-provider@npm:0.4.2" dependencies: - "@babel/helper-compilation-targets": ^7.17.7 - "@babel/helper-plugin-utils": ^7.16.7 + "@babel/helper-compilation-targets": ^7.22.6 + "@babel/helper-plugin-utils": ^7.22.5 debug: ^4.1.1 lodash.debounce: ^4.0.8 resolve: ^1.14.2 - semver: ^6.1.2 peerDependencies: - "@babel/core": ^7.4.0-0 - checksum: 8e3fe75513302e34f6d92bd67b53890e8545e6c5bca8fe757b9979f09d68d7e259f6daea90dc9e01e332c4f8781bda31c5fe551c82a277f9bc0bec007aed497c + "@babel/core": ^7.4.0 || ^8.0.0-0 <8.0.0 + checksum: 1f6dec0c5d0876d278fe15b71238eccc5f74c4e2efa2c78aaafa8bc2cc96336b8e68d94cd1a78497356c96e8b91b8c1f4452179820624d1702aee2f9832e6569 languageName: node linkType: hard -"@babel/helper-environment-visitor@npm:^7.18.9, @babel/helper-environment-visitor@npm:^7.21.5, @babel/helper-environment-visitor@npm:^7.22.1": - version: 7.22.1 - resolution: "@babel/helper-environment-visitor@npm:7.22.1" - checksum: a6b4bb5505453bff95518d361ac1de393f0029aeb8b690c70540f4317934c53c43cc4afcda8c752ffa8c272e63ed6b929a56eca28e4978424177b24238b21bf9 +"@babel/helper-environment-visitor@npm:^7.22.5": + version: 7.22.5 + resolution: "@babel/helper-environment-visitor@npm:7.22.5" + checksum: 248532077d732a34cd0844eb7b078ff917c3a8ec81a7f133593f71a860a582f05b60f818dc5049c2212e5baa12289c27889a4b81d56ef409b4863db49646c4b1 languageName: node linkType: hard -"@babel/helper-function-name@npm:^7.18.9, @babel/helper-function-name@npm:^7.19.0, @babel/helper-function-name@npm:^7.21.0": - version: 7.21.0 - resolution: "@babel/helper-function-name@npm:7.21.0" +"@babel/helper-function-name@npm:^7.22.5": + version: 7.22.5 + resolution: "@babel/helper-function-name@npm:7.22.5" dependencies: - "@babel/template": ^7.20.7 - "@babel/types": ^7.21.0 - checksum: d63e63c3e0e3e8b3138fa47b0cd321148a300ef12b8ee951196994dcd2a492cc708aeda94c2c53759a5c9177fffaac0fd8778791286746f72a000976968daf4e + "@babel/template": ^7.22.5 + "@babel/types": ^7.22.5 + checksum: 6b1f6ce1b1f4e513bf2c8385a557ea0dd7fa37971b9002ad19268ca4384bbe90c09681fe4c076013f33deabc63a53b341ed91e792de741b4b35e01c00238177a languageName: node linkType: hard -"@babel/helper-hoist-variables@npm:^7.18.6": - version: 7.18.6 - resolution: "@babel/helper-hoist-variables@npm:7.18.6" +"@babel/helper-hoist-variables@npm:^7.22.5": + version: 7.22.5 + resolution: "@babel/helper-hoist-variables@npm:7.22.5" dependencies: - "@babel/types": ^7.18.6 - checksum: fd9c35bb435fda802bf9ff7b6f2df06308a21277c6dec2120a35b09f9de68f68a33972e2c15505c1a1a04b36ec64c9ace97d4a9e26d6097b76b4396b7c5fa20f + "@babel/types": ^7.22.5 + checksum: 394ca191b4ac908a76e7c50ab52102669efe3a1c277033e49467913c7ed6f7c64d7eacbeabf3bed39ea1f41731e22993f763b1edce0f74ff8563fd1f380d92cc languageName: node linkType: hard -"@babel/helper-member-expression-to-functions@npm:^7.21.5": - version: 7.21.5 - resolution: "@babel/helper-member-expression-to-functions@npm:7.21.5" +"@babel/helper-member-expression-to-functions@npm:^7.22.5": + version: 7.22.5 + resolution: "@babel/helper-member-expression-to-functions@npm:7.22.5" dependencies: - "@babel/types": ^7.21.5 - checksum: c404b4a0271c640b7dc8c34af7b683c70a43200259e02330cfc02e79e6b271e9227f35554cd6ad015eabcfa1fea75b9d0b87b69f3d1e6c2af6edd224060b1732 + "@babel/types": ^7.22.5 + checksum: 4bd5791529c280c00743e8bdc669ef0d4cd1620d6e3d35e0d42b862f8262bc2364973e5968007f960780344c539a4b9cf92ab41f5b4f94560a9620f536de2a39 languageName: node linkType: hard -"@babel/helper-module-imports@npm:^7.18.6, @babel/helper-module-imports@npm:^7.21.4": - version: 7.21.4 - resolution: "@babel/helper-module-imports@npm:7.21.4" +"@babel/helper-module-imports@npm:^7.22.5": + version: 7.22.5 + resolution: "@babel/helper-module-imports@npm:7.22.5" dependencies: - "@babel/types": ^7.21.4 - checksum: bd330a2edaafeb281fbcd9357652f8d2666502567c0aad71db926e8499c773c9ea9c10dfaae30122452940326d90c8caff5c649ed8e1bf15b23f858758d3abc6 + "@babel/types": ^7.22.5 + checksum: 9ac2b0404fa38b80bdf2653fbeaf8e8a43ccb41bd505f9741d820ed95d3c4e037c62a1bcdcb6c9527d7798d2e595924c4d025daed73283badc180ada2c9c49ad languageName: node linkType: hard -"@babel/helper-module-transforms@npm:^7.18.6, @babel/helper-module-transforms@npm:^7.20.11, @babel/helper-module-transforms@npm:^7.21.5, @babel/helper-module-transforms@npm:^7.22.1": - version: 7.22.1 - resolution: "@babel/helper-module-transforms@npm:7.22.1" +"@babel/helper-module-transforms@npm:^7.22.5, @babel/helper-module-transforms@npm:^7.22.9": + version: 7.22.9 + resolution: "@babel/helper-module-transforms@npm:7.22.9" dependencies: - "@babel/helper-environment-visitor": ^7.22.1 - "@babel/helper-module-imports": ^7.21.4 - "@babel/helper-simple-access": ^7.21.5 - "@babel/helper-split-export-declaration": ^7.18.6 - "@babel/helper-validator-identifier": ^7.19.1 - "@babel/template": ^7.21.9 - "@babel/traverse": ^7.22.1 - "@babel/types": ^7.22.0 - checksum: dfa084211a93c9f0174ab07385fdbf7831bbf5c1ff3d4f984effc489f48670825ad8b817b9e9d2ec6492fde37ed6518c15944e9dd7a60b43a3d9874c9250f5f8 + "@babel/helper-environment-visitor": ^7.22.5 + "@babel/helper-module-imports": ^7.22.5 + "@babel/helper-simple-access": ^7.22.5 + "@babel/helper-split-export-declaration": ^7.22.6 + "@babel/helper-validator-identifier": ^7.22.5 + peerDependencies: + "@babel/core": ^7.0.0 + checksum: 2751f77660518cf4ff027514d6f4794f04598c6393be7b04b8e46c6e21606e11c19f3f57ab6129a9c21bacdf8b3ffe3af87bb401d972f34af2d0ffde02ac3001 languageName: node linkType: hard -"@babel/helper-optimise-call-expression@npm:^7.18.6": - version: 7.18.6 - resolution: "@babel/helper-optimise-call-expression@npm:7.18.6" +"@babel/helper-optimise-call-expression@npm:^7.22.5": + version: 7.22.5 + resolution: "@babel/helper-optimise-call-expression@npm:7.22.5" dependencies: - "@babel/types": ^7.18.6 - checksum: e518fe8418571405e21644cfb39cf694f30b6c47b10b006609a92469ae8b8775cbff56f0b19732343e2ea910641091c5a2dc73b56ceba04e116a33b0f8bd2fbd + "@babel/types": ^7.22.5 + checksum: c70ef6cc6b6ed32eeeec4482127e8be5451d0e5282d5495d5d569d39eb04d7f1d66ec99b327f45d1d5842a9ad8c22d48567e93fc502003a47de78d122e355f7c languageName: node linkType: hard -"@babel/helper-plugin-utils@npm:^7.0.0, @babel/helper-plugin-utils@npm:^7.10.4, @babel/helper-plugin-utils@npm:^7.12.13, @babel/helper-plugin-utils@npm:^7.14.5, @babel/helper-plugin-utils@npm:^7.16.7, @babel/helper-plugin-utils@npm:^7.18.6, @babel/helper-plugin-utils@npm:^7.18.9, @babel/helper-plugin-utils@npm:^7.19.0, @babel/helper-plugin-utils@npm:^7.20.2, @babel/helper-plugin-utils@npm:^7.21.5, @babel/helper-plugin-utils@npm:^7.8.0, @babel/helper-plugin-utils@npm:^7.8.3": - version: 7.21.5 - resolution: "@babel/helper-plugin-utils@npm:7.21.5" - checksum: 6f086e9a84a50ea7df0d5639c8f9f68505af510ea3258b3c8ac8b175efdfb7f664436cb48996f71791a1350ba68f47ad3424131e8e718c5e2ad45564484cbb36 +"@babel/helper-plugin-utils@npm:^7.0.0, @babel/helper-plugin-utils@npm:^7.10.4, @babel/helper-plugin-utils@npm:^7.12.13, @babel/helper-plugin-utils@npm:^7.14.5, @babel/helper-plugin-utils@npm:^7.18.6, @babel/helper-plugin-utils@npm:^7.22.5, @babel/helper-plugin-utils@npm:^7.8.0, @babel/helper-plugin-utils@npm:^7.8.3": + version: 7.22.5 + resolution: "@babel/helper-plugin-utils@npm:7.22.5" + checksum: c0fc7227076b6041acd2f0e818145d2e8c41968cc52fb5ca70eed48e21b8fe6dd88a0a91cbddf4951e33647336eb5ae184747ca706817ca3bef5e9e905151ff5 languageName: node linkType: hard -"@babel/helper-remap-async-to-generator@npm:^7.18.9": - version: 7.18.9 - resolution: "@babel/helper-remap-async-to-generator@npm:7.18.9" +"@babel/helper-remap-async-to-generator@npm:^7.22.5": + version: 7.22.9 + resolution: "@babel/helper-remap-async-to-generator@npm:7.22.9" dependencies: - "@babel/helper-annotate-as-pure": ^7.18.6 - "@babel/helper-environment-visitor": ^7.18.9 - "@babel/helper-wrap-function": ^7.18.9 - "@babel/types": ^7.18.9 + "@babel/helper-annotate-as-pure": ^7.22.5 + "@babel/helper-environment-visitor": ^7.22.5 + "@babel/helper-wrap-function": ^7.22.9 peerDependencies: "@babel/core": ^7.0.0 - checksum: 4be6076192308671b046245899b703ba090dbe7ad03e0bea897bb2944ae5b88e5e85853c9d1f83f643474b54c578d8ac0800b80341a86e8538264a725fbbefec + checksum: 05538079447829b13512157491cc77f9cf1ea7e1680e15cff0682c3ed9ee162de0c4862ece20a6d6b2df28177a1520bcfe45993fbeccf2747a81795a7c3f6290 languageName: node linkType: hard -"@babel/helper-replace-supers@npm:^7.18.6, @babel/helper-replace-supers@npm:^7.20.7, @babel/helper-replace-supers@npm:^7.21.5": - version: 7.21.5 - resolution: "@babel/helper-replace-supers@npm:7.21.5" +"@babel/helper-replace-supers@npm:^7.22.5, @babel/helper-replace-supers@npm:^7.22.9": + version: 7.22.9 + resolution: "@babel/helper-replace-supers@npm:7.22.9" dependencies: - "@babel/helper-environment-visitor": ^7.21.5 - "@babel/helper-member-expression-to-functions": ^7.21.5 - "@babel/helper-optimise-call-expression": ^7.18.6 - "@babel/template": ^7.20.7 - "@babel/traverse": ^7.21.5 - "@babel/types": ^7.21.5 - checksum: 4fd343e6f90533743d8e8a1f42e50377b3d6b27f524a27eb97ff28f075e4e55cca2383adb1b0973de358b08022aef0fec4c8d69711e1da43bf9b887b5a893677 + "@babel/helper-environment-visitor": ^7.22.5 + "@babel/helper-member-expression-to-functions": ^7.22.5 + "@babel/helper-optimise-call-expression": ^7.22.5 + peerDependencies: + "@babel/core": ^7.0.0 + checksum: d41471f56ff2616459d35a5df1900d5f0756ae78b1027040365325ef332d66e08e3be02a9489756d870887585ff222403a228546e93dd7019e19e59c0c0fe586 languageName: node linkType: hard -"@babel/helper-simple-access@npm:^7.21.5": - version: 7.21.5 - resolution: "@babel/helper-simple-access@npm:7.21.5" +"@babel/helper-simple-access@npm:^7.22.5": + version: 7.22.5 + resolution: "@babel/helper-simple-access@npm:7.22.5" dependencies: - "@babel/types": ^7.21.5 - checksum: ad212beaa24be3864c8c95bee02f840222457ccf5419991e2d3e3e39b0f75b77e7e857e0bf4ed428b1cd97acefc87f3831bdb0b9696d5ad0557421f398334fc3 + "@babel/types": ^7.22.5 + checksum: fe9686714caf7d70aedb46c3cce090f8b915b206e09225f1e4dbc416786c2fdbbee40b38b23c268b7ccef749dd2db35f255338fb4f2444429874d900dede5ad2 languageName: node linkType: hard -"@babel/helper-skip-transparent-expression-wrappers@npm:^7.20.0": - version: 7.20.0 - resolution: "@babel/helper-skip-transparent-expression-wrappers@npm:7.20.0" +"@babel/helper-skip-transparent-expression-wrappers@npm:^7.22.5": + version: 7.22.5 + resolution: "@babel/helper-skip-transparent-expression-wrappers@npm:7.22.5" dependencies: - "@babel/types": ^7.20.0 - checksum: 34da8c832d1c8a546e45d5c1d59755459ffe43629436707079989599b91e8c19e50e73af7a4bd09c95402d389266731b0d9c5f69e372d8ebd3a709c05c80d7dd + "@babel/types": ^7.22.5 + checksum: 1012ef2295eb12dc073f2b9edf3425661e9b8432a3387e62a8bc27c42963f1f216ab3124228015c748770b2257b4f1fda882ca8fa34c0bf485e929ae5bc45244 languageName: node linkType: hard -"@babel/helper-split-export-declaration@npm:^7.18.6": - version: 7.18.6 - resolution: "@babel/helper-split-export-declaration@npm:7.18.6" +"@babel/helper-split-export-declaration@npm:^7.22.6": + version: 7.22.6 + resolution: "@babel/helper-split-export-declaration@npm:7.22.6" dependencies: - "@babel/types": ^7.18.6 - checksum: c6d3dede53878f6be1d869e03e9ffbbb36f4897c7cc1527dc96c56d127d834ffe4520a6f7e467f5b6f3c2843ea0e81a7819d66ae02f707f6ac057f3d57943a2b + "@babel/types": ^7.22.5 + checksum: e141cace583b19d9195f9c2b8e17a3ae913b7ee9b8120246d0f9ca349ca6f03cb2c001fd5ec57488c544347c0bb584afec66c936511e447fd20a360e591ac921 languageName: node linkType: hard -"@babel/helper-string-parser@npm:^7.21.5": - version: 7.21.5 - resolution: "@babel/helper-string-parser@npm:7.21.5" - checksum: 36c0ded452f3858e67634b81960d4bde1d1cd2a56b82f4ba2926e97864816021c885f111a7cf81de88a0ed025f49d84a393256700e9acbca2d99462d648705d8 - languageName: node - linkType: hard - -"@babel/helper-validator-identifier@npm:^7.18.6, @babel/helper-validator-identifier@npm:^7.19.1": - version: 7.19.1 - resolution: "@babel/helper-validator-identifier@npm:7.19.1" - checksum: 0eca5e86a729162af569b46c6c41a63e18b43dbe09fda1d2a3c8924f7d617116af39cac5e4cd5d431bb760b4dca3c0970e0c444789b1db42bcf1fa41fbad0a3a +"@babel/helper-string-parser@npm:^7.22.5": + version: 7.22.5 + resolution: "@babel/helper-string-parser@npm:7.22.5" + checksum: 836851ca5ec813077bbb303acc992d75a360267aa3b5de7134d220411c852a6f17de7c0d0b8c8dcc0f567f67874c00f4528672b2a4f1bc978a3ada64c8c78467 languageName: node linkType: hard @@ -308,44 +288,32 @@ __metadata: languageName: node linkType: hard -"@babel/helper-validator-option@npm:^7.21.0": - version: 7.21.0 - resolution: "@babel/helper-validator-option@npm:7.21.0" - checksum: 8ece4c78ffa5461fd8ab6b6e57cc51afad59df08192ed5d84b475af4a7193fc1cb794b59e3e7be64f3cdc4df7ac78bf3dbb20c129d7757ae078e6279ff8c2f07 - languageName: node - linkType: hard - -"@babel/helper-wrap-function@npm:^7.18.9": - version: 7.20.5 - resolution: "@babel/helper-wrap-function@npm:7.20.5" - dependencies: - "@babel/helper-function-name": ^7.19.0 - "@babel/template": ^7.18.10 - "@babel/traverse": ^7.20.5 - "@babel/types": ^7.20.5 - checksum: 11a6fc28334368a193a9cb3ad16f29cd7603bab958433efc82ebe59fa6556c227faa24f07ce43983f7a85df826f71d441638442c4315e90a554fe0a70ca5005b +"@babel/helper-validator-option@npm:^7.22.5": + version: 7.22.5 + resolution: "@babel/helper-validator-option@npm:7.22.5" + checksum: bbeca8a85ee86990215c0424997438b388b8d642d69b9f86c375a174d3cdeb270efafd1ff128bc7a1d370923d13b6e45829ba8581c027620e83e3a80c5c414b3 languageName: node linkType: hard -"@babel/helpers@npm:^7.22.0": - version: 7.22.3 - resolution: "@babel/helpers@npm:7.22.3" +"@babel/helper-wrap-function@npm:^7.22.9": + version: 7.22.9 + resolution: "@babel/helper-wrap-function@npm:7.22.9" dependencies: - "@babel/template": ^7.21.9 - "@babel/traverse": ^7.22.1 - "@babel/types": ^7.22.3 - checksum: 385289ee8b87cf9af448bbb9fcf747f6e67600db5f7f64eb4ad97761ee387819bf2212b6a757008286c6bfacf4f3fc0b6de88686f2e517a70fb59996bdfbd1e9 + "@babel/helper-function-name": ^7.22.5 + "@babel/template": ^7.22.5 + "@babel/types": ^7.22.5 + checksum: 037317dc06dac6593e388738ae1d3e43193bc1d31698f067c0ef3d4dc6f074dbed860ed42aa137b48a67aa7cb87336826c4bdc13189260481bcf67eb7256c789 languageName: node linkType: hard -"@babel/highlight@npm:^7.18.6": - version: 7.18.6 - resolution: "@babel/highlight@npm:7.18.6" +"@babel/helpers@npm:^7.22.6": + version: 7.22.6 + resolution: "@babel/helpers@npm:7.22.6" dependencies: - "@babel/helper-validator-identifier": ^7.18.6 - chalk: ^2.0.0 - js-tokens: ^4.0.0 - checksum: 92d8ee61549de5ff5120e945e774728e5ccd57fd3b2ed6eace020ec744823d4a98e242be1453d21764a30a14769ecd62170fba28539b211799bbaf232bbb2789 + "@babel/template": ^7.22.5 + "@babel/traverse": ^7.22.6 + "@babel/types": ^7.22.5 + checksum: 5c1f33241fe7bf7709868c2105134a0a86dca26a0fbd508af10a89312b1f77ca38ebae43e50be3b208613c5eacca1559618af4ca236f0abc55d294800faeff30 languageName: node linkType: hard @@ -360,217 +328,49 @@ __metadata: languageName: node linkType: hard -"@babel/parser@npm:^7.1.0, @babel/parser@npm:^7.14.7, @babel/parser@npm:^7.20.7, @babel/parser@npm:^7.21.9, @babel/parser@npm:^7.22.0, @babel/parser@npm:^7.22.4": - version: 7.22.4 - resolution: "@babel/parser@npm:7.22.4" +"@babel/parser@npm:^7.1.0, @babel/parser@npm:^7.14.7, @babel/parser@npm:^7.20.7, @babel/parser@npm:^7.22.5, @babel/parser@npm:^7.22.7": + version: 7.22.7 + resolution: "@babel/parser@npm:7.22.7" bin: parser: ./bin/babel-parser.js - checksum: 0ca6d3a2d9aae2504ba1bc494704b64a83140884f7379f609de69bd39b60adb58a4f8ec692fe53fef8657dd82705d01b7e6efb65e18296326bdd66f71d52d9a9 + checksum: 02209ddbd445831ee8bf966fdf7c29d189ed4b14343a68eb2479d940e7e3846340d7cc6bd654a5f3d87d19dc84f49f50a58cf9363bee249dc5409ff3ba3dab54 languageName: node linkType: hard -"@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression@npm:^7.18.6": - version: 7.18.6 - resolution: "@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression@npm:7.18.6" +"@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression@npm:^7.22.5": + version: 7.22.5 + resolution: "@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression@npm:7.22.5" dependencies: - "@babel/helper-plugin-utils": ^7.18.6 + "@babel/helper-plugin-utils": ^7.22.5 peerDependencies: "@babel/core": ^7.0.0 - checksum: 845bd280c55a6a91d232cfa54eaf9708ec71e594676fe705794f494bb8b711d833b752b59d1a5c154695225880c23dbc9cab0e53af16fd57807976cd3ff41b8d + checksum: 1e353a060fb2cd8f1256d28cd768f16fb02513f905b9b6d656fb0242c96c341a196fa188b27c2701506a6e27515359fbcc1a5ca7fa8b9b530cf88fbd137baefc languageName: node linkType: hard -"@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining@npm:^7.20.7": - version: 7.20.7 - resolution: "@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining@npm:7.20.7" +"@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining@npm:^7.22.5": + version: 7.22.5 + resolution: "@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining@npm:7.22.5" dependencies: - "@babel/helper-plugin-utils": ^7.20.2 - "@babel/helper-skip-transparent-expression-wrappers": ^7.20.0 - "@babel/plugin-proposal-optional-chaining": ^7.20.7 + "@babel/helper-plugin-utils": ^7.22.5 + "@babel/helper-skip-transparent-expression-wrappers": ^7.22.5 + "@babel/plugin-transform-optional-chaining": ^7.22.5 peerDependencies: "@babel/core": ^7.13.0 - checksum: d610f532210bee5342f5b44a12395ccc6d904e675a297189bc1e401cc185beec09873da523466d7fec34ae1574f7a384235cba1ccc9fe7b89ba094167897c845 + checksum: 16e7a5f3bf2f2ac0ca032a70bf0ebd7e886d84dbb712b55c0643c04c495f0f221fbcbca14b5f8f8027fa6c87a3dafae0934022ad2b409384af6c5c356495b7bd languageName: node linkType: hard -"@babel/plugin-proposal-async-generator-functions@npm:^7.20.7": - version: 7.20.7 - resolution: "@babel/plugin-proposal-async-generator-functions@npm:7.20.7" - dependencies: - "@babel/helper-environment-visitor": ^7.18.9 - "@babel/helper-plugin-utils": ^7.20.2 - "@babel/helper-remap-async-to-generator": ^7.18.9 - "@babel/plugin-syntax-async-generators": ^7.8.4 +"@babel/plugin-proposal-private-property-in-object@npm:7.21.0-placeholder-for-preset-env.2": + version: 7.21.0-placeholder-for-preset-env.2 + resolution: "@babel/plugin-proposal-private-property-in-object@npm:7.21.0-placeholder-for-preset-env.2" peerDependencies: "@babel/core": ^7.0.0-0 - checksum: 111109ee118c9e69982f08d5e119eab04190b36a0f40e22e873802d941956eee66d2aa5a15f5321e51e3f9aa70a91136451b987fe15185ef8cc547ac88937723 + checksum: d97745d098b835d55033ff3a7fb2b895b9c5295b08a5759e4f20df325aa385a3e0bc9bd5ad8f2ec554a44d4e6525acfc257b8c5848a1345cb40f26a30e277e91 languageName: node linkType: hard -"@babel/plugin-proposal-class-properties@npm:^7.18.6": - version: 7.18.6 - resolution: "@babel/plugin-proposal-class-properties@npm:7.18.6" - dependencies: - "@babel/helper-create-class-features-plugin": ^7.18.6 - "@babel/helper-plugin-utils": ^7.18.6 - peerDependencies: - "@babel/core": ^7.0.0-0 - checksum: 49a78a2773ec0db56e915d9797e44fd079ab8a9b2e1716e0df07c92532f2c65d76aeda9543883916b8e0ff13606afeffa67c5b93d05b607bc87653ad18a91422 - languageName: node - linkType: hard - -"@babel/plugin-proposal-class-static-block@npm:^7.21.0": - version: 7.21.0 - resolution: "@babel/plugin-proposal-class-static-block@npm:7.21.0" - dependencies: - "@babel/helper-create-class-features-plugin": ^7.21.0 - "@babel/helper-plugin-utils": ^7.20.2 - "@babel/plugin-syntax-class-static-block": ^7.14.5 - peerDependencies: - "@babel/core": ^7.12.0 - checksum: 236c0ad089e7a7acab776cc1d355330193314bfcd62e94e78f2df35817c6144d7e0e0368976778afd6b7c13e70b5068fa84d7abbf967d4f182e60d03f9ef802b - languageName: node - linkType: hard - -"@babel/plugin-proposal-dynamic-import@npm:^7.18.6": - version: 7.18.6 - resolution: "@babel/plugin-proposal-dynamic-import@npm:7.18.6" - dependencies: - "@babel/helper-plugin-utils": ^7.18.6 - "@babel/plugin-syntax-dynamic-import": ^7.8.3 - peerDependencies: - "@babel/core": ^7.0.0-0 - checksum: 96b1c8a8ad8171d39e9ab106be33bde37ae09b22fb2c449afee9a5edf3c537933d79d963dcdc2694d10677cb96da739cdf1b53454e6a5deab9801f28a818bb2f - languageName: node - linkType: hard - -"@babel/plugin-proposal-export-namespace-from@npm:^7.18.9": - version: 7.18.9 - resolution: "@babel/plugin-proposal-export-namespace-from@npm:7.18.9" - dependencies: - "@babel/helper-plugin-utils": ^7.18.9 - "@babel/plugin-syntax-export-namespace-from": ^7.8.3 - peerDependencies: - "@babel/core": ^7.0.0-0 - checksum: 84ff22bacc5d30918a849bfb7e0e90ae4c5b8d8b65f2ac881803d1cf9068dffbe53bd657b0e4bc4c20b4db301b1c85f1e74183cf29a0dd31e964bd4e97c363ef - languageName: node - linkType: hard - -"@babel/plugin-proposal-json-strings@npm:^7.18.6": - version: 7.18.6 - resolution: "@babel/plugin-proposal-json-strings@npm:7.18.6" - dependencies: - "@babel/helper-plugin-utils": ^7.18.6 - "@babel/plugin-syntax-json-strings": ^7.8.3 - peerDependencies: - "@babel/core": ^7.0.0-0 - checksum: 25ba0e6b9d6115174f51f7c6787e96214c90dd4026e266976b248a2ed417fe50fddae72843ffb3cbe324014a18632ce5648dfac77f089da858022b49fd608cb3 - languageName: node - linkType: hard - -"@babel/plugin-proposal-logical-assignment-operators@npm:^7.20.7": - version: 7.20.7 - resolution: "@babel/plugin-proposal-logical-assignment-operators@npm:7.20.7" - dependencies: - "@babel/helper-plugin-utils": ^7.20.2 - "@babel/plugin-syntax-logical-assignment-operators": ^7.10.4 - peerDependencies: - "@babel/core": ^7.0.0-0 - checksum: cdd7b8136cc4db3f47714d5266f9e7b592a2ac5a94a5878787ce08890e97c8ab1ca8e94b27bfeba7b0f2b1549a026d9fc414ca2196de603df36fb32633bbdc19 - languageName: node - linkType: hard - -"@babel/plugin-proposal-nullish-coalescing-operator@npm:^7.18.6": - version: 7.18.6 - resolution: "@babel/plugin-proposal-nullish-coalescing-operator@npm:7.18.6" - dependencies: - "@babel/helper-plugin-utils": ^7.18.6 - "@babel/plugin-syntax-nullish-coalescing-operator": ^7.8.3 - peerDependencies: - "@babel/core": ^7.0.0-0 - checksum: 949c9ddcdecdaec766ee610ef98f965f928ccc0361dd87cf9f88cf4896a6ccd62fce063d4494778e50da99dea63d270a1be574a62d6ab81cbe9d85884bf55a7d - languageName: node - linkType: hard - -"@babel/plugin-proposal-numeric-separator@npm:^7.18.6": - version: 7.18.6 - resolution: "@babel/plugin-proposal-numeric-separator@npm:7.18.6" - dependencies: - "@babel/helper-plugin-utils": ^7.18.6 - "@babel/plugin-syntax-numeric-separator": ^7.10.4 - peerDependencies: - "@babel/core": ^7.0.0-0 - checksum: f370ea584c55bf4040e1f78c80b4eeb1ce2e6aaa74f87d1a48266493c33931d0b6222d8cee3a082383d6bb648ab8d6b7147a06f974d3296ef3bc39c7851683ec - languageName: node - linkType: hard - -"@babel/plugin-proposal-object-rest-spread@npm:^7.20.7": - version: 7.20.7 - resolution: "@babel/plugin-proposal-object-rest-spread@npm:7.20.7" - dependencies: - "@babel/compat-data": ^7.20.5 - "@babel/helper-compilation-targets": ^7.20.7 - "@babel/helper-plugin-utils": ^7.20.2 - "@babel/plugin-syntax-object-rest-spread": ^7.8.3 - "@babel/plugin-transform-parameters": ^7.20.7 - peerDependencies: - "@babel/core": ^7.0.0-0 - checksum: 1329db17009964bc644484c660eab717cb3ca63ac0ab0f67c651a028d1bc2ead51dc4064caea283e46994f1b7221670a35cbc0b4beb6273f55e915494b5aa0b2 - languageName: node - linkType: hard - -"@babel/plugin-proposal-optional-catch-binding@npm:^7.18.6": - version: 7.18.6 - resolution: "@babel/plugin-proposal-optional-catch-binding@npm:7.18.6" - dependencies: - "@babel/helper-plugin-utils": ^7.18.6 - "@babel/plugin-syntax-optional-catch-binding": ^7.8.3 - peerDependencies: - "@babel/core": ^7.0.0-0 - checksum: 7b5b39fb5d8d6d14faad6cb68ece5eeb2fd550fb66b5af7d7582402f974f5bc3684641f7c192a5a57e0f59acfae4aada6786be1eba030881ddc590666eff4d1e - languageName: node - linkType: hard - -"@babel/plugin-proposal-optional-chaining@npm:^7.20.7, @babel/plugin-proposal-optional-chaining@npm:^7.21.0": - version: 7.21.0 - resolution: "@babel/plugin-proposal-optional-chaining@npm:7.21.0" - dependencies: - "@babel/helper-plugin-utils": ^7.20.2 - "@babel/helper-skip-transparent-expression-wrappers": ^7.20.0 - "@babel/plugin-syntax-optional-chaining": ^7.8.3 - peerDependencies: - "@babel/core": ^7.0.0-0 - checksum: 11c5449e01b18bb8881e8e005a577fa7be2fe5688e2382c8822d51f8f7005342a301a46af7b273b1f5645f9a7b894c428eee8526342038a275ef6ba4c8d8d746 - languageName: node - linkType: hard - -"@babel/plugin-proposal-private-methods@npm:^7.18.6": - version: 7.18.6 - resolution: "@babel/plugin-proposal-private-methods@npm:7.18.6" - dependencies: - "@babel/helper-create-class-features-plugin": ^7.18.6 - "@babel/helper-plugin-utils": ^7.18.6 - peerDependencies: - "@babel/core": ^7.0.0-0 - checksum: 22d8502ee96bca99ad2c8393e8493e2b8d4507576dd054490fd8201a36824373440106f5b098b6d821b026c7e72b0424ff4aeca69ed5f42e48f029d3a156d5ad - languageName: node - linkType: hard - -"@babel/plugin-proposal-private-property-in-object@npm:^7.21.0": - version: 7.21.0 - resolution: "@babel/plugin-proposal-private-property-in-object@npm:7.21.0" - dependencies: - "@babel/helper-annotate-as-pure": ^7.18.6 - "@babel/helper-create-class-features-plugin": ^7.21.0 - "@babel/helper-plugin-utils": ^7.20.2 - "@babel/plugin-syntax-private-property-in-object": ^7.14.5 - peerDependencies: - "@babel/core": ^7.0.0-0 - checksum: add881a6a836635c41d2710551fdf777e2c07c0b691bf2baacc5d658dd64107479df1038680d6e67c468bfc6f36fb8920025d6bac2a1df0a81b867537d40ae78 - languageName: node - linkType: hard - -"@babel/plugin-proposal-unicode-property-regex@npm:^7.18.6, @babel/plugin-proposal-unicode-property-regex@npm:^7.4.4": +"@babel/plugin-proposal-unicode-property-regex@npm:^7.4.4": version: 7.18.6 resolution: "@babel/plugin-proposal-unicode-property-regex@npm:7.18.6" dependencies: @@ -648,14 +448,25 @@ __metadata: languageName: node linkType: hard -"@babel/plugin-syntax-import-assertions@npm:^7.20.0": - version: 7.20.0 - resolution: "@babel/plugin-syntax-import-assertions@npm:7.20.0" +"@babel/plugin-syntax-import-assertions@npm:^7.22.5": + version: 7.22.5 + resolution: "@babel/plugin-syntax-import-assertions@npm:7.22.5" dependencies: - "@babel/helper-plugin-utils": ^7.19.0 + "@babel/helper-plugin-utils": ^7.22.5 peerDependencies: "@babel/core": ^7.0.0-0 - checksum: 6a86220e0aae40164cd3ffaf80e7c076a1be02a8f3480455dddbae05fda8140f429290027604df7a11b3f3f124866e8a6d69dbfa1dda61ee7377b920ad144d5b + checksum: 2b8b5572db04a7bef1e6cd20debf447e4eef7cb012616f5eceb8fa3e23ce469b8f76ee74fd6d1e158ba17a8f58b0aec579d092fb67c5a30e83ccfbc5754916c1 + languageName: node + linkType: hard + +"@babel/plugin-syntax-import-attributes@npm:^7.22.5": + version: 7.22.5 + resolution: "@babel/plugin-syntax-import-attributes@npm:7.22.5" + dependencies: + "@babel/helper-plugin-utils": ^7.22.5 + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: 197b3c5ea2a9649347f033342cb222ab47f4645633695205c0250c6bf2af29e643753b8bb24a2db39948bef08e7c540babfd365591eb57fc110cb30b425ffc47 languageName: node linkType: hard @@ -681,14 +492,14 @@ __metadata: languageName: node linkType: hard -"@babel/plugin-syntax-jsx@npm:^7.21.4, @babel/plugin-syntax-jsx@npm:^7.7.2": - version: 7.21.4 - resolution: "@babel/plugin-syntax-jsx@npm:7.21.4" +"@babel/plugin-syntax-jsx@npm:^7.22.5, @babel/plugin-syntax-jsx@npm:^7.7.2": + version: 7.22.5 + resolution: "@babel/plugin-syntax-jsx@npm:7.22.5" dependencies: - "@babel/helper-plugin-utils": ^7.20.2 + "@babel/helper-plugin-utils": ^7.22.5 peerDependencies: "@babel/core": ^7.0.0-0 - checksum: bb7309402a1d4e155f32aa0cf216e1fa8324d6c4cfd248b03280028a015a10e46b6efd6565f515f8913918a3602b39255999c06046f7d4b8a5106be2165d724a + checksum: 8829d30c2617ab31393d99cec2978e41f014f4ac6f01a1cecf4c4dd8320c3ec12fdc3ce121126b2d8d32f6887e99ca1a0bad53dedb1e6ad165640b92b24980ce languageName: node linkType: hard @@ -780,441 +591,641 @@ __metadata: languageName: node linkType: hard -"@babel/plugin-syntax-typescript@npm:^7.20.0, @babel/plugin-syntax-typescript@npm:^7.7.2": - version: 7.21.4 - resolution: "@babel/plugin-syntax-typescript@npm:7.21.4" +"@babel/plugin-syntax-typescript@npm:^7.22.5, @babel/plugin-syntax-typescript@npm:^7.7.2": + version: 7.22.5 + resolution: "@babel/plugin-syntax-typescript@npm:7.22.5" dependencies: - "@babel/helper-plugin-utils": ^7.20.2 + "@babel/helper-plugin-utils": ^7.22.5 peerDependencies: "@babel/core": ^7.0.0-0 - checksum: a59ce2477b7ae8c8945dc37dda292fef9ce46a6507b3d76b03ce7f3a6c9451a6567438b20a78ebcb3955d04095fd1ccd767075a863f79fcc30aa34dcfa441fe0 + checksum: 8ab7718fbb026d64da93681a57797d60326097fd7cb930380c8bffd9eb101689e90142c760a14b51e8e69c88a73ba3da956cb4520a3b0c65743aee5c71ef360a + languageName: node + linkType: hard + +"@babel/plugin-syntax-unicode-sets-regex@npm:^7.18.6": + version: 7.18.6 + resolution: "@babel/plugin-syntax-unicode-sets-regex@npm:7.18.6" + dependencies: + "@babel/helper-create-regexp-features-plugin": ^7.18.6 + "@babel/helper-plugin-utils": ^7.18.6 + peerDependencies: + "@babel/core": ^7.0.0 + checksum: a651d700fe63ff0ddfd7186f4ebc24447ca734f114433139e3c027bc94a900d013cf1ef2e2db8430425ba542e39ae160c3b05f06b59fd4656273a3df97679e9c languageName: node linkType: hard -"@babel/plugin-transform-arrow-functions@npm:^7.21.5": - version: 7.21.5 - resolution: "@babel/plugin-transform-arrow-functions@npm:7.21.5" +"@babel/plugin-transform-arrow-functions@npm:^7.22.5": + version: 7.22.5 + resolution: "@babel/plugin-transform-arrow-functions@npm:7.22.5" dependencies: - "@babel/helper-plugin-utils": ^7.21.5 + "@babel/helper-plugin-utils": ^7.22.5 peerDependencies: "@babel/core": ^7.0.0-0 - checksum: c7c281cdf37c33a584102d9fd1793e85c96d4d320cdfb7c43f1ce581323d057f13b53203994fcc7ee1f8dc1ff013498f258893aa855a06c6f830fcc4c33d6e44 + checksum: 35abb6c57062802c7ce8bd96b2ef2883e3124370c688bbd67609f7d2453802fb73944df8808f893b6c67de978eb2bcf87bbfe325e46d6f39b5fcb09ece11d01a languageName: node linkType: hard -"@babel/plugin-transform-async-to-generator@npm:^7.20.7": - version: 7.20.7 - resolution: "@babel/plugin-transform-async-to-generator@npm:7.20.7" +"@babel/plugin-transform-async-generator-functions@npm:^7.22.7": + version: 7.22.7 + resolution: "@babel/plugin-transform-async-generator-functions@npm:7.22.7" dependencies: - "@babel/helper-module-imports": ^7.18.6 - "@babel/helper-plugin-utils": ^7.20.2 - "@babel/helper-remap-async-to-generator": ^7.18.9 + "@babel/helper-environment-visitor": ^7.22.5 + "@babel/helper-plugin-utils": ^7.22.5 + "@babel/helper-remap-async-to-generator": ^7.22.5 + "@babel/plugin-syntax-async-generators": ^7.8.4 peerDependencies: "@babel/core": ^7.0.0-0 - checksum: fe9ee8a5471b4317c1b9ea92410ace8126b52a600d7cfbfe1920dcac6fb0fad647d2e08beb4fd03c630eb54430e6c72db11e283e3eddc49615c68abd39430904 + checksum: 57cd2cce3fb696dadf00e88f168683df69e900b92dadeae07429243c43bc21d5ccdc0c2db61cf5c37bd0fbd893fc455466bef6babe4aa5b79d9cb8ba89f40ae7 languageName: node linkType: hard -"@babel/plugin-transform-block-scoped-functions@npm:^7.18.6": - version: 7.18.6 - resolution: "@babel/plugin-transform-block-scoped-functions@npm:7.18.6" +"@babel/plugin-transform-async-to-generator@npm:^7.22.5": + version: 7.22.5 + resolution: "@babel/plugin-transform-async-to-generator@npm:7.22.5" dependencies: - "@babel/helper-plugin-utils": ^7.18.6 + "@babel/helper-module-imports": ^7.22.5 + "@babel/helper-plugin-utils": ^7.22.5 + "@babel/helper-remap-async-to-generator": ^7.22.5 + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: b95f23f99dcb379a9f0a1c2a3bbea3f8dc0e1b16dc1ac8b484fe378370169290a7a63d520959a9ba1232837cf74a80e23f6facbe14fd42a3cda6d3c2d7168e62 + languageName: node + linkType: hard + +"@babel/plugin-transform-block-scoped-functions@npm:^7.22.5": + version: 7.22.5 + resolution: "@babel/plugin-transform-block-scoped-functions@npm:7.22.5" + dependencies: + "@babel/helper-plugin-utils": ^7.22.5 + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: 416b1341858e8ca4e524dee66044735956ced5f478b2c3b9bc11ec2285b0c25d7dbb96d79887169eb938084c95d0a89338c8b2fe70d473bd9dc92e5d9db1732c + languageName: node + linkType: hard + +"@babel/plugin-transform-block-scoping@npm:^7.22.5": + version: 7.22.5 + resolution: "@babel/plugin-transform-block-scoping@npm:7.22.5" + dependencies: + "@babel/helper-plugin-utils": ^7.22.5 peerDependencies: "@babel/core": ^7.0.0-0 - checksum: 0a0df61f94601e3666bf39f2cc26f5f7b22a94450fb93081edbed967bd752ce3f81d1227fefd3799f5ee2722171b5e28db61379234d1bb85b6ec689589f99d7e + checksum: 26987002cfe6e24544e60fa35f07052b6557f590c1a1cc5cf35d6dc341d7fea163c1222a2d70d5d2692f0b9860d942fd3ba979848b2995d4debffa387b9b19ae languageName: node linkType: hard -"@babel/plugin-transform-block-scoping@npm:^7.21.0": - version: 7.21.0 - resolution: "@babel/plugin-transform-block-scoping@npm:7.21.0" +"@babel/plugin-transform-class-properties@npm:^7.22.5": + version: 7.22.5 + resolution: "@babel/plugin-transform-class-properties@npm:7.22.5" dependencies: - "@babel/helper-plugin-utils": ^7.20.2 + "@babel/helper-create-class-features-plugin": ^7.22.5 + "@babel/helper-plugin-utils": ^7.22.5 peerDependencies: "@babel/core": ^7.0.0-0 - checksum: 15aacaadbecf96b53a750db1be4990b0d89c7f5bc3e1794b63b49fb219638c1fd25d452d15566d7e5ddf5b5f4e1a0a0055c35c1c7aee323c7b114bf49f66f4b0 + checksum: b830152dfc2ff2f647f0abe76e6251babdfbef54d18c4b2c73a6bf76b1a00050a5d998dac80dc901a48514e95604324943a9dd39317073fe0928b559e0e0c579 + languageName: node + linkType: hard + +"@babel/plugin-transform-class-static-block@npm:^7.22.5": + version: 7.22.5 + resolution: "@babel/plugin-transform-class-static-block@npm:7.22.5" + dependencies: + "@babel/helper-create-class-features-plugin": ^7.22.5 + "@babel/helper-plugin-utils": ^7.22.5 + "@babel/plugin-syntax-class-static-block": ^7.14.5 + peerDependencies: + "@babel/core": ^7.12.0 + checksum: bc48b92dbaf625a14f2bf62382384eef01e0515802426841636ae9146e27395d068c7a8a45e9e15699491b0a01d990f38f179cbc9dc89274a393f85648772f12 languageName: node linkType: hard -"@babel/plugin-transform-classes@npm:^7.21.0": - version: 7.21.0 - resolution: "@babel/plugin-transform-classes@npm:7.21.0" +"@babel/plugin-transform-classes@npm:^7.22.6": + version: 7.22.6 + resolution: "@babel/plugin-transform-classes@npm:7.22.6" dependencies: - "@babel/helper-annotate-as-pure": ^7.18.6 - "@babel/helper-compilation-targets": ^7.20.7 - "@babel/helper-environment-visitor": ^7.18.9 - "@babel/helper-function-name": ^7.21.0 - "@babel/helper-optimise-call-expression": ^7.18.6 - "@babel/helper-plugin-utils": ^7.20.2 - "@babel/helper-replace-supers": ^7.20.7 - "@babel/helper-split-export-declaration": ^7.18.6 + "@babel/helper-annotate-as-pure": ^7.22.5 + "@babel/helper-compilation-targets": ^7.22.6 + "@babel/helper-environment-visitor": ^7.22.5 + "@babel/helper-function-name": ^7.22.5 + "@babel/helper-optimise-call-expression": ^7.22.5 + "@babel/helper-plugin-utils": ^7.22.5 + "@babel/helper-replace-supers": ^7.22.5 + "@babel/helper-split-export-declaration": ^7.22.6 globals: ^11.1.0 peerDependencies: "@babel/core": ^7.0.0-0 - checksum: 088ae152074bd0e90f64659169255bfe50393e637ec8765cb2a518848b11b0299e66b91003728fd0a41563a6fdc6b8d548ece698a314fd5447f5489c22e466b7 + checksum: 8380e855c01033dbc7460d9acfbc1fc37c880350fa798c2de8c594ef818ade0e4c96173ec72f05f2a4549d8d37135e18cb62548352d51557b45a0fb4388d2f3f languageName: node linkType: hard -"@babel/plugin-transform-computed-properties@npm:^7.21.5": - version: 7.21.5 - resolution: "@babel/plugin-transform-computed-properties@npm:7.21.5" +"@babel/plugin-transform-computed-properties@npm:^7.22.5": + version: 7.22.5 + resolution: "@babel/plugin-transform-computed-properties@npm:7.22.5" dependencies: - "@babel/helper-plugin-utils": ^7.21.5 - "@babel/template": ^7.20.7 + "@babel/helper-plugin-utils": ^7.22.5 + "@babel/template": ^7.22.5 peerDependencies: "@babel/core": ^7.0.0-0 - checksum: e819780ab30fc40d7802ffb75b397eff63ca4942a1873058f81c80f660189b78e158fa03fd3270775f0477c4c33cee3d8d40270e64404bbf24aa6cdccb197e7b + checksum: c2a77a0f94ec71efbc569109ec14ea2aa925b333289272ced8b33c6108bdbb02caf01830ffc7e49486b62dec51911924d13f3a76f1149f40daace1898009e131 languageName: node linkType: hard -"@babel/plugin-transform-destructuring@npm:^7.21.3": - version: 7.21.3 - resolution: "@babel/plugin-transform-destructuring@npm:7.21.3" +"@babel/plugin-transform-destructuring@npm:^7.22.5": + version: 7.22.5 + resolution: "@babel/plugin-transform-destructuring@npm:7.22.5" dependencies: - "@babel/helper-plugin-utils": ^7.20.2 + "@babel/helper-plugin-utils": ^7.22.5 peerDependencies: "@babel/core": ^7.0.0-0 - checksum: 43ebbe0bfa20287e34427be7c2200ce096c20913775ea75268fb47fe0e55f9510800587e6052c42fe6dffa0daaad95dd465c3e312fd1ef9785648384c45417ac + checksum: 76f6ea2aee1fcfa1c3791eb7a5b89703c6472650b993e8666fff0f1d6e9d737a84134edf89f63c92297f3e75064c1263219463b02dd9bc7434b6e5b9935e3f20 languageName: node linkType: hard -"@babel/plugin-transform-dotall-regex@npm:^7.18.6, @babel/plugin-transform-dotall-regex@npm:^7.4.4": - version: 7.18.6 - resolution: "@babel/plugin-transform-dotall-regex@npm:7.18.6" +"@babel/plugin-transform-dotall-regex@npm:^7.22.5, @babel/plugin-transform-dotall-regex@npm:^7.4.4": + version: 7.22.5 + resolution: "@babel/plugin-transform-dotall-regex@npm:7.22.5" dependencies: - "@babel/helper-create-regexp-features-plugin": ^7.18.6 - "@babel/helper-plugin-utils": ^7.18.6 + "@babel/helper-create-regexp-features-plugin": ^7.22.5 + "@babel/helper-plugin-utils": ^7.22.5 peerDependencies: "@babel/core": ^7.0.0-0 - checksum: cbe5d7063eb8f8cca24cd4827bc97f5641166509e58781a5f8aa47fb3d2d786ce4506a30fca2e01f61f18792783a5cb5d96bf5434c3dd1ad0de8c9cc625a53da + checksum: 409b658d11e3082c8f69e9cdef2d96e4d6d11256f005772425fb230cc48fd05945edbfbcb709dab293a1a2f01f9c8a5bb7b4131e632b23264039d9f95864b453 languageName: node linkType: hard -"@babel/plugin-transform-duplicate-keys@npm:^7.18.9": - version: 7.18.9 - resolution: "@babel/plugin-transform-duplicate-keys@npm:7.18.9" +"@babel/plugin-transform-duplicate-keys@npm:^7.22.5": + version: 7.22.5 + resolution: "@babel/plugin-transform-duplicate-keys@npm:7.22.5" dependencies: - "@babel/helper-plugin-utils": ^7.18.9 + "@babel/helper-plugin-utils": ^7.22.5 peerDependencies: "@babel/core": ^7.0.0-0 - checksum: 220bf4a9fec5c4d4a7b1de38810350260e8ea08481bf78332a464a21256a95f0df8cd56025f346238f09b04f8e86d4158fafc9f4af57abaef31637e3b58bd4fe + checksum: bb1280fbabaab6fab2ede585df34900712698210a3bd413f4df5bae6d8c24be36b496c92722ae676a7a67d060a4624f4d6c23b923485f906bfba8773c69f55b4 languageName: node linkType: hard -"@babel/plugin-transform-exponentiation-operator@npm:^7.18.6": - version: 7.18.6 - resolution: "@babel/plugin-transform-exponentiation-operator@npm:7.18.6" +"@babel/plugin-transform-dynamic-import@npm:^7.22.5": + version: 7.22.5 + resolution: "@babel/plugin-transform-dynamic-import@npm:7.22.5" dependencies: - "@babel/helper-builder-binary-assignment-operator-visitor": ^7.18.6 - "@babel/helper-plugin-utils": ^7.18.6 + "@babel/helper-plugin-utils": ^7.22.5 + "@babel/plugin-syntax-dynamic-import": ^7.8.3 peerDependencies: "@babel/core": ^7.0.0-0 - checksum: 7f70222f6829c82a36005508d34ddbe6fd0974ae190683a8670dd6ff08669aaf51fef2209d7403f9bd543cb2d12b18458016c99a6ed0332ccedb3ea127b01229 + checksum: 186a6d59f36eb3c5824739fc9c22ed0f4ca68e001662aa3a302634346a8b785cb9579b23b0c158f4570604d697d19598ca09b58c60a7fa2894da1163c4eb1907 languageName: node linkType: hard -"@babel/plugin-transform-for-of@npm:^7.21.5": - version: 7.21.5 - resolution: "@babel/plugin-transform-for-of@npm:7.21.5" +"@babel/plugin-transform-exponentiation-operator@npm:^7.22.5": + version: 7.22.5 + resolution: "@babel/plugin-transform-exponentiation-operator@npm:7.22.5" dependencies: - "@babel/helper-plugin-utils": ^7.21.5 + "@babel/helper-builder-binary-assignment-operator-visitor": ^7.22.5 + "@babel/helper-plugin-utils": ^7.22.5 peerDependencies: "@babel/core": ^7.0.0-0 - checksum: b6666b24e8ca1ffbf7452a0042149724e295965aad55070dc9ee992451d69d855fc9db832c1c5fb4d3dc532f4a18a2974d5f8524f5c2250dda888d05f6f3cadb + checksum: f2d660c1b1d51ad5fec1cd5ad426a52187204068c4158f8c4aa977b31535c61b66898d532603eef21c15756827be8277f724c869b888d560f26d7fe848bb5eae languageName: node linkType: hard -"@babel/plugin-transform-function-name@npm:^7.18.9": - version: 7.18.9 - resolution: "@babel/plugin-transform-function-name@npm:7.18.9" +"@babel/plugin-transform-export-namespace-from@npm:^7.22.5": + version: 7.22.5 + resolution: "@babel/plugin-transform-export-namespace-from@npm:7.22.5" dependencies: - "@babel/helper-compilation-targets": ^7.18.9 - "@babel/helper-function-name": ^7.18.9 - "@babel/helper-plugin-utils": ^7.18.9 + "@babel/helper-plugin-utils": ^7.22.5 + "@babel/plugin-syntax-export-namespace-from": ^7.8.3 peerDependencies: "@babel/core": ^7.0.0-0 - checksum: 62dd9c6cdc9714704efe15545e782ee52d74dc73916bf954b4d3bee088fb0ec9e3c8f52e751252433656c09f744b27b757fc06ed99bcde28e8a21600a1d8e597 + checksum: 3d197b788758044983c96b9c49bed4b456055f35a388521a405968db0f6e2ffb6fd59110e3931f4dcc5e126ae9e5e00e154a0afb47a7ea359d8d0dea79f480d7 languageName: node linkType: hard -"@babel/plugin-transform-literals@npm:^7.18.9": - version: 7.18.9 - resolution: "@babel/plugin-transform-literals@npm:7.18.9" +"@babel/plugin-transform-for-of@npm:^7.22.5": + version: 7.22.5 + resolution: "@babel/plugin-transform-for-of@npm:7.22.5" dependencies: - "@babel/helper-plugin-utils": ^7.18.9 + "@babel/helper-plugin-utils": ^7.22.5 peerDependencies: "@babel/core": ^7.0.0-0 - checksum: 3458dd2f1a47ac51d9d607aa18f3d321cbfa8560a985199185bed5a906bb0c61ba85575d386460bac9aed43fdd98940041fae5a67dff286f6f967707cff489f8 + checksum: d7b8d4db010bce7273674caa95c4e6abd909362866ce297e86a2ecaa9ae636e05d525415811db9b3c942155df7f3651d19b91dd6c41f142f7308a97c7cb06023 languageName: node linkType: hard -"@babel/plugin-transform-member-expression-literals@npm:^7.18.6": - version: 7.18.6 - resolution: "@babel/plugin-transform-member-expression-literals@npm:7.18.6" +"@babel/plugin-transform-function-name@npm:^7.22.5": + version: 7.22.5 + resolution: "@babel/plugin-transform-function-name@npm:7.22.5" dependencies: - "@babel/helper-plugin-utils": ^7.18.6 + "@babel/helper-compilation-targets": ^7.22.5 + "@babel/helper-function-name": ^7.22.5 + "@babel/helper-plugin-utils": ^7.22.5 + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: cff3b876357999cb8ae30e439c3ec6b0491a53b0aa6f722920a4675a6dd5b53af97a833051df4b34791fe5b3dd326ccf769d5c8e45b322aa50ee11a660b17845 + languageName: node + linkType: hard + +"@babel/plugin-transform-json-strings@npm:^7.22.5": + version: 7.22.5 + resolution: "@babel/plugin-transform-json-strings@npm:7.22.5" + dependencies: + "@babel/helper-plugin-utils": ^7.22.5 + "@babel/plugin-syntax-json-strings": ^7.8.3 peerDependencies: "@babel/core": ^7.0.0-0 - checksum: 35a3d04f6693bc6b298c05453d85ee6e41cc806538acb6928427e0e97ae06059f97d2f07d21495fcf5f70d3c13a242e2ecbd09d5c1fcb1b1a73ff528dcb0b695 + checksum: 4e00b902487a670b6c8948f33f9108133fd745cf9d1478aca515fb460b9b2f12e137988ebc1663630fb82070a870aed8b0c1aa4d007a841c18004619798f255c languageName: node linkType: hard -"@babel/plugin-transform-modules-amd@npm:^7.20.11": - version: 7.20.11 - resolution: "@babel/plugin-transform-modules-amd@npm:7.20.11" +"@babel/plugin-transform-literals@npm:^7.22.5": + version: 7.22.5 + resolution: "@babel/plugin-transform-literals@npm:7.22.5" dependencies: - "@babel/helper-module-transforms": ^7.20.11 - "@babel/helper-plugin-utils": ^7.20.2 + "@babel/helper-plugin-utils": ^7.22.5 peerDependencies: "@babel/core": ^7.0.0-0 - checksum: 23665c1c20c8f11c89382b588fb9651c0756d130737a7625baeaadbd3b973bc5bfba1303bedffa8fb99db1e6d848afb01016e1df2b69b18303e946890c790001 + checksum: ec37cc2ffb32667af935ab32fe28f00920ec8a1eb999aa6dc6602f2bebd8ba205a558aeedcdccdebf334381d5c57106c61f52332045730393e73410892a9735b languageName: node linkType: hard -"@babel/plugin-transform-modules-commonjs@npm:^7.21.5": - version: 7.21.5 - resolution: "@babel/plugin-transform-modules-commonjs@npm:7.21.5" +"@babel/plugin-transform-logical-assignment-operators@npm:^7.22.5": + version: 7.22.5 + resolution: "@babel/plugin-transform-logical-assignment-operators@npm:7.22.5" dependencies: - "@babel/helper-module-transforms": ^7.21.5 - "@babel/helper-plugin-utils": ^7.21.5 - "@babel/helper-simple-access": ^7.21.5 + "@babel/helper-plugin-utils": ^7.22.5 + "@babel/plugin-syntax-logical-assignment-operators": ^7.10.4 peerDependencies: "@babel/core": ^7.0.0-0 - checksum: d9ff7a21baaa60c08a0c86c5e468bb4b2bd85caf51ba78712d8f45e9afa2498d50d6cdf349696e08aa820cafed65f19b70e5938613db9ebb095f7aba1127f282 + checksum: 18748e953c08f64885f18c224eac58df10a13eac4d845d16b5d9b6276907da7ca2530dfebe6ed41cdc5f8a75d9db3e36d8eb54ddce7cd0364af1cab09b435302 languageName: node linkType: hard -"@babel/plugin-transform-modules-systemjs@npm:^7.20.11": - version: 7.20.11 - resolution: "@babel/plugin-transform-modules-systemjs@npm:7.20.11" +"@babel/plugin-transform-member-expression-literals@npm:^7.22.5": + version: 7.22.5 + resolution: "@babel/plugin-transform-member-expression-literals@npm:7.22.5" dependencies: - "@babel/helper-hoist-variables": ^7.18.6 - "@babel/helper-module-transforms": ^7.20.11 - "@babel/helper-plugin-utils": ^7.20.2 - "@babel/helper-validator-identifier": ^7.19.1 + "@babel/helper-plugin-utils": ^7.22.5 peerDependencies: "@babel/core": ^7.0.0-0 - checksum: 4546c47587f88156d66c7eb7808e903cf4bb3f6ba6ac9bc8e3af2e29e92eb9f0b3f44d52043bfd24eb25fa7827fd7b6c8bfeac0cac7584e019b87e1ecbd0e673 + checksum: ec4b0e07915ddd4fda0142fd104ee61015c208608a84cfa13643a95d18760b1dc1ceb6c6e0548898b8c49e5959a994e46367260176dbabc4467f729b21868504 languageName: node linkType: hard -"@babel/plugin-transform-modules-umd@npm:^7.18.6": - version: 7.18.6 - resolution: "@babel/plugin-transform-modules-umd@npm:7.18.6" +"@babel/plugin-transform-modules-amd@npm:^7.22.5": + version: 7.22.5 + resolution: "@babel/plugin-transform-modules-amd@npm:7.22.5" dependencies: - "@babel/helper-module-transforms": ^7.18.6 - "@babel/helper-plugin-utils": ^7.18.6 + "@babel/helper-module-transforms": ^7.22.5 + "@babel/helper-plugin-utils": ^7.22.5 peerDependencies: "@babel/core": ^7.0.0-0 - checksum: c3b6796c6f4579f1ba5ab0cdcc73910c1e9c8e1e773c507c8bb4da33072b3ae5df73c6d68f9126dab6e99c24ea8571e1563f8710d7c421fac1cde1e434c20153 + checksum: 7da4c4ebbbcf7d182abb59b2046b22d86eee340caf8a22a39ef6a727da2d8acfec1f714fcdcd5054110b280e4934f735e80a6848d192b6834c5d4459a014f04d languageName: node linkType: hard -"@babel/plugin-transform-named-capturing-groups-regex@npm:^7.20.5": - version: 7.20.5 - resolution: "@babel/plugin-transform-named-capturing-groups-regex@npm:7.20.5" +"@babel/plugin-transform-modules-commonjs@npm:^7.22.5": + version: 7.22.5 + resolution: "@babel/plugin-transform-modules-commonjs@npm:7.22.5" dependencies: - "@babel/helper-create-regexp-features-plugin": ^7.20.5 - "@babel/helper-plugin-utils": ^7.20.2 + "@babel/helper-module-transforms": ^7.22.5 + "@babel/helper-plugin-utils": ^7.22.5 + "@babel/helper-simple-access": ^7.22.5 + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: 2067aca8f6454d54ffcce69b02c457cfa61428e11372f6a1d99ff4fcfbb55c396ed2ca6ca886bf06c852e38c1a205b8095921b2364fd0243f3e66bc1dda61caa + languageName: node + linkType: hard + +"@babel/plugin-transform-modules-systemjs@npm:^7.22.5": + version: 7.22.5 + resolution: "@babel/plugin-transform-modules-systemjs@npm:7.22.5" + dependencies: + "@babel/helper-hoist-variables": ^7.22.5 + "@babel/helper-module-transforms": ^7.22.5 + "@babel/helper-plugin-utils": ^7.22.5 + "@babel/helper-validator-identifier": ^7.22.5 + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: 04f4178589543396b3c24330a67a59c5e69af5e96119c9adda730c0f20122deaff54671ebbc72ad2df6495a5db8a758bd96942de95fba7ad427de9c80b1b38c8 + languageName: node + linkType: hard + +"@babel/plugin-transform-modules-umd@npm:^7.22.5": + version: 7.22.5 + resolution: "@babel/plugin-transform-modules-umd@npm:7.22.5" + dependencies: + "@babel/helper-module-transforms": ^7.22.5 + "@babel/helper-plugin-utils": ^7.22.5 + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: 46622834c54c551b231963b867adbc80854881b3e516ff29984a8da989bd81665bd70e8cba6710345248e97166689310f544aee1a5773e262845a8f1b3e5b8b4 + languageName: node + linkType: hard + +"@babel/plugin-transform-named-capturing-groups-regex@npm:^7.22.5": + version: 7.22.5 + resolution: "@babel/plugin-transform-named-capturing-groups-regex@npm:7.22.5" + dependencies: + "@babel/helper-create-regexp-features-plugin": ^7.22.5 + "@babel/helper-plugin-utils": ^7.22.5 peerDependencies: "@babel/core": ^7.0.0 - checksum: 528c95fb1087e212f17e1c6456df041b28a83c772b9c93d2e407c9d03b72182b0d9d126770c1d6e0b23aab052599ceaf25ed6a2c0627f4249be34a83f6fae853 + checksum: 3ee564ddee620c035b928fdc942c5d17e9c4b98329b76f9cefac65c111135d925eb94ed324064cd7556d4f5123beec79abea1d4b97d1c8a2a5c748887a2eb623 languageName: node linkType: hard -"@babel/plugin-transform-new-target@npm:^7.18.6": - version: 7.18.6 - resolution: "@babel/plugin-transform-new-target@npm:7.18.6" +"@babel/plugin-transform-new-target@npm:^7.22.5": + version: 7.22.5 + resolution: "@babel/plugin-transform-new-target@npm:7.22.5" dependencies: - "@babel/helper-plugin-utils": ^7.18.6 + "@babel/helper-plugin-utils": ^7.22.5 peerDependencies: "@babel/core": ^7.0.0-0 - checksum: bd780e14f46af55d0ae8503b3cb81ca86dcc73ed782f177e74f498fff934754f9e9911df1f8f3bd123777eed7c1c1af4d66abab87c8daae5403e7719a6b845d1 + checksum: 6b72112773487a881a1d6ffa680afde08bad699252020e86122180ee7a88854d5da3f15d9bca3331cf2e025df045604494a8208a2e63b486266b07c14e2ffbf3 languageName: node linkType: hard -"@babel/plugin-transform-object-super@npm:^7.18.6": - version: 7.18.6 - resolution: "@babel/plugin-transform-object-super@npm:7.18.6" +"@babel/plugin-transform-nullish-coalescing-operator@npm:^7.22.5": + version: 7.22.5 + resolution: "@babel/plugin-transform-nullish-coalescing-operator@npm:7.22.5" dependencies: - "@babel/helper-plugin-utils": ^7.18.6 - "@babel/helper-replace-supers": ^7.18.6 + "@babel/helper-plugin-utils": ^7.22.5 + "@babel/plugin-syntax-nullish-coalescing-operator": ^7.8.3 peerDependencies: "@babel/core": ^7.0.0-0 - checksum: 0fcb04e15deea96ae047c21cb403607d49f06b23b4589055993365ebd7a7d7541334f06bf9642e90075e66efce6ebaf1eb0ef066fbbab802d21d714f1aac3aef + checksum: e6a059169d257fc61322d0708edae423072449b7c33de396261e68dee582aec5396789a1c22bce84e5bd88a169623c2e750b513fc222930979e6accd52a44bf2 languageName: node linkType: hard -"@babel/plugin-transform-parameters@npm:^7.20.7, @babel/plugin-transform-parameters@npm:^7.21.3": - version: 7.21.3 - resolution: "@babel/plugin-transform-parameters@npm:7.21.3" +"@babel/plugin-transform-numeric-separator@npm:^7.22.5": + version: 7.22.5 + resolution: "@babel/plugin-transform-numeric-separator@npm:7.22.5" dependencies: - "@babel/helper-plugin-utils": ^7.20.2 + "@babel/helper-plugin-utils": ^7.22.5 + "@babel/plugin-syntax-numeric-separator": ^7.10.4 peerDependencies: "@babel/core": ^7.0.0-0 - checksum: c92128d7b1fcf54e2cab186c196bbbf55a9a6de11a83328dc2602649c9dc6d16ef73712beecd776cd49bfdc624b5f56740f4a53568d3deb9505ec666bc869da3 + checksum: 9e7837d4eae04f211ebaa034fe5003d2927b6bf6d5b9dc09f2b1183c01482cdde5a75b8bd5c7ff195c2abc7b923339eb0b2a9d27cb78359d38248a3b2c2367c4 languageName: node linkType: hard -"@babel/plugin-transform-property-literals@npm:^7.18.6": - version: 7.18.6 - resolution: "@babel/plugin-transform-property-literals@npm:7.18.6" +"@babel/plugin-transform-object-rest-spread@npm:^7.22.5": + version: 7.22.5 + resolution: "@babel/plugin-transform-object-rest-spread@npm:7.22.5" dependencies: - "@babel/helper-plugin-utils": ^7.18.6 + "@babel/compat-data": ^7.22.5 + "@babel/helper-compilation-targets": ^7.22.5 + "@babel/helper-plugin-utils": ^7.22.5 + "@babel/plugin-syntax-object-rest-spread": ^7.8.3 + "@babel/plugin-transform-parameters": ^7.22.5 peerDependencies: "@babel/core": ^7.0.0-0 - checksum: 1c16e64de554703f4b547541de2edda6c01346dd3031d4d29e881aa7733785cd26d53611a4ccf5353f4d3e69097bb0111c0a93ace9e683edd94fea28c4484144 + checksum: 3b5e091f0dc67108f2e41ed5a97e15bbe4381a19d9a7eea80b71c7de1d8169fd28784e1e41a3d2ad12709ab212e58fc481282a5bb65d591fae7b443048de3330 languageName: node linkType: hard -"@babel/plugin-transform-regenerator@npm:^7.21.5": - version: 7.21.5 - resolution: "@babel/plugin-transform-regenerator@npm:7.21.5" +"@babel/plugin-transform-object-super@npm:^7.22.5": + version: 7.22.5 + resolution: "@babel/plugin-transform-object-super@npm:7.22.5" dependencies: - "@babel/helper-plugin-utils": ^7.21.5 + "@babel/helper-plugin-utils": ^7.22.5 + "@babel/helper-replace-supers": ^7.22.5 + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: b71887877d74cb64dbccb5c0324fa67e31171e6a5311991f626650e44a4083e5436a1eaa89da78c0474fb095d4ec322d63ee778b202d33aa2e4194e1ed8e62d7 + languageName: node + linkType: hard + +"@babel/plugin-transform-optional-catch-binding@npm:^7.22.5": + version: 7.22.5 + resolution: "@babel/plugin-transform-optional-catch-binding@npm:7.22.5" + dependencies: + "@babel/helper-plugin-utils": ^7.22.5 + "@babel/plugin-syntax-optional-catch-binding": ^7.8.3 + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: b0e8b4233ff06b5c9d285257f49c5bd441f883189b24282e6200f9ebdf5db29aeeebbffae57fbbcd5df9f4387b3e66e5d322aaae5652a78e89685ddbae46bbd1 + languageName: node + linkType: hard + +"@babel/plugin-transform-optional-chaining@npm:^7.22.5, @babel/plugin-transform-optional-chaining@npm:^7.22.6": + version: 7.22.6 + resolution: "@babel/plugin-transform-optional-chaining@npm:7.22.6" + dependencies: + "@babel/helper-plugin-utils": ^7.22.5 + "@babel/helper-skip-transparent-expression-wrappers": ^7.22.5 + "@babel/plugin-syntax-optional-chaining": ^7.8.3 + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: 9713f7920ed04090c149fc5ec024dd1638e8b97aa4ae3753b93072d84103b8de380afb96d6cf03e53b285420db4f705f3ac13149c6fd54f322b61dc19e33c54f + languageName: node + linkType: hard + +"@babel/plugin-transform-parameters@npm:^7.22.5": + version: 7.22.5 + resolution: "@babel/plugin-transform-parameters@npm:7.22.5" + dependencies: + "@babel/helper-plugin-utils": ^7.22.5 + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: b44f89cf97daf23903776ba27c2ab13b439d80d8c8a95be5c476ab65023b1e0c0e94c28d3745f3b60a58edc4e590fa0cd4287a0293e51401ca7d29a2ddb13b8e + languageName: node + linkType: hard + +"@babel/plugin-transform-private-methods@npm:^7.22.5": + version: 7.22.5 + resolution: "@babel/plugin-transform-private-methods@npm:7.22.5" + dependencies: + "@babel/helper-create-class-features-plugin": ^7.22.5 + "@babel/helper-plugin-utils": ^7.22.5 + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: 321479b4fcb6d3b3ef622ab22fd24001e43d46e680e8e41324c033d5810c84646e470f81b44cbcbef5c22e99030784f7cac92f1829974da7a47a60a7139082c3 + languageName: node + linkType: hard + +"@babel/plugin-transform-private-property-in-object@npm:^7.22.5": + version: 7.22.5 + resolution: "@babel/plugin-transform-private-property-in-object@npm:7.22.5" + dependencies: + "@babel/helper-annotate-as-pure": ^7.22.5 + "@babel/helper-create-class-features-plugin": ^7.22.5 + "@babel/helper-plugin-utils": ^7.22.5 + "@babel/plugin-syntax-private-property-in-object": ^7.14.5 + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: 9ac019fb2772f3af6278a7f4b8b14b0663accb3fd123d87142ceb2fbc57fd1afa07c945d1329029b026b9ee122096ef71a3f34f257a9e04cf4245b87298c38b4 + languageName: node + linkType: hard + +"@babel/plugin-transform-property-literals@npm:^7.22.5": + version: 7.22.5 + resolution: "@babel/plugin-transform-property-literals@npm:7.22.5" + dependencies: + "@babel/helper-plugin-utils": ^7.22.5 + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: 796176a3176106f77fcb8cd04eb34a8475ce82d6d03a88db089531b8f0453a2fb8b0c6ec9a52c27948bc0ea478becec449893741fc546dfc3930ab927e3f9f2e + languageName: node + linkType: hard + +"@babel/plugin-transform-regenerator@npm:^7.22.5": + version: 7.22.5 + resolution: "@babel/plugin-transform-regenerator@npm:7.22.5" + dependencies: + "@babel/helper-plugin-utils": ^7.22.5 regenerator-transform: ^0.15.1 peerDependencies: "@babel/core": ^7.0.0-0 - checksum: 5291f6871276f57a6004f16d50ae9ad57f22a6aa2a183b8c84de8126f1066c6c9f9bbeadb282b5207fa9e7b0f57e40a8421d46cb5c60caf7e2848e98224d5639 + checksum: f7c5ca5151321963df777cc02725d10d1ccc3b3b8323da0423aecd9ac6144cbdd2274af5281a5580db2fc2f8b234e318517b5d76b85669118906533a559f2b6a languageName: node linkType: hard -"@babel/plugin-transform-reserved-words@npm:^7.18.6": - version: 7.18.6 - resolution: "@babel/plugin-transform-reserved-words@npm:7.18.6" +"@babel/plugin-transform-reserved-words@npm:^7.22.5": + version: 7.22.5 + resolution: "@babel/plugin-transform-reserved-words@npm:7.22.5" dependencies: - "@babel/helper-plugin-utils": ^7.18.6 + "@babel/helper-plugin-utils": ^7.22.5 peerDependencies: "@babel/core": ^7.0.0-0 - checksum: 0738cdc30abdae07c8ec4b233b30c31f68b3ff0eaa40eddb45ae607c066127f5fa99ddad3c0177d8e2832e3a7d3ad115775c62b431ebd6189c40a951b867a80c + checksum: 3ffd7dbc425fe8132bfec118b9817572799cab1473113a635d25ab606c1f5a2341a636c04cf6b22df3813320365ed5a965b5eeb3192320a10e4cc2c137bd8bfc languageName: node linkType: hard -"@babel/plugin-transform-shorthand-properties@npm:^7.18.6": - version: 7.18.6 - resolution: "@babel/plugin-transform-shorthand-properties@npm:7.18.6" +"@babel/plugin-transform-shorthand-properties@npm:^7.22.5": + version: 7.22.5 + resolution: "@babel/plugin-transform-shorthand-properties@npm:7.22.5" dependencies: - "@babel/helper-plugin-utils": ^7.18.6 + "@babel/helper-plugin-utils": ^7.22.5 peerDependencies: "@babel/core": ^7.0.0-0 - checksum: b8e4e8acc2700d1e0d7d5dbfd4fdfb935651913de6be36e6afb7e739d8f9ca539a5150075a0f9b79c88be25ddf45abb912fe7abf525f0b80f5b9d9860de685d7 + checksum: a5ac902c56ea8effa99f681340ee61bac21094588f7aef0bc01dff98246651702e677552fa6d10e548c4ac22a3ffad047dd2f8c8f0540b68316c2c203e56818b languageName: node linkType: hard -"@babel/plugin-transform-spread@npm:^7.20.7": - version: 7.20.7 - resolution: "@babel/plugin-transform-spread@npm:7.20.7" +"@babel/plugin-transform-spread@npm:^7.22.5": + version: 7.22.5 + resolution: "@babel/plugin-transform-spread@npm:7.22.5" dependencies: - "@babel/helper-plugin-utils": ^7.20.2 - "@babel/helper-skip-transparent-expression-wrappers": ^7.20.0 + "@babel/helper-plugin-utils": ^7.22.5 + "@babel/helper-skip-transparent-expression-wrappers": ^7.22.5 peerDependencies: "@babel/core": ^7.0.0-0 - checksum: 8ea698a12da15718aac7489d4cde10beb8a3eea1f66167d11ab1e625033641e8b328157fd1a0b55dd6531933a160c01fc2e2e61132a385cece05f26429fd0cc2 + checksum: 5587f0deb60b3dfc9b274e269031cc45ec75facccf1933ea2ea71ced9fd3ce98ed91bb36d6cd26817c14474b90ed998c5078415f0eab531caf301496ce24c95c languageName: node linkType: hard -"@babel/plugin-transform-sticky-regex@npm:^7.18.6": - version: 7.18.6 - resolution: "@babel/plugin-transform-sticky-regex@npm:7.18.6" +"@babel/plugin-transform-sticky-regex@npm:^7.22.5": + version: 7.22.5 + resolution: "@babel/plugin-transform-sticky-regex@npm:7.22.5" dependencies: - "@babel/helper-plugin-utils": ^7.18.6 + "@babel/helper-plugin-utils": ^7.22.5 + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: 63b2c575e3e7f96c32d52ed45ee098fb7d354b35c2223b8c8e76840b32cc529ee0c0ceb5742fd082e56e91e3d82842a367ce177e82b05039af3d602c9627a729 + languageName: node + linkType: hard + +"@babel/plugin-transform-template-literals@npm:^7.22.5": + version: 7.22.5 + resolution: "@babel/plugin-transform-template-literals@npm:7.22.5" + dependencies: + "@babel/helper-plugin-utils": ^7.22.5 peerDependencies: "@babel/core": ^7.0.0-0 - checksum: 68ea18884ae9723443ffa975eb736c8c0d751265859cd3955691253f7fee37d7a0f7efea96c8a062876af49a257a18ea0ed5fea0d95a7b3611ce40f7ee23aee3 + checksum: 27e9bb030654cb425381c69754be4abe6a7c75b45cd7f962cd8d604b841b2f0fb7b024f2efc1c25cc53f5b16d79d5e8cfc47cacbdaa983895b3aeefa3e7e24ff languageName: node linkType: hard -"@babel/plugin-transform-template-literals@npm:^7.18.9": - version: 7.18.9 - resolution: "@babel/plugin-transform-template-literals@npm:7.18.9" +"@babel/plugin-transform-typeof-symbol@npm:^7.22.5": + version: 7.22.5 + resolution: "@babel/plugin-transform-typeof-symbol@npm:7.22.5" dependencies: - "@babel/helper-plugin-utils": ^7.18.9 + "@babel/helper-plugin-utils": ^7.22.5 peerDependencies: "@babel/core": ^7.0.0-0 - checksum: 3d2fcd79b7c345917f69b92a85bdc3ddd68ce2c87dc70c7d61a8373546ccd1f5cb8adc8540b49dfba08e1b82bb7b3bbe23a19efdb2b9c994db2db42906ca9fb2 + checksum: 82a53a63ffc3010b689ca9a54e5f53b2718b9f4b4a9818f36f9b7dba234f38a01876680553d2716a645a61920b5e6e4aaf8d4a0064add379b27ca0b403049512 languageName: node linkType: hard -"@babel/plugin-transform-typeof-symbol@npm:^7.18.9": - version: 7.18.9 - resolution: "@babel/plugin-transform-typeof-symbol@npm:7.18.9" +"@babel/plugin-transform-typescript@npm:^7.22.5": + version: 7.22.9 + resolution: "@babel/plugin-transform-typescript@npm:7.22.9" dependencies: - "@babel/helper-plugin-utils": ^7.18.9 + "@babel/helper-annotate-as-pure": ^7.22.5 + "@babel/helper-create-class-features-plugin": ^7.22.9 + "@babel/helper-plugin-utils": ^7.22.5 + "@babel/plugin-syntax-typescript": ^7.22.5 peerDependencies: "@babel/core": ^7.0.0-0 - checksum: e754e0d8b8a028c52e10c148088606e3f7a9942c57bd648fc0438e5b4868db73c386a5ed47ab6d6f0594aae29ee5ffc2ffc0f7ebee7fae560a066d6dea811cd4 + checksum: 6d1317a54d093b302599a4bee8ba9865d0de8b7b6ac1a0746c4316231d632f75b7f086e6e78acb9ac95ba12ba3b9da462dc9ca69370abb4603c4cc987f62e67e languageName: node linkType: hard -"@babel/plugin-transform-typescript@npm:^7.21.3": - version: 7.21.3 - resolution: "@babel/plugin-transform-typescript@npm:7.21.3" +"@babel/plugin-transform-unicode-escapes@npm:^7.22.5": + version: 7.22.5 + resolution: "@babel/plugin-transform-unicode-escapes@npm:7.22.5" dependencies: - "@babel/helper-annotate-as-pure": ^7.18.6 - "@babel/helper-create-class-features-plugin": ^7.21.0 - "@babel/helper-plugin-utils": ^7.20.2 - "@babel/plugin-syntax-typescript": ^7.20.0 + "@babel/helper-plugin-utils": ^7.22.5 peerDependencies: "@babel/core": ^7.0.0-0 - checksum: c16fd577bf43f633deb76fca2a8527d8ae25968c8efdf327c1955472c3e0257e62992473d1ad7f9ee95379ce2404699af405ea03346055adadd3478ad0ecd117 + checksum: da5e85ab3bb33a75cbf6181bfd236b208dc934702fd304db127232f17b4e0f42c6d3f238de8589470b4190906967eea8ca27adf3ae9d8ee4de2a2eae906ed186 languageName: node linkType: hard -"@babel/plugin-transform-unicode-escapes@npm:^7.21.5": - version: 7.21.5 - resolution: "@babel/plugin-transform-unicode-escapes@npm:7.21.5" +"@babel/plugin-transform-unicode-property-regex@npm:^7.22.5": + version: 7.22.5 + resolution: "@babel/plugin-transform-unicode-property-regex@npm:7.22.5" dependencies: - "@babel/helper-plugin-utils": ^7.21.5 + "@babel/helper-create-regexp-features-plugin": ^7.22.5 + "@babel/helper-plugin-utils": ^7.22.5 peerDependencies: "@babel/core": ^7.0.0-0 - checksum: 6504d642d0449a275191b624bd94d3e434ae154e610bf2f0e3c109068b287d2474f68e1da64b47f21d193cd67b27ee4643877d530187670565cac46e29fd257d + checksum: 2495e5f663cb388e3d888b4ba3df419ac436a5012144ac170b622ddfc221f9ea9bdba839fa2bc0185cb776b578030666406452ec7791cbf0e7a3d4c88ae9574c languageName: node linkType: hard -"@babel/plugin-transform-unicode-regex@npm:^7.18.6": - version: 7.18.6 - resolution: "@babel/plugin-transform-unicode-regex@npm:7.18.6" +"@babel/plugin-transform-unicode-regex@npm:^7.22.5": + version: 7.22.5 + resolution: "@babel/plugin-transform-unicode-regex@npm:7.22.5" dependencies: - "@babel/helper-create-regexp-features-plugin": ^7.18.6 - "@babel/helper-plugin-utils": ^7.18.6 + "@babel/helper-create-regexp-features-plugin": ^7.22.5 + "@babel/helper-plugin-utils": ^7.22.5 peerDependencies: "@babel/core": ^7.0.0-0 - checksum: d9e18d57536a2d317fb0b7c04f8f55347f3cfacb75e636b4c6fa2080ab13a3542771b5120e726b598b815891fc606d1472ac02b749c69fd527b03847f22dc25e - languageName: node - linkType: hard - -"@babel/preset-env@npm:^7.21.5": - version: 7.21.5 - resolution: "@babel/preset-env@npm:7.21.5" - dependencies: - "@babel/compat-data": ^7.21.5 - "@babel/helper-compilation-targets": ^7.21.5 - "@babel/helper-plugin-utils": ^7.21.5 - "@babel/helper-validator-option": ^7.21.0 - "@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression": ^7.18.6 - "@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining": ^7.20.7 - "@babel/plugin-proposal-async-generator-functions": ^7.20.7 - "@babel/plugin-proposal-class-properties": ^7.18.6 - "@babel/plugin-proposal-class-static-block": ^7.21.0 - "@babel/plugin-proposal-dynamic-import": ^7.18.6 - "@babel/plugin-proposal-export-namespace-from": ^7.18.9 - "@babel/plugin-proposal-json-strings": ^7.18.6 - "@babel/plugin-proposal-logical-assignment-operators": ^7.20.7 - "@babel/plugin-proposal-nullish-coalescing-operator": ^7.18.6 - "@babel/plugin-proposal-numeric-separator": ^7.18.6 - "@babel/plugin-proposal-object-rest-spread": ^7.20.7 - "@babel/plugin-proposal-optional-catch-binding": ^7.18.6 - "@babel/plugin-proposal-optional-chaining": ^7.21.0 - "@babel/plugin-proposal-private-methods": ^7.18.6 - "@babel/plugin-proposal-private-property-in-object": ^7.21.0 - "@babel/plugin-proposal-unicode-property-regex": ^7.18.6 + checksum: 6b5d1404c8c623b0ec9bd436c00d885a17d6a34f3f2597996343ddb9d94f6379705b21582dfd4cec2c47fd34068872e74ab6b9580116c0566b3f9447e2a7fa06 + languageName: node + linkType: hard + +"@babel/plugin-transform-unicode-sets-regex@npm:^7.22.5": + version: 7.22.5 + resolution: "@babel/plugin-transform-unicode-sets-regex@npm:7.22.5" + dependencies: + "@babel/helper-create-regexp-features-plugin": ^7.22.5 + "@babel/helper-plugin-utils": ^7.22.5 + peerDependencies: + "@babel/core": ^7.0.0 + checksum: c042070f980b139547f8b0179efbc049ac5930abec7fc26ed7a41d89a048d8ab17d362200e204b6f71c3c20d6991a0e74415e1a412a49adc8131c2a40c04822e + languageName: node + linkType: hard + +"@babel/preset-env@npm:^7.22.9": + version: 7.22.9 + resolution: "@babel/preset-env@npm:7.22.9" + dependencies: + "@babel/compat-data": ^7.22.9 + "@babel/helper-compilation-targets": ^7.22.9 + "@babel/helper-plugin-utils": ^7.22.5 + "@babel/helper-validator-option": ^7.22.5 + "@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression": ^7.22.5 + "@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining": ^7.22.5 + "@babel/plugin-proposal-private-property-in-object": 7.21.0-placeholder-for-preset-env.2 "@babel/plugin-syntax-async-generators": ^7.8.4 "@babel/plugin-syntax-class-properties": ^7.12.13 "@babel/plugin-syntax-class-static-block": ^7.14.5 "@babel/plugin-syntax-dynamic-import": ^7.8.3 "@babel/plugin-syntax-export-namespace-from": ^7.8.3 - "@babel/plugin-syntax-import-assertions": ^7.20.0 + "@babel/plugin-syntax-import-assertions": ^7.22.5 + "@babel/plugin-syntax-import-attributes": ^7.22.5 "@babel/plugin-syntax-import-meta": ^7.10.4 "@babel/plugin-syntax-json-strings": ^7.8.3 "@babel/plugin-syntax-logical-assignment-operators": ^7.10.4 @@ -1225,54 +1236,71 @@ __metadata: "@babel/plugin-syntax-optional-chaining": ^7.8.3 "@babel/plugin-syntax-private-property-in-object": ^7.14.5 "@babel/plugin-syntax-top-level-await": ^7.14.5 - "@babel/plugin-transform-arrow-functions": ^7.21.5 - "@babel/plugin-transform-async-to-generator": ^7.20.7 - "@babel/plugin-transform-block-scoped-functions": ^7.18.6 - "@babel/plugin-transform-block-scoping": ^7.21.0 - "@babel/plugin-transform-classes": ^7.21.0 - "@babel/plugin-transform-computed-properties": ^7.21.5 - "@babel/plugin-transform-destructuring": ^7.21.3 - "@babel/plugin-transform-dotall-regex": ^7.18.6 - "@babel/plugin-transform-duplicate-keys": ^7.18.9 - "@babel/plugin-transform-exponentiation-operator": ^7.18.6 - "@babel/plugin-transform-for-of": ^7.21.5 - "@babel/plugin-transform-function-name": ^7.18.9 - "@babel/plugin-transform-literals": ^7.18.9 - "@babel/plugin-transform-member-expression-literals": ^7.18.6 - "@babel/plugin-transform-modules-amd": ^7.20.11 - "@babel/plugin-transform-modules-commonjs": ^7.21.5 - "@babel/plugin-transform-modules-systemjs": ^7.20.11 - "@babel/plugin-transform-modules-umd": ^7.18.6 - "@babel/plugin-transform-named-capturing-groups-regex": ^7.20.5 - "@babel/plugin-transform-new-target": ^7.18.6 - "@babel/plugin-transform-object-super": ^7.18.6 - "@babel/plugin-transform-parameters": ^7.21.3 - "@babel/plugin-transform-property-literals": ^7.18.6 - "@babel/plugin-transform-regenerator": ^7.21.5 - "@babel/plugin-transform-reserved-words": ^7.18.6 - "@babel/plugin-transform-shorthand-properties": ^7.18.6 - "@babel/plugin-transform-spread": ^7.20.7 - "@babel/plugin-transform-sticky-regex": ^7.18.6 - "@babel/plugin-transform-template-literals": ^7.18.9 - "@babel/plugin-transform-typeof-symbol": ^7.18.9 - "@babel/plugin-transform-unicode-escapes": ^7.21.5 - "@babel/plugin-transform-unicode-regex": ^7.18.6 + "@babel/plugin-syntax-unicode-sets-regex": ^7.18.6 + "@babel/plugin-transform-arrow-functions": ^7.22.5 + "@babel/plugin-transform-async-generator-functions": ^7.22.7 + "@babel/plugin-transform-async-to-generator": ^7.22.5 + "@babel/plugin-transform-block-scoped-functions": ^7.22.5 + "@babel/plugin-transform-block-scoping": ^7.22.5 + "@babel/plugin-transform-class-properties": ^7.22.5 + "@babel/plugin-transform-class-static-block": ^7.22.5 + "@babel/plugin-transform-classes": ^7.22.6 + "@babel/plugin-transform-computed-properties": ^7.22.5 + "@babel/plugin-transform-destructuring": ^7.22.5 + "@babel/plugin-transform-dotall-regex": ^7.22.5 + "@babel/plugin-transform-duplicate-keys": ^7.22.5 + "@babel/plugin-transform-dynamic-import": ^7.22.5 + "@babel/plugin-transform-exponentiation-operator": ^7.22.5 + "@babel/plugin-transform-export-namespace-from": ^7.22.5 + "@babel/plugin-transform-for-of": ^7.22.5 + "@babel/plugin-transform-function-name": ^7.22.5 + "@babel/plugin-transform-json-strings": ^7.22.5 + "@babel/plugin-transform-literals": ^7.22.5 + "@babel/plugin-transform-logical-assignment-operators": ^7.22.5 + "@babel/plugin-transform-member-expression-literals": ^7.22.5 + "@babel/plugin-transform-modules-amd": ^7.22.5 + "@babel/plugin-transform-modules-commonjs": ^7.22.5 + "@babel/plugin-transform-modules-systemjs": ^7.22.5 + "@babel/plugin-transform-modules-umd": ^7.22.5 + "@babel/plugin-transform-named-capturing-groups-regex": ^7.22.5 + "@babel/plugin-transform-new-target": ^7.22.5 + "@babel/plugin-transform-nullish-coalescing-operator": ^7.22.5 + "@babel/plugin-transform-numeric-separator": ^7.22.5 + "@babel/plugin-transform-object-rest-spread": ^7.22.5 + "@babel/plugin-transform-object-super": ^7.22.5 + "@babel/plugin-transform-optional-catch-binding": ^7.22.5 + "@babel/plugin-transform-optional-chaining": ^7.22.6 + "@babel/plugin-transform-parameters": ^7.22.5 + "@babel/plugin-transform-private-methods": ^7.22.5 + "@babel/plugin-transform-private-property-in-object": ^7.22.5 + "@babel/plugin-transform-property-literals": ^7.22.5 + "@babel/plugin-transform-regenerator": ^7.22.5 + "@babel/plugin-transform-reserved-words": ^7.22.5 + "@babel/plugin-transform-shorthand-properties": ^7.22.5 + "@babel/plugin-transform-spread": ^7.22.5 + "@babel/plugin-transform-sticky-regex": ^7.22.5 + "@babel/plugin-transform-template-literals": ^7.22.5 + "@babel/plugin-transform-typeof-symbol": ^7.22.5 + "@babel/plugin-transform-unicode-escapes": ^7.22.5 + "@babel/plugin-transform-unicode-property-regex": ^7.22.5 + "@babel/plugin-transform-unicode-regex": ^7.22.5 + "@babel/plugin-transform-unicode-sets-regex": ^7.22.5 "@babel/preset-modules": ^0.1.5 - "@babel/types": ^7.21.5 - babel-plugin-polyfill-corejs2: ^0.3.3 - babel-plugin-polyfill-corejs3: ^0.6.0 - babel-plugin-polyfill-regenerator: ^0.4.1 - core-js-compat: ^3.25.1 - semver: ^6.3.0 + "@babel/types": ^7.22.5 + babel-plugin-polyfill-corejs2: ^0.4.4 + babel-plugin-polyfill-corejs3: ^0.8.2 + babel-plugin-polyfill-regenerator: ^0.5.1 + core-js-compat: ^3.31.0 + semver: ^6.3.1 peerDependencies: "@babel/core": ^7.0.0-0 - checksum: 86e167f3a351c89f8cd1409262481ece6ddc085b76147e801530ce29d60b1cfda8b264b1efd1ae27b8181b073a923c7161f21e2ebc0a41d652d717b10cf1c829 + checksum: 6caa2897bbda30c6932aed0a03827deb1337c57108050c9f97dc9a857e1533c7125b168b6d70b9d191965bf05f9f233f0ad20303080505dff7ce39740aaa759d languageName: node linkType: hard "@babel/preset-modules@npm:^0.1.5": - version: 0.1.5 - resolution: "@babel/preset-modules@npm:0.1.5" + version: 0.1.6 + resolution: "@babel/preset-modules@npm:0.1.6" dependencies: "@babel/helper-plugin-utils": ^7.0.0 "@babel/plugin-proposal-unicode-property-regex": ^7.4.4 @@ -1280,23 +1308,23 @@ __metadata: "@babel/types": ^7.4.4 esutils: ^2.0.2 peerDependencies: - "@babel/core": ^7.0.0-0 - checksum: 8430e0e9e9d520b53e22e8c4c6a5a080a12b63af6eabe559c2310b187bd62ae113f3da82ba33e9d1d0f3230930ca702843aae9dd226dec51f7d7114dc1f51c10 + "@babel/core": ^7.0.0-0 || ^8.0.0-0 <8.0.0 + checksum: 9700992d2b9526e703ab49eb8c4cd0b26bec93594d57c6b808967619df1a387565e0e58829b65b5bd6d41049071ea0152c9195b39599515fddb3e52b09a55ff0 languageName: node linkType: hard -"@babel/preset-typescript@npm:^7.21.5": - version: 7.21.5 - resolution: "@babel/preset-typescript@npm:7.21.5" +"@babel/preset-typescript@npm:^7.22.5": + version: 7.22.5 + resolution: "@babel/preset-typescript@npm:7.22.5" dependencies: - "@babel/helper-plugin-utils": ^7.21.5 - "@babel/helper-validator-option": ^7.21.0 - "@babel/plugin-syntax-jsx": ^7.21.4 - "@babel/plugin-transform-modules-commonjs": ^7.21.5 - "@babel/plugin-transform-typescript": ^7.21.3 + "@babel/helper-plugin-utils": ^7.22.5 + "@babel/helper-validator-option": ^7.22.5 + "@babel/plugin-syntax-jsx": ^7.22.5 + "@babel/plugin-transform-modules-commonjs": ^7.22.5 + "@babel/plugin-transform-typescript": ^7.22.5 peerDependencies: "@babel/core": ^7.0.0-0 - checksum: e7b35c435139eec1d6bd9f57e8f3eb79bfc2da2c57a34ad9e9ea848ba4ecd72791cf4102df456604ab07c7f4518525b0764754b6dd5898036608b351e0792448 + checksum: 7be1670cb4404797d3a473bd72d66eb2b3e0f2f8a672a5e40bdb0812cc66085ec84bcd7b896709764cabf042fdc6b7f2d4755ac7cce10515eb596ff61dab5154 languageName: node linkType: hard @@ -1308,51 +1336,51 @@ __metadata: linkType: hard "@babel/runtime@npm:^7.8.4": - version: 7.17.2 - resolution: "@babel/runtime@npm:7.17.2" + version: 7.22.6 + resolution: "@babel/runtime@npm:7.22.6" dependencies: - regenerator-runtime: ^0.13.4 - checksum: a48702d271ecc59c09c397856407afa29ff980ab537b3da58eeee1aeaa0f545402d340a1680c9af58aec94dfdcbccfb6abb211991b74686a86d03d3f6956cacd + regenerator-runtime: ^0.13.11 + checksum: e585338287c4514a713babf4fdb8fc2a67adcebab3e7723a739fc62c79cfda875b314c90fd25f827afb150d781af97bc16c85bfdbfa2889f06053879a1ddb597 languageName: node linkType: hard -"@babel/template@npm:^7.18.10, @babel/template@npm:^7.20.7, @babel/template@npm:^7.21.9, @babel/template@npm:^7.3.3": - version: 7.21.9 - resolution: "@babel/template@npm:7.21.9" +"@babel/template@npm:^7.22.5, @babel/template@npm:^7.3.3": + version: 7.22.5 + resolution: "@babel/template@npm:7.22.5" dependencies: - "@babel/code-frame": ^7.21.4 - "@babel/parser": ^7.21.9 - "@babel/types": ^7.21.5 - checksum: 6ec2c60d4d53b2a9230ab82c399ba6525df87e9a4e01e4b111e071cbad283b1362e7c99a1bc50027073f44f2de36a495a89c27112c4e7efe7ef9c8d9c84de2ec + "@babel/code-frame": ^7.22.5 + "@babel/parser": ^7.22.5 + "@babel/types": ^7.22.5 + checksum: c5746410164039aca61829cdb42e9a55410f43cace6f51ca443313f3d0bdfa9a5a330d0b0df73dc17ef885c72104234ae05efede37c1cc8a72dc9f93425977a3 languageName: node linkType: hard -"@babel/traverse@npm:^7.20.5, @babel/traverse@npm:^7.21.5, @babel/traverse@npm:^7.22.1, @babel/traverse@npm:^7.7.2": - version: 7.22.4 - resolution: "@babel/traverse@npm:7.22.4" +"@babel/traverse@npm:^7.22.6, @babel/traverse@npm:^7.22.8, @babel/traverse@npm:^7.7.2": + version: 7.22.8 + resolution: "@babel/traverse@npm:7.22.8" dependencies: - "@babel/code-frame": ^7.21.4 - "@babel/generator": ^7.22.3 - "@babel/helper-environment-visitor": ^7.22.1 - "@babel/helper-function-name": ^7.21.0 - "@babel/helper-hoist-variables": ^7.18.6 - "@babel/helper-split-export-declaration": ^7.18.6 - "@babel/parser": ^7.22.4 - "@babel/types": ^7.22.4 + "@babel/code-frame": ^7.22.5 + "@babel/generator": ^7.22.7 + "@babel/helper-environment-visitor": ^7.22.5 + "@babel/helper-function-name": ^7.22.5 + "@babel/helper-hoist-variables": ^7.22.5 + "@babel/helper-split-export-declaration": ^7.22.6 + "@babel/parser": ^7.22.7 + "@babel/types": ^7.22.5 debug: ^4.1.0 globals: ^11.1.0 - checksum: 9560ae22092d5a7c52849145dd3e5aed2ffb73d61255e70e19e3fbd06bcbafbbdecea28df40a42ee3b60b01e85a42224ec841df93e867547e329091cc2f2bb6f + checksum: a381369bc3eedfd13ed5fef7b884657f1c29024ea7388198149f0edc34bd69ce3966e9f40188d15f56490a5e12ba250ccc485f2882b53d41b054fccefb233e33 languageName: node linkType: hard -"@babel/types@npm:^7.0.0, @babel/types@npm:^7.18.6, @babel/types@npm:^7.18.9, @babel/types@npm:^7.20.0, @babel/types@npm:^7.20.5, @babel/types@npm:^7.20.7, @babel/types@npm:^7.21.0, @babel/types@npm:^7.21.4, @babel/types@npm:^7.21.5, @babel/types@npm:^7.22.0, @babel/types@npm:^7.22.3, @babel/types@npm:^7.22.4, @babel/types@npm:^7.3.0, @babel/types@npm:^7.3.3, @babel/types@npm:^7.4.4, @babel/types@npm:^7.8.3": - version: 7.22.4 - resolution: "@babel/types@npm:7.22.4" +"@babel/types@npm:^7.0.0, @babel/types@npm:^7.20.7, @babel/types@npm:^7.22.5, @babel/types@npm:^7.3.0, @babel/types@npm:^7.3.3, @babel/types@npm:^7.4.4, @babel/types@npm:^7.8.3": + version: 7.22.5 + resolution: "@babel/types@npm:7.22.5" dependencies: - "@babel/helper-string-parser": ^7.21.5 - "@babel/helper-validator-identifier": ^7.19.1 + "@babel/helper-string-parser": ^7.22.5 + "@babel/helper-validator-identifier": ^7.22.5 to-fast-properties: ^2.0.0 - checksum: ffe36bb4f4a99ad13c426a98c3b508d70736036cae4e471d9c862e3a579847ed4f480686af0fce2633f6f7c0f0d3bf02da73da36e7edd3fde0b2061951dcba9a + checksum: c13a9c1dc7d2d1a241a2f8363540cb9af1d66e978e8984b400a20c4f38ba38ca29f06e26a0f2d49a70bad9e57615dac09c35accfddf1bb90d23cd3e0a0bab892 languageName: node linkType: hard @@ -1678,7 +1706,7 @@ __metadata: languageName: node linkType: hard -"@jest/expect-utils@npm:^29.5.0": +"@jest/expect-utils@npm:25.1.0 - 29, @jest/expect-utils@npm:^29.5.0": version: 29.5.0 resolution: "@jest/expect-utils@npm:29.5.0" dependencies: @@ -3207,6 +3235,7 @@ __metadata: simple-git: ^3.16.0 term-size: 2.1.0 tmp: ~0.2.1 + ts-jest: ^29.1.0 typescript: ^4.9.5 widest-line: 3.1.0 yargs: ^17.7.2 @@ -3892,39 +3921,39 @@ __metadata: languageName: node linkType: hard -"babel-plugin-polyfill-corejs2@npm:^0.3.3": - version: 0.3.3 - resolution: "babel-plugin-polyfill-corejs2@npm:0.3.3" +"babel-plugin-polyfill-corejs2@npm:^0.4.4": + version: 0.4.5 + resolution: "babel-plugin-polyfill-corejs2@npm:0.4.5" dependencies: - "@babel/compat-data": ^7.17.7 - "@babel/helper-define-polyfill-provider": ^0.3.3 - semver: ^6.1.1 + "@babel/compat-data": ^7.22.6 + "@babel/helper-define-polyfill-provider": ^0.4.2 + semver: ^6.3.1 peerDependencies: - "@babel/core": ^7.0.0-0 - checksum: 7db3044993f3dddb3cc3d407bc82e640964a3bfe22de05d90e1f8f7a5cb71460011ab136d3c03c6c1ba428359ebf635688cd6205e28d0469bba221985f5c6179 + "@babel/core": ^7.4.0 || ^8.0.0-0 <8.0.0 + checksum: 33a8e06aa54e2858d211c743d179f0487b03222f9ca1bfd7c4865bca243fca942a3358cb75f6bb894ed476cbddede834811fbd6903ff589f055821146f053e1a languageName: node linkType: hard -"babel-plugin-polyfill-corejs3@npm:^0.6.0": - version: 0.6.0 - resolution: "babel-plugin-polyfill-corejs3@npm:0.6.0" +"babel-plugin-polyfill-corejs3@npm:^0.8.2": + version: 0.8.3 + resolution: "babel-plugin-polyfill-corejs3@npm:0.8.3" dependencies: - "@babel/helper-define-polyfill-provider": ^0.3.3 - core-js-compat: ^3.25.1 + "@babel/helper-define-polyfill-provider": ^0.4.2 + core-js-compat: ^3.31.0 peerDependencies: - "@babel/core": ^7.0.0-0 - checksum: 470bb8c59f7c0912bd77fe1b5a2e72f349b3f65bbdee1d60d6eb7e1f4a085c6f24b2dd5ab4ac6c2df6444a96b070ef6790eccc9edb6a2668c60d33133bfb62c6 + "@babel/core": ^7.4.0 || ^8.0.0-0 <8.0.0 + checksum: dcbb30e551702a82cfd4d2c375da2c317658e55f95e9edcda93b9bbfdcc8fb6e5344efcb144e04d3406859e7682afce7974c60ededd9f12072a48a83dd22a0da languageName: node linkType: hard -"babel-plugin-polyfill-regenerator@npm:^0.4.1": - version: 0.4.1 - resolution: "babel-plugin-polyfill-regenerator@npm:0.4.1" +"babel-plugin-polyfill-regenerator@npm:^0.5.1": + version: 0.5.2 + resolution: "babel-plugin-polyfill-regenerator@npm:0.5.2" dependencies: - "@babel/helper-define-polyfill-provider": ^0.3.3 + "@babel/helper-define-polyfill-provider": ^0.4.2 peerDependencies: - "@babel/core": ^7.0.0-0 - checksum: ab0355efbad17d29492503230387679dfb780b63b25408990d2e4cf421012dae61d6199ddc309f4d2409ce4e9d3002d187702700dd8f4f8770ebbba651ed066c + "@babel/core": ^7.4.0 || ^8.0.0-0 <8.0.0 + checksum: d962200f604016a9a09bc9b4aaf60a3db7af876bb65bcefaeac04d44ac9d9ec4037cf24ce117760cc141d7046b6394c7eb0320ba9665cb4a2ee64df2be187c93 languageName: node linkType: hard @@ -4102,17 +4131,17 @@ __metadata: languageName: node linkType: hard -"browserslist@npm:^4.14.5, browserslist@npm:^4.21.3, browserslist@npm:^4.21.5": - version: 4.21.5 - resolution: "browserslist@npm:4.21.5" +"browserslist@npm:^4.14.5, browserslist@npm:^4.21.9": + version: 4.21.9 + resolution: "browserslist@npm:4.21.9" dependencies: - caniuse-lite: ^1.0.30001449 - electron-to-chromium: ^1.4.284 - node-releases: ^2.0.8 - update-browserslist-db: ^1.0.10 + caniuse-lite: ^1.0.30001503 + electron-to-chromium: ^1.4.431 + node-releases: ^2.0.12 + update-browserslist-db: ^1.0.11 bin: browserslist: cli.js - checksum: 9755986b22e73a6a1497fd8797aedd88e04270be33ce66ed5d85a1c8a798292a65e222b0f251bafa1c2522261e237d73b08b58689d4920a607e5a53d56dc4706 + checksum: 80d3820584e211484ad1b1a5cfdeca1dd00442f47be87e117e1dda34b628c87e18b81ae7986fa5977b3e6a03154f6d13cd763baa6b8bf5dd9dd19f4926603698 languageName: node linkType: hard @@ -4243,10 +4272,10 @@ __metadata: languageName: node linkType: hard -"caniuse-lite@npm:^1.0.30001449": - version: 1.0.30001465 - resolution: "caniuse-lite@npm:1.0.30001465" - checksum: c991ecdfff378a22b268f9b1eb732d003c8ad89db3241a4cdec3b3ec3354aa966a44171cb806c90abe2e3f0573d67dc29a7dce2478b1f070b23747c392244c5d +"caniuse-lite@npm:^1.0.30001503": + version: 1.0.30001517 + resolution: "caniuse-lite@npm:1.0.30001517" + checksum: e4e87436ae1c4408cf4438aac22902b31eb03f3f5bad7f33bc518d12ffb35f3fd9395ccf7efc608ee046f90ce324ec6f7f26f8a8172b8c43c26a06ecee612a29 languageName: node linkType: hard @@ -4604,12 +4633,12 @@ __metadata: languageName: node linkType: hard -"core-js-compat@npm:^3.25.1": - version: 3.30.1 - resolution: "core-js-compat@npm:3.30.1" +"core-js-compat@npm:^3.31.0": + version: 3.32.0 + resolution: "core-js-compat@npm:3.32.0" dependencies: - browserslist: ^4.21.5 - checksum: e450a9771fc927ce982333929e1c4b32f180f641e4cfff9de6ed44b5930de19be7707cf74f45d1746ca69b8e8ac0698a555cb7244fbfbed6c38ca93844207bf7 + browserslist: ^4.21.9 + checksum: e740b348dfd8dc25ac851ab625a1d5a63c012252bdd6d8ae92d1b2ebf46e6cf57ca6cbec4494cbacdd90d3f8ed822480c8a7106c990dbe9055ebdf5b79fbb92e languageName: node linkType: hard @@ -5133,10 +5162,10 @@ __metadata: languageName: node linkType: hard -"electron-to-chromium@npm:^1.4.284": - version: 1.4.328 - resolution: "electron-to-chromium@npm:1.4.328" - checksum: 82c1617a77e40ac4ca5011749318a2fee8f8c75f8b517fcff7602219c85fd97a9fab2d5a1353ea10fb7f9c7d18acb90c9ed58c2292256f81e2ffa42ee66c4b0b +"electron-to-chromium@npm:^1.4.431": + version: 1.4.476 + resolution: "electron-to-chromium@npm:1.4.476" + checksum: 0b769c3b85bf4f28f58bf1c61caf14e7d5ef50e7e8250fc7594e81f4213e2c928f650a68edb8f0580e1f479b4548f34ca2096f661d44faf5de8141593d31a9b6 languageName: node linkType: hard @@ -6939,12 +6968,12 @@ __metadata: languageName: node linkType: hard -"is-core-module@npm:^2.11.0, is-core-module@npm:^2.9.0": - version: 2.11.0 - resolution: "is-core-module@npm:2.11.0" +"is-core-module@npm:^2.11.0, is-core-module@npm:^2.12.0": + version: 2.12.1 + resolution: "is-core-module@npm:2.12.1" dependencies: has: ^1.0.3 - checksum: f96fd490c6b48eb4f6d10ba815c6ef13f410b0ba6f7eb8577af51697de523e5f2cd9de1c441b51d27251bf0e4aebc936545e33a5d26d5d51f28d25698d4a8bab + checksum: f04ea30533b5e62764e7b2e049d3157dc0abd95ef44275b32489ea2081176ac9746ffb1cdb107445cf1ff0e0dfcad522726ca27c27ece64dadf3795428b8e468 languageName: node linkType: hard @@ -7460,7 +7489,7 @@ __metadata: languageName: node linkType: hard -"jest-get-type@npm:^29.4.3": +"jest-get-type@npm:25.1.0 - 29, jest-get-type@npm:^29.4.3": version: 29.4.3 resolution: "jest-get-type@npm:29.4.3" checksum: 6ac7f2dde1c65e292e4355b6c63b3a4897d7e92cb4c8afcf6d397f2682f8080e094c8b0b68205a74d269882ec06bf696a9de6cd3e1b7333531e5ed7b112605ce @@ -7499,6 +7528,7 @@ __metadata: jest: 25.1.0 - 29 jest-each: 25.1.0 - 29 jest-environment-node: 25.1.0 - 29 + typescript: ^4.9.5 languageName: unknown linkType: soft @@ -7506,14 +7536,22 @@ __metadata: version: 0.0.0-use.local resolution: "jest-integration@workspace:packages/jest-plugin/test/integration" dependencies: + "@babel/core": ^7.22.9 + "@babel/preset-env": ^7.22.9 + "@babel/preset-typescript": ^7.22.5 + "@jest/expect-utils": 25.1.0 - 29 "@types/temp": ^0.9.1 "@unflakable/jest-plugin": "workspace:^" "@unflakable/js-api": "workspace:^" escape-string-regexp: ^4.0.0 + expect: 25.1.0 - 29 jest: 25.1.0 - 29 jest-environment-node: 25.1.0 - 29 + jest-get-type: 25.1.0 - 29 + jest-matcher-utils: 25.1.0 - 29 mockttp: ^3.7.5 temp: ^0.9.4 + typescript: ^4.9.5 unflakable-test-common: "workspace:^" languageName: unknown linkType: soft @@ -7528,7 +7566,7 @@ __metadata: languageName: node linkType: hard -"jest-matcher-utils@npm:^29.5.0": +"jest-matcher-utils@npm:25.1.0 - 29, jest-matcher-utils@npm:^29.5.0": version: 29.5.0 resolution: "jest-matcher-utils@npm:29.5.0" dependencies: @@ -8709,10 +8747,10 @@ __metadata: languageName: node linkType: hard -"node-releases@npm:^2.0.8": - version: 2.0.10 - resolution: "node-releases@npm:2.0.10" - checksum: d784ecde25696a15d449c4433077f5cce620ed30a1656c4abf31282bfc691a70d9618bae6868d247a67914d1be5cc4fde22f65a05f4398cdfb92e0fc83cadfbc +"node-releases@npm:^2.0.12": + version: 2.0.13 + resolution: "node-releases@npm:2.0.13" + checksum: 17ec8f315dba62710cae71a8dad3cd0288ba943d2ece43504b3b1aa8625bf138637798ab470b1d9035b0545996f63000a8a926e0f6d35d0996424f8b6d36dda3 languageName: node linkType: hard @@ -9477,10 +9515,10 @@ __metadata: languageName: node linkType: hard -"regenerator-runtime@npm:^0.13.4": - version: 0.13.9 - resolution: "regenerator-runtime@npm:0.13.9" - checksum: 65ed455fe5afd799e2897baf691ca21c2772e1a969d19bb0c4695757c2d96249eb74ee3553ea34a91062b2a676beedf630b4c1551cc6299afb937be1426ec55e +"regenerator-runtime@npm:^0.13.11": + version: 0.13.11 + resolution: "regenerator-runtime@npm:0.13.11" + checksum: 27481628d22a1c4e3ff551096a683b424242a216fee44685467307f14d58020af1e19660bf2e26064de946bad7eff28950eae9f8209d55723e2d9351e632bbb4 languageName: node linkType: hard @@ -9590,28 +9628,28 @@ __metadata: linkType: hard "resolve@npm:^1.14.2, resolve@npm:^1.20.0, resolve@npm:^1.22.1": - version: 1.22.1 - resolution: "resolve@npm:1.22.1" + version: 1.22.3 + resolution: "resolve@npm:1.22.3" dependencies: - is-core-module: ^2.9.0 + is-core-module: ^2.12.0 path-parse: ^1.0.7 supports-preserve-symlinks-flag: ^1.0.0 bin: resolve: bin/resolve - checksum: 07af5fc1e81aa1d866cbc9e9460fbb67318a10fa3c4deadc35c3ad8a898ee9a71a86a65e4755ac3195e0ea0cfbe201eb323ebe655ce90526fd61917313a34e4e + checksum: fb834b81348428cb545ff1b828a72ea28feb5a97c026a1cf40aa1008352c72811ff4d4e71f2035273dc536dcfcae20c13604ba6283c612d70fa0b6e44519c374 languageName: node linkType: hard "resolve@patch:resolve@^1.14.2#~builtin, resolve@patch:resolve@^1.20.0#~builtin, resolve@patch:resolve@^1.22.1#~builtin": - version: 1.22.1 - resolution: "resolve@patch:resolve@npm%3A1.22.1#~builtin::version=1.22.1&hash=c3c19d" + version: 1.22.3 + resolution: "resolve@patch:resolve@npm%3A1.22.3#~builtin::version=1.22.3&hash=c3c19d" dependencies: - is-core-module: ^2.9.0 + is-core-module: ^2.12.0 path-parse: ^1.0.7 supports-preserve-symlinks-flag: ^1.0.0 bin: resolve: bin/resolve - checksum: 5656f4d0bedcf8eb52685c1abdf8fbe73a1603bb1160a24d716e27a57f6cecbe2432ff9c89c2bd57542c3a7b9d14b1882b73bfe2e9d7849c9a4c0b8b39f02b8b + checksum: ad59734723b596d0891321c951592ed9015a77ce84907f89c9d9307dd0c06e11a67906a3e628c4cae143d3e44898603478af0ddeb2bba3f229a9373efe342665 languageName: node linkType: hard @@ -9720,9 +9758,6 @@ __metadata: version: 0.0.0-use.local resolution: "root-workspace-0b6124@workspace:." dependencies: - "@babel/core": ^7.21.5 - "@babel/preset-env": ^7.21.5 - "@babel/preset-typescript": ^7.21.5 "@types/debug": ^4.1.7 "@types/js-yaml": ^4.0.5 "@types/node": ^14.18.43 @@ -9825,17 +9860,6 @@ __metadata: languageName: node linkType: hard -"semver@npm:7.x, semver@npm:^7.3.2, semver@npm:^7.3.4, semver@npm:^7.3.5, semver@npm:^7.3.7, semver@npm:^7.5.1": - version: 7.5.1 - resolution: "semver@npm:7.5.1" - dependencies: - lru-cache: ^6.0.0 - bin: - semver: bin/semver.js - checksum: d16dbedad53c65b086f79524b9ef766bf38670b2395bdad5c957f824dcc566b624988013564f4812bcace3f9d405355c3635e2007396a39d1bffc71cfec4a2fc - languageName: node - linkType: hard - "semver@npm:^5.7.0, semver@npm:^5.7.1": version: 5.7.1 resolution: "semver@npm:5.7.1" @@ -9845,12 +9869,23 @@ __metadata: languageName: node linkType: hard -"semver@npm:^6.0.0, semver@npm:^6.1.1, semver@npm:^6.1.2, semver@npm:^6.3.0": - version: 6.3.0 - resolution: "semver@npm:6.3.0" +"semver@npm:^6.0.0, semver@npm:^6.3.0, semver@npm:^6.3.1": + version: 6.3.1 + resolution: "semver@npm:6.3.1" bin: - semver: ./bin/semver.js - checksum: 1b26ecf6db9e8292dd90df4e781d91875c0dcc1b1909e70f5d12959a23c7eebb8f01ea581c00783bbee72ceeaad9505797c381756326073850dc36ed284b21b9 + semver: bin/semver.js + checksum: ae47d06de28836adb9d3e25f22a92943477371292d9b665fb023fae278d345d508ca1958232af086d85e0155aee22e313e100971898bbb8d5d89b8b1d4054ca2 + languageName: node + linkType: hard + +"semver@npm:^7.3.2, semver@npm:^7.3.4, semver@npm:^7.3.5, semver@npm:^7.3.7, semver@npm:^7.5.1, semver@npm:^7.5.3": + version: 7.5.4 + resolution: "semver@npm:7.5.4" + dependencies: + lru-cache: ^6.0.0 + bin: + semver: bin/semver.js + checksum: 12d8ad952fa353b0995bf180cdac205a4068b759a140e5d3c608317098b3575ac2f1e09182206bf2eb26120e1c0ed8fb92c48c592f6099680de56bb071423ca3 languageName: node linkType: hard @@ -10580,8 +10615,8 @@ __metadata: linkType: hard "ts-jest@npm:^29.1.0": - version: 29.1.0 - resolution: "ts-jest@npm:29.1.0" + version: 29.1.1 + resolution: "ts-jest@npm:29.1.1" dependencies: bs-logger: 0.x fast-json-stable-stringify: 2.x @@ -10589,7 +10624,7 @@ __metadata: json5: ^2.2.3 lodash.memoize: 4.x make-error: 1.x - semver: 7.x + semver: ^7.5.3 yargs-parser: ^21.0.1 peerDependencies: "@babel/core": ">=7.0.0-beta.0 <8" @@ -10608,7 +10643,7 @@ __metadata: optional: true bin: ts-jest: cli.js - checksum: 535dc42ad523cbe1e387701fb2e448518419b515c082f09b25411f0b3dd0b854cf3e8141c316d6f4b99883aeb4a4f94159cbb1edfb06d7f77ea6229fadb2e1bf + checksum: a8c9e284ed4f819526749f6e4dc6421ec666f20ab44d31b0f02b4ed979975f7580b18aea4813172d43e39b29464a71899f8893dd29b06b4a351a3af8ba47b402 languageName: node linkType: hard @@ -10865,9 +10900,9 @@ __metadata: linkType: hard "unicode-property-aliases-ecmascript@npm:^2.0.0": - version: 2.0.0 - resolution: "unicode-property-aliases-ecmascript@npm:2.0.0" - checksum: dda4d39128cbbede2ac60fbb85493d979ec65913b8a486bf7cb7a375a2346fa48cbf9dc6f1ae23376e7e8e684c2b411434891e151e865a661b40a85407db51d0 + version: 2.1.0 + resolution: "unicode-property-aliases-ecmascript@npm:2.1.0" + checksum: 243524431893649b62cc674d877bd64ef292d6071dd2fd01ab4d5ad26efbc104ffcd064f93f8a06b7e4ec54c172bf03f6417921a0d8c3a9994161fe1f88f815b languageName: node linkType: hard @@ -10917,17 +10952,17 @@ __metadata: languageName: node linkType: hard -"update-browserslist-db@npm:^1.0.10": - version: 1.0.10 - resolution: "update-browserslist-db@npm:1.0.10" +"update-browserslist-db@npm:^1.0.11": + version: 1.0.11 + resolution: "update-browserslist-db@npm:1.0.11" dependencies: escalade: ^3.1.1 picocolors: ^1.0.0 peerDependencies: browserslist: ">= 4.21.0" bin: - browserslist-lint: cli.js - checksum: 12db73b4f63029ac407b153732e7cd69a1ea8206c9100b482b7d12859cd3cd0bc59c602d7ae31e652706189f1acb90d42c53ab24a5ba563ed13aebdddc5561a0 + update-browserslist-db: cli.js + checksum: b98327518f9a345c7cad5437afae4d2ae7d865f9779554baf2a200fdf4bac4969076b679b1115434bd6557376bdd37ca7583d0f9b8f8e302d7d4cc1e91b5f231 languageName: node linkType: hard From 89111d286cb7dc862f35d70e5cb9e9bfebfa799c Mon Sep 17 00:00:00 2001 From: "David A. Ramos" Date: Mon, 24 Jul 2023 01:47:44 -0700 Subject: [PATCH 18/53] [jest] Implement experimental test-independence --- .github/workflows/ci.yaml | 18 +- packages/cypress-plugin/src/main.ts | 6 +- .../test/integration/src/run-test-case.ts | 2 +- .../test/integration/tsconfig.json | 2 +- .../test/integration/unflakable.js | 12 + packages/jest-plugin/jest-circus.d.ts | 41 ++ packages/jest-plugin/jest.config.js | 18 + packages/jest-plugin/package.json | 14 +- packages/jest-plugin/rollup.config.mjs | 3 +- packages/jest-plugin/src/config.test.ts | 130 ++++++ packages/jest-plugin/src/config.ts | 92 ++++ packages/jest-plugin/src/reporter.ts | 66 ++- packages/jest-plugin/src/runner.ts | 232 +++++++--- packages/jest-plugin/src/test-runner.ts | 87 ++++ packages/jest-plugin/src/types.ts | 34 ++ .../jest-plugin/src/vendored/getSummary.ts | 24 +- .../test/integration-input/jest.config.js | 2 +- .../test/integration-input/src/fail.test.ts | 4 +- .../test/integration-input/src/flake.test.ts | 21 +- .../test/integration-input/src/mixed.test.ts | 9 +- .../integration-input/src/quarantined.test.ts | 2 +- .../jest-plugin/test/integration/package.json | 4 +- .../test/integration/src/basic.test.ts | 6 + .../integration/src/disable-plugin.test.ts | 4 + .../test/integration/src/long-names.test.ts | 2 + .../integration/src/no-quarantine.test.ts | 2 + .../integration/src/plugin-failures.test.ts | 4 + .../test/integration/src/retries.test.ts | 2 + .../test/integration/src/run-test-case.ts | 321 ++++++++------ .../test/integration/src/skip-tests.test.ts | 8 + .../integration/src/test-independence.test.ts | 404 ++++++++++++++++++ .../integration/src/test-name-pattern.test.ts | 6 + .../test/integration/src/test-wrappers.ts | 10 + .../test/integration/src/verify-output.ts | 299 +++++++++++-- packages/jest-plugin/tsconfig.json | 8 +- packages/js-api/src/index.ts | 2 + packages/plugins-common/src/config.ts | 72 +++- packages/plugins-common/src/index.ts | 1 + packages/test-common/src/config.ts | 42 +- yarn.lock | 37 +- 40 files changed, 1714 insertions(+), 339 deletions(-) create mode 100644 packages/cypress-plugin/test/integration/unflakable.js create mode 100644 packages/jest-plugin/jest-circus.d.ts create mode 100644 packages/jest-plugin/jest.config.js create mode 100644 packages/jest-plugin/src/config.test.ts create mode 100644 packages/jest-plugin/src/config.ts create mode 100644 packages/jest-plugin/src/test-runner.ts create mode 100644 packages/jest-plugin/test/integration/src/test-independence.test.ts diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 2dfabd9..72abb77 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -28,6 +28,8 @@ jobs: cache: yarn - id: install + env: + CYPRESS_INSTALL_BINARY: "0" run: yarn install --immutable - run: yarn build @@ -50,6 +52,16 @@ jobs: name: js-api path: packages/js-api/package.tgz + - name: Run @unflakable/cypress-plugin unit tests + env: + FORCE_COLOR: "1" + run: yarn workspace @unflakable/cypress-plugin test + + - name: Run @unflakable/jest-plugin unit tests + env: + FORCE_COLOR: "1" + run: yarn workspace @unflakable/jest-plugin test + - if: ${{ always() && steps.install.outcome == 'success' }} run: yarn lint @@ -189,7 +201,8 @@ jobs: UNFLAKABLE_API_KEY=${{ secrets.UNFLAKABLE_API_KEY }} \ yarn workspace cypress-integration test \ --reporters @unflakable/jest-plugin/dist/reporter \ - --runner @unflakable/jest-plugin/dist/runner + --runner @unflakable/jest-plugin/dist/runner \ + --testRunner @unflakable/jest-plugin/dist/test-runner cypress_windows_integration_tests: name: "Cypress ${{ matrix.cypress }} Windows Node ${{ matrix.node }} Integration Tests" @@ -294,7 +307,8 @@ jobs: run: | yarn workspace cypress-integration test ` --reporters @unflakable/jest-plugin/dist/reporter ` - --runner @unflakable/jest-plugin/dist/runner + --runner @unflakable/jest-plugin/dist/runner ` + --testRunner @unflakable/jest-plugin/dist/test-runner jest_linux_integration_tests: name: "Jest ${{ matrix.jest }} Linux Node ${{ matrix.node }} Integration Tests" diff --git a/packages/cypress-plugin/src/main.ts b/packages/cypress-plugin/src/main.ts index 9c7f65c..c1deb1a 100755 --- a/packages/cypress-plugin/src/main.ts +++ b/packages/cypress-plugin/src/main.ts @@ -265,7 +265,11 @@ const main = async (): Promise => { ? path.resolve(process.cwd(), runOptions.project) : process.cwd(); - const unflakableConfig = await loadConfig(projectRoot, args["test-suite-id"]); + const unflakableConfig = await loadConfig( + projectRoot, + () => [{}, []], + args["test-suite-id"] + ); debug(`Unflakable plugin is ${unflakableConfig.enabled ? "en" : "dis"}abled`); let configFile: string | undefined = undefined; diff --git a/packages/cypress-plugin/test/integration/src/run-test-case.ts b/packages/cypress-plugin/test/integration/src/run-test-case.ts index ad3399b..35ac129 100644 --- a/packages/cypress-plugin/test/integration/src/run-test-case.ts +++ b/packages/cypress-plugin/test/integration/src/run-test-case.ts @@ -615,7 +615,7 @@ export const runTestCase = async ( ); const configMockParams: CosmiconfigMockParams = { - searchFrom: path.resolve(projectPath(params)), + expectedSearchFrom: path.resolve(projectPath(params)), searchResult: params.config !== null ? { diff --git a/packages/cypress-plugin/test/integration/tsconfig.json b/packages/cypress-plugin/test/integration/tsconfig.json index 75be32b..e935350 100644 --- a/packages/cypress-plugin/test/integration/tsconfig.json +++ b/packages/cypress-plugin/test/integration/tsconfig.json @@ -7,5 +7,5 @@ // that involve lots of Node.JS processes from inside the browser. "types": ["jest", "jest-expect-message", "node"] }, - "include": [".eslintrc.js", "jest.config.js", "src"] + "include": [".eslintrc.js", "jest.config.js", "src", "unflakable.js"] } diff --git a/packages/cypress-plugin/test/integration/unflakable.js b/packages/cypress-plugin/test/integration/unflakable.js new file mode 100644 index 0000000..ccb959b --- /dev/null +++ b/packages/cypress-plugin/test/integration/unflakable.js @@ -0,0 +1,12 @@ +// Copyright (c) 2023 Developer Innovations, LLC + +module.exports = { + __unstableIsFailureTestIndependent: [ + // Cypress sometimes hangs waiting for Chrome tabs to close. See: + // https://github.com/cypress-io/cypress/issues/27360 + // https://github.com/cypress-io/cypress/blob/fe54cf504aefcfa6b621a90baa57e345cfa09548/packages/server/lib/modes/run.ts#L676-L680 + // NB: This requires DEBUG="cypress:server:run" (at a minimum). + /attempting to close the browser tab(?:(?!resetting server state).)*$/s, + /Still waiting to connect to Edge, retrying in 1 second.*(?:Error: Test timed out after|All promises were rejected)/s, + ], +}; diff --git a/packages/jest-plugin/jest-circus.d.ts b/packages/jest-plugin/jest-circus.d.ts new file mode 100644 index 0000000..c17e897 --- /dev/null +++ b/packages/jest-plugin/jest-circus.d.ts @@ -0,0 +1,41 @@ +// Copyright (c) 2023 Developer Innovations, LLC + +// jest-circus doesn't export the types for runner. See: +// https://github.com/jestjs/jest/blob/6d2632adae0f0fa1fe116d3b475fd9783d0de1b5/packages/jest-circus/runner.js#L10-L9 +// https://github.com/jestjs/jest/blob/6d2632adae0f0fa1fe116d3b475fd9783d0de1b5/packages/jest-runner/src/types.ts#L34 +declare module "jest-circus/runner" { + import { Config } from "@jest/types"; + import { JestEnvironment } from "@jest/environment"; + import { + TestResult, + AssertionResult, + SerializableError, + Test, + } from "@jest/test-result"; + + // Exported by newer Jest versions but not older ones prior to 26.2.0. + export declare type TestEvents = { + "test-file-start": [Test]; + "test-file-success": [Test, TestResult]; + "test-file-failure": [Test, SerializableError]; + "test-case-result": [string, AssertionResult]; + }; + + export declare type TestFileEvent< + T extends keyof TestEvents = keyof TestEvents + > = (eventName: T, args: TestEvents[T]) => unknown; + + export declare type UnsubscribeFn = () => void; + + export declare type TestFramework = ( + globalConfig: Config.GlobalConfig, + config: Config.ProjectConfig, + environment: JestEnvironment, + runtime: unknown, + testPath: string, + sendMessageToJest?: TestFileEvent + ) => Promise; + + const initialize: TestFramework; + export default initialize; +} diff --git a/packages/jest-plugin/jest.config.js b/packages/jest-plugin/jest.config.js new file mode 100644 index 0000000..6318fbd --- /dev/null +++ b/packages/jest-plugin/jest.config.js @@ -0,0 +1,18 @@ +// Copyright (c) 2023 Developer Innovations, LLC + +/** @type {import('ts-jest').JestConfigWithTsJest} */ +module.exports = { + extensionsToTreatAsEsm: [".ts"], + roots: ["src"], + testEnvironment: "node", + testTimeout: 60000, + transform: { + "^.+\\.[tj]s$": [ + "ts-jest", + { + tsconfig: "tsconfig.json", + }, + ], + }, + verbose: true, +}; diff --git a/packages/jest-plugin/package.json b/packages/jest-plugin/package.json index a7bd454..1174ab1 100644 --- a/packages/jest-plugin/package.json +++ b/packages/jest-plugin/package.json @@ -17,13 +17,18 @@ "./dist/runner": { "types": "./dist/runner.d.ts", "default": "./dist/runner.js" + }, + "./dist/test-runner": { + "types": "./dist/test-runner.d.ts", + "default": "./dist/test-runner.js" } }, "files": [ "README.md", "dist/**/*.js", "dist/reporter.d.ts", - "dist/runner.d.ts" + "dist/runner.d.ts", + "dist/test-runner.d.ts" ], "dependencies": { "@unflakable/js-api": "workspace:^", @@ -32,6 +37,7 @@ "debug": "^4.3.3", "deep-equal": "^2.0.5", "escape-string-regexp": "^4.0.0", + "semver": "^7.5.4", "simple-git": "^3.16.0" }, "devDependencies": { @@ -50,6 +56,9 @@ "@types/jest": "25.1.0 - 29", "@unflakable/plugins-common": "workspace:^", "exit": "^0.1.2", + "jest": "25.1.0 - 29", + "jest-circus": "25.1.0 - 29", + "jest-environment-node": "25.1.0 - 29", "jest-runner": "25.1.0 - 29", "jest-util": "25.1.0 - 29", "rimraf": "^5.0.1", @@ -62,6 +71,7 @@ "scripts": { "build": "yarn clean && tsc --noEmit && rollup --config", "build:watch": "rollup --config --watch", - "clean": "rimraf dist/" + "clean": "rimraf dist/", + "test": "jest --useStderr --verbose" } } diff --git a/packages/jest-plugin/rollup.config.mjs b/packages/jest-plugin/rollup.config.mjs index e234e90..797cee5 100644 --- a/packages/jest-plugin/rollup.config.mjs +++ b/packages/jest-plugin/rollup.config.mjs @@ -29,7 +29,7 @@ const isExternal = (id) => */ export default [ { - input: ["src/reporter.ts", "src/runner.ts"], + input: ["src/reporter.ts", "src/runner.ts", "src/test-runner.ts"], output: { dir: "dist", format: "cjs", @@ -57,6 +57,7 @@ export default [ // NB: This should include every exported .d.ts from package.json. "dist/reporter.d.ts", "dist/runner.d.ts", + "dist/test-runner.d.ts", ], output: { dir: ".", diff --git a/packages/jest-plugin/src/config.test.ts b/packages/jest-plugin/src/config.test.ts new file mode 100644 index 0000000..ece7c1d --- /dev/null +++ b/packages/jest-plugin/src/config.test.ts @@ -0,0 +1,130 @@ +// Copyright (c) 2023 Developer Innovations, LLC + +import { loadConfig } from "./config"; +import { cosmiconfigSync, Options } from "cosmiconfig"; +import { + setCosmiconfigSync, + UnflakableConfigFile, +} from "@unflakable/plugins-common"; +import { IsFailureTestIndependentFn } from "./types"; + +const MOCK_SUITE_ID = "MOCK_SUITE_ID"; +const SEARCH_FROM = "."; + +const throwUnimplemented = (): never => { + throw new Error("unimplemented"); +}; + +const setMockConfig = ( + config: Partial< + UnflakableConfigFile & { + __unstableIsFailureTestIndependent: + | string + | RegExp + | (string | RegExp)[] + | IsFailureTestIndependentFn; + } + > +): void => { + setCosmiconfigSync( + ( + moduleName: string, + options?: Options + ): ReturnType => { + expect(moduleName).toBe("unflakable"); + expect(options?.searchPlaces).toContain("package.json"); + expect(options?.searchPlaces).toContain("unflakable.json"); + expect(options?.searchPlaces).toContain("unflakable.js"); + expect(options?.searchPlaces).toContain("unflakable.yaml"); + expect(options?.searchPlaces).toContain("unflakable.yml"); + return { + clearCaches: throwUnimplemented, + clearLoadCache: throwUnimplemented, + clearSearchCache: throwUnimplemented, + load: throwUnimplemented, + search: ( + searchFrom?: string + ): ReturnType["search"]> => { + expect(searchFrom).toBe(SEARCH_FROM); + return { + config, + filepath: "unflakable.js", + isEmpty: false, + }; + }, + }; + } + ); +}; + +describe("__unstableIsFailureTestIndependent", () => { + it("should default to undefined", () => { + setMockConfig({ testSuiteId: MOCK_SUITE_ID }); + const config = loadConfig(SEARCH_FROM); + expect(config.testSuiteId).toBe(MOCK_SUITE_ID); + expect(config.isFailureTestIndependent).toBeUndefined(); + }); + + it("should accept a string regex", () => { + setMockConfig({ + testSuiteId: MOCK_SUITE_ID, + __unstableIsFailureTestIndependent: ".*", + }); + const config = loadConfig(SEARCH_FROM); + expect(config.testSuiteId).toBe(MOCK_SUITE_ID); + expect(Array.isArray(config.isFailureTestIndependent)).toBe(true); + expect(config.isFailureTestIndependent).toHaveLength(1); + expect((config.isFailureTestIndependent as RegExp[])[0]).toBeInstanceOf( + RegExp + ); + expect((config.isFailureTestIndependent as RegExp[])[0].source).toBe(".*"); + expect((config.isFailureTestIndependent as RegExp[])[0].flags).toBe(""); + }); + + it("should accept a RegExp object", () => { + setMockConfig({ + testSuiteId: MOCK_SUITE_ID, + __unstableIsFailureTestIndependent: /.*/gs, + }); + const config = loadConfig(SEARCH_FROM); + expect(config.testSuiteId).toBe(MOCK_SUITE_ID); + expect(Array.isArray(config.isFailureTestIndependent)).toBe(true); + expect(config.isFailureTestIndependent).toHaveLength(1); + expect((config.isFailureTestIndependent as RegExp[])[0]).toBeInstanceOf( + RegExp + ); + expect((config.isFailureTestIndependent as RegExp[])[0].source).toBe(".*"); + expect((config.isFailureTestIndependent as RegExp[])[0].flags).toBe("gs"); + }); + + it("should accept an array of strings/RegExps object", () => { + setMockConfig({ + testSuiteId: MOCK_SUITE_ID, + __unstableIsFailureTestIndependent: [/foo/s, /bar/g, "baz", ".*"], + }); + const config = loadConfig(SEARCH_FROM); + expect(config.testSuiteId).toBe(MOCK_SUITE_ID); + expect(Array.isArray(config.isFailureTestIndependent)).toBe(true); + expect(config.isFailureTestIndependent).toHaveLength(4); + expect((config.isFailureTestIndependent as RegExp[])[0]).toBeInstanceOf( + RegExp + ); + expect((config.isFailureTestIndependent as RegExp[])[0].source).toBe("foo"); + expect((config.isFailureTestIndependent as RegExp[])[0].flags).toBe("s"); + expect((config.isFailureTestIndependent as RegExp[])[1]).toBeInstanceOf( + RegExp + ); + expect((config.isFailureTestIndependent as RegExp[])[1].source).toBe("bar"); + expect((config.isFailureTestIndependent as RegExp[])[1].flags).toBe("g"); + expect((config.isFailureTestIndependent as RegExp[])[2]).toBeInstanceOf( + RegExp + ); + expect((config.isFailureTestIndependent as RegExp[])[2].source).toBe("baz"); + expect((config.isFailureTestIndependent as RegExp[])[2].flags).toBe(""); + expect((config.isFailureTestIndependent as RegExp[])[3]).toBeInstanceOf( + RegExp + ); + expect((config.isFailureTestIndependent as RegExp[])[3].source).toBe(".*"); + expect((config.isFailureTestIndependent as RegExp[])[3].flags).toBe(""); + }); +}); diff --git a/packages/jest-plugin/src/config.ts b/packages/jest-plugin/src/config.ts new file mode 100644 index 0000000..b3a87d0 --- /dev/null +++ b/packages/jest-plugin/src/config.ts @@ -0,0 +1,92 @@ +// Copyright (c) 2023 Developer Innovations, LLC + +import { + IsFailureTestIndependentFn, + UnflakableJestConfig, + UnflakableJestConfigInner, +} from "./types"; +import { loadConfigSync } from "@unflakable/plugins-common"; +import util from "util"; +import semverLt from "semver/functions/lt"; +import jestPackage from "jest/package.json"; + +const parseIsFailureTestIndependent = ( + isFailureTestIndependent: unknown, + filepath: string +): RegExp[] | IsFailureTestIndependentFn => { + if (typeof isFailureTestIndependent === "function") { + return isFailureTestIndependent as IsFailureTestIndependentFn; + } else if (Array.isArray(isFailureTestIndependent)) { + return isFailureTestIndependent.map((entry) => { + if (typeof entry === "string") { + try { + return new RegExp(entry); + } catch (e: unknown) { + throw new Error( + `Invalid \`__unstableIsFailureTestIndependent\` regex \`${util.format( + entry + )}\` found in ${filepath}: ${util.inspect(e)}` + ); + } + } else if (entry instanceof RegExp) { + return entry; + } else { + throw new Error( + `Unexpected \`__unstableIsFailureTestIndependent\` value \`${util.format( + entry + )}\` found in ${filepath}` + ); + } + }); + } else if (typeof isFailureTestIndependent === "string") { + try { + return [new RegExp(isFailureTestIndependent)]; + } catch (e: unknown) { + throw new Error( + `Invalid \`__unstableIsFailureTestIndependent\` regex \`${util.format( + isFailureTestIndependent + )}\` found in ${filepath}: ${util.inspect(e)}` + ); + } + } else if (isFailureTestIndependent instanceof RegExp) { + return [isFailureTestIndependent]; + } else { + throw new Error( + `Unexpected \`__unstableIsFailureTestIndependent\` value \`${util.format( + isFailureTestIndependent + )}\` found in ${filepath}` + ); + } +}; + +export const loadConfig = (searchFrom: string): UnflakableJestConfig => + loadConfigSync( + searchFrom, + (configResult): [UnflakableJestConfigInner, string[]] => { + const config = + configResult !== null && typeof configResult.config === "object" + ? (configResult.config as { [s: string]: unknown }) + : null; + if ( + configResult !== null && + config?.__unstableIsFailureTestIndependent !== undefined + ) { + if (semverLt(jestPackage.version, "28.0.0")) { + throw new Error( + "__unstableIsFailureTestIndependent requires Jest version 28+" + ); + } + return [ + { + isFailureTestIndependent: parseIsFailureTestIndependent( + config.__unstableIsFailureTestIndependent, + configResult.filepath + ), + }, + ["__unstableIsFailureTestIndependent"], + ]; + } else { + return [{}, []]; + } + } + ); diff --git a/packages/jest-plugin/src/reporter.ts b/packages/jest-plugin/src/reporter.ts index 8d4d8de..89e0ae9 100644 --- a/packages/jest-plugin/src/reporter.ts +++ b/packages/jest-plugin/src/reporter.ts @@ -24,6 +24,7 @@ import { UnflakableAggregatedResult, UnflakableAggregatedResultWithCounts, UnflakableAssertionResult, + UnflakableJestConfig, UnflakableTestResult, UnflakableTestResultWithCounts, } from "./types"; @@ -41,12 +42,11 @@ import { commitOverride, getRepoRoot, loadApiKey, - loadConfigSync, loadGitRepo, toPosix, - UnflakableConfig, UnflakableConfigEnabled, } from "@unflakable/plugins-common"; +import { loadConfig } from "./config"; import { addResult, makeEmptyAggregatedTestResult } from "@jest/test-result"; const debug = _debug("unflakable:reporter"); @@ -99,8 +99,8 @@ const getIcon = (test: UnflakableAssertionResult): string => { } }; -// Recomputes the aggregated stats after taking into account retries, flakes, and quarantined -// failures. This function returns new objects and does NOT modify its input. +// Recomputes the aggregated stats after taking into account retries, flakes, quarantine, and +// test-independent failures. This function returns new objects and does NOT modify its input. const computeResultsForReporter = ( origAggregatedResults: UnflakableAggregatedResult ): UnflakableAggregatedResultWithCounts => { @@ -120,7 +120,7 @@ const computeResultsForReporter = ( ); // Only includes the first attempt of each test file, but the stats take into account subsequent - // attempts to determine flakiness. + // attempts to determine flakiness and test-independence. const updatedTestResults: UnflakableTestResultWithCounts[] = origAggregatedResults.testResults .filter((testResult) => (testResult._unflakableAttempt ?? 0) === 0) @@ -132,6 +132,7 @@ const computeResultsForReporter = ( numFailingTests, numFlakyTests, numPassingTests, + numPassingTestsWithIndependentFailures, numQuarantinedTests, } = testResult.testResults.reduce( ( @@ -139,6 +140,7 @@ const computeResultsForReporter = ( numFailingTests, numFlakyTests, numPassingTests, + numPassingTestsWithIndependentFailures, numQuarantinedTests, }, assertionResult @@ -151,7 +153,16 @@ const computeResultsForReporter = ( attempts.some((attempt) => attempt.status === "passed") && attempts.every( (attempt) => - attempt.status === "passed" || attempt.status === "pending" + attempt.status === "passed" || + attempt.status === "pending" || + attempt._unflakableIsFailureTestIndependent === true + ); + const isPassingWithTestIndependentFailures = + isPassing && + attempts.some( + (attempt) => + attempt.status === "failed" && + attempt._unflakableIsFailureTestIndependent === true ); const isQuarantined = !isPassing && @@ -166,6 +177,7 @@ const computeResultsForReporter = ( attempts.some( (attempt) => attempt.status === "failed" && + attempt._unflakableIsFailureTestIndependent !== true && attempt._unflakableIsQuarantined !== true ); const isFailing = @@ -179,6 +191,9 @@ const computeResultsForReporter = ( numFailingTests: numFailingTests + (isFailing ? 1 : 0), numFlakyTests: numFlakyTests + (isFlaky ? 1 : 0), numPassingTests: numPassingTests + (isPassing ? 1 : 0), + numPassingTestsWithIndependentFailures: + numPassingTestsWithIndependentFailures + + (isPassingWithTestIndependentFailures ? 1 : 0), numQuarantinedTests: numQuarantinedTests + (isQuarantined ? 1 : 0), }; @@ -187,6 +202,7 @@ const computeResultsForReporter = ( numFailingTests: 0, numFlakyTests: 0, numPassingTests: 0, + numPassingTestsWithIndependentFailures: 0, numQuarantinedTests: 0, } ); @@ -197,6 +213,8 @@ const computeResultsForReporter = ( numPassingTests, _unflakableNumFlakyTests: numFlakyTests, _unflakableNumQuarantinedTests: numQuarantinedTests, + _unflakableNumPassingTestsWithIndependentFailures: + numPassingTestsWithIndependentFailures, }; }); @@ -213,9 +231,12 @@ const computeResultsForReporter = ( _unflakableNumFlakyTests: 0, _unflakableNumQuarantinedTests: 0, _unflakableNumQuarantinedSuites: 0, + _unflakableNumPassedTestsWithIndependentFailures: 0, + _unflakableNumPassedTestSuitesWithIndependentFailures: 0, }; return updatedTestResults.reduce((aggregatedResults, testResult) => { + const prevNumPassedTestSuites = aggregatedResults.numPassedTestSuites; addResult(aggregatedResults, testResult); aggregatedResults.numTotalTests += @@ -227,20 +248,23 @@ const computeResultsForReporter = ( aggregatedResults._unflakableNumQuarantinedTests += testResult._unflakableNumQuarantinedTests; - // Handle edge cases that onResult() considers a suite pass but that should be failed or - // quarantined: - // https://github.com/jestjs/jest/blob/6d2632adae0f0fa1fe116d3b475fd9783d0de1b5/packages/jest-test-result/src/helpers.ts#L110 - if ( - !testResult.skipped && - testResult.testExecError === undefined && - testResult.numFailingTests === 0 - ) { + aggregatedResults._unflakableNumPassedTestsWithIndependentFailures += + testResult._unflakableNumPassingTestsWithIndependentFailures; + + if (aggregatedResults.numPassedTestSuites > prevNumPassedTestSuites) { + // Handle edge cases that onResult() considers a suite pass but that should be failed or + // quarantined: + // https://github.com/jestjs/jest/blob/6d2632adae0f0fa1fe116d3b475fd9783d0de1b5/packages/jest-test-result/src/helpers.ts#L110 if (testResult._unflakableNumFlakyTests > 0) { aggregatedResults.numFailedTestSuites += 1; aggregatedResults.numPassedTestSuites -= 1; } else if (testResult._unflakableNumQuarantinedTests > 0) { aggregatedResults._unflakableNumQuarantinedSuites += 1; aggregatedResults.numPassedTestSuites -= 1; + } else if ( + testResult._unflakableNumPassingTestsWithIndependentFailures > 0 + ) { + aggregatedResults._unflakableNumPassedTestSuitesWithIndependentFailures++; } } @@ -250,7 +274,7 @@ const computeResultsForReporter = ( export default class UnflakableReporter extends BaseReporter { private readonly apiKey: string; - private readonly unflakableConfig: UnflakableConfig; + private readonly unflakableConfig: UnflakableJestConfig; private readonly rootDir: string; private readonly defaultReporter: DefaultReporter & { @@ -263,7 +287,7 @@ export default class UnflakableReporter extends BaseReporter { debug("constructor"); super(); this.rootDir = globalConfig.rootDir; - this.unflakableConfig = loadConfigSync(globalConfig.rootDir); + this.unflakableConfig = loadConfig(globalConfig.rootDir); this.apiKey = this.unflakableConfig.enabled ? loadApiKey() : ""; if (globalConfig.verbose === true) { @@ -307,6 +331,9 @@ export default class UnflakableReporter extends BaseReporter { " " + chalk.dim( assertionResult.title + + (assertionResult._unflakableIsFailureTestIndependent === true + ? chalk.red(" [test independent]") + : "") + (assertionResult._unflakableIsQuarantined === true ? chalk.yellow(" [quarantined]") : "") + @@ -424,6 +451,7 @@ export default class UnflakableReporter extends BaseReporter { ).length, // We don't know when tests are flaky until the end. _unflakableNumFlakyTests: 0, + _unflakableNumPassingTestsWithIndependentFailures: 0, _unflakableNumQuarantinedTests: testResult.testResults.filter( (attempt) => attempt.status === "failed" && @@ -521,6 +549,12 @@ export default class UnflakableReporter extends BaseReporter { ? Math.floor(testResult.duration) : undefined, result: result as NonNullable, + ...((result === "fail" || result === "quarantined") && + testResult._unflakableIsFailureTestIndependent === true + ? { + failure_reason: "independent", + } + : {}), })), }) ) diff --git a/packages/jest-plugin/src/runner.ts b/packages/jest-plugin/src/runner.ts index 0aac0eb..7097f01 100644 --- a/packages/jest-plugin/src/runner.ts +++ b/packages/jest-plugin/src/runner.ts @@ -1,11 +1,7 @@ // Copyright (c) 2022-2023 Developer Innovations, LLC import * as path from "path"; -import type { - AssertionResult, - SerializableError, - TestResult, -} from "@jest/test-result"; +import type { SerializableError, TestResult } from "@jest/test-result"; import { FAILED, groupBy, testKey, USER_AGENT } from "./utils"; import TestRunner, { OnTestFailure, @@ -20,7 +16,11 @@ import { TEST_NAME_ENTRY_MAX_LENGTH, TestSuiteManifest, } from "@unflakable/js-api"; -import { UnflakableAssertionResult, UnflakableTestResult } from "./types"; +import { + UnflakableAssertionResult, + UnflakableJestConfig, + UnflakableTestResult, +} from "./types"; import type { Config } from "@jest/types"; import chalk from "chalk"; import escapeStringRegexp from "escape-string-regexp"; @@ -30,25 +30,16 @@ import { getTestSuiteManifest, isTestQuarantined, loadApiKey, - loadConfigSync, loadGitRepo, toPosix, - UnflakableConfig, } from "@unflakable/plugins-common"; +import { TestEvents, UnsubscribeFn } from "jest-circus/runner"; +import { loadConfig } from "./config"; +import util from "util"; const debug = _debug("unflakable:runner"); -// Exported by newer Jest versions but not older ones prior to 26.2.0. -export declare type TestEvents = { - "test-file-start": [Test]; - "test-file-success": [Test, TestResult]; - "test-file-failure": [Test, SerializableError]; - "test-case-result": [string, AssertionResult]; -}; - -export declare type UnsubscribeFn = () => void; - -type TestFailure = { test: Test; testResult: TestResult }; +type TestFailure = { test: Test; testResult: UnflakableTestResult }; class UnflakableRunner { readonly supportsEventEmitters = true; @@ -56,7 +47,14 @@ class UnflakableRunner { private readonly context?: TestRunnerContext; private readonly globalConfig: Config.GlobalConfig; private readonly manifest: Promise; - private readonly unflakableConfig: UnflakableConfig; + private readonly unflakableConfig: UnflakableJestConfig; + + private readonly capturedOutput: { + [key in string]: { + stderr: string; + stdout: string; + }; + } = {}; private testEventHandlers: { [key in keyof TestEvents]?: (( @@ -66,7 +64,7 @@ class UnflakableRunner { constructor(globalConfig: Config.GlobalConfig, context?: TestRunnerContext) { debug("constructor"); - this.unflakableConfig = loadConfigSync(globalConfig.rootDir); + this.unflakableConfig = loadConfig(globalConfig.rootDir); const testSuiteId = this.unflakableConfig.enabled ? this.unflakableConfig.testSuiteId @@ -122,63 +120,128 @@ class UnflakableRunner { }; } + private async isFailureTestIndependent( + testFilePath: string, + assertionResult: UnflakableAssertionResult + ): Promise { + if (typeof this.unflakableConfig.isFailureTestIndependent === "function") { + return this.unflakableConfig.isFailureTestIndependent({ + failure: assertionResult.failureMessages.join("\n"), + stdout: assertionResult._unflakableCapturedStdout ?? "", + stderr: assertionResult._unflakableCapturedStderr ?? "", + testFilePath, + testName: [...assertionResult.ancestorTitles, assertionResult.title], + }); + } else if (Array.isArray(this.unflakableConfig.isFailureTestIndependent)) { + return this.unflakableConfig.isFailureTestIndependent.some( + (regex) => + regex.test(assertionResult.failureMessages.join("\n")) || + regex.test(assertionResult._unflakableCapturedStdout ?? "") || + regex.test(assertionResult._unflakableCapturedStderr ?? "") + ); + } + return false; + } + // Called after each test *file* runs successfully (which may include failed tests, but the test // file itself didn't throw any errors when it was loaded). This function modifies // `testResult.testResults` by adding our own fields, and returns an updated // `UnflakableTestResult` that also includes some of our own fields. In the case of retries, we // also clear stats that would otherwise result in double-counted tests being emitted by the // SummaryReporter. - private onResult( + private async onResult( attempt: number, manifest: TestSuiteManifest | undefined, repoRoot: string, testsToRetry: TestFailure[], test: Test, testResult: TestResult - ): void { + ): Promise { debug(`onResult attempt=${attempt} path=\`${test.path}\``); - testResult.testResults.forEach( - (assertionResult: UnflakableAssertionResult): void => { - const testFilename = toPosix(path.relative(repoRoot, test.path)); - if (assertionResult.status === FAILED) { - if (manifest === undefined) { - debug( - "Not quarantining test failure due to failure to fetch manifest" - ); - } else if (this.unflakableConfig.quarantineMode === "no_quarantine") { - debug( - "Not quarantining test failure because quarantineMode is set to `no_quarantine`" - ); - } else { - const isQuarantined = isTestQuarantined( - manifest, - testFilename, - testKey(assertionResult, true) - ); - debug( - `Test is ${ - isQuarantined ? "" : "NOT " - }quarantined: ${JSON.stringify( - testKey(assertionResult, false) - )} in file ${testFilename}` - ); + await Promise.all( + testResult.testResults.map( + async (assertionResult: UnflakableAssertionResult): Promise => { + const testFilename = toPosix(path.relative(repoRoot, test.path)); + + const key = JSON.stringify(testKey(assertionResult, false)); + if (this.capturedOutput[key] !== undefined) { + assertionResult._unflakableCapturedStderr = + this.capturedOutput[key].stderr; + assertionResult._unflakableCapturedStdout = + this.capturedOutput[key].stdout; + } + + delete this.capturedOutput[key]; + + if (assertionResult.status === FAILED) { + try { + assertionResult._unflakableIsFailureTestIndependent = + await this.isFailureTestIndependent( + testResult.testFilePath, + assertionResult + ); + debug( + `Failure is${ + assertionResult._unflakableIsFailureTestIndependent === true + ? "" + : " not" + } test independent` + ); + } catch (e) { + process.stderr.write( + chalk.red( + `ERROR: Failed to evaluate isFailureTestIndependent: ${util.inspect( + e + )}\n` + ) + ); + } + + if (manifest === undefined) { + debug( + "Not quarantining test failure due to failure to fetch manifest" + ); + } else if ( + this.unflakableConfig.quarantineMode === "no_quarantine" + ) { + debug( + "Not quarantining test failure because quarantineMode is set to `no_quarantine`" + ); + } else { + const isQuarantined = isTestQuarantined( + manifest, + testFilename, + testKey(assertionResult, true) + ); + debug( + `Test is ${ + isQuarantined ? "" : "NOT " + }quarantined: ${JSON.stringify( + testKey(assertionResult, false) + )} in file ${testFilename}` + ); - // Use a separate field instead of adding a new `status` to avoid confusing third- - // party code that consumes the `Status` enum. - assertionResult._unflakableIsQuarantined = isQuarantined; + // Use a separate field instead of adding a new `status` to avoid confusing third- + // party code that consumes the `Status` enum. + assertionResult._unflakableIsQuarantined = isQuarantined; + } } } - } + ) ); + // We don't treat test-independent failures as failing tests at this point because that would + // cause Jest to terminate with a non-zero exit code, and we don't know yet if any subsequent + // attempts will pass. const numFailingTests = testResult.testResults.filter( (assertionResult: UnflakableAssertionResult) => assertionResult.status === FAILED && - assertionResult._unflakableIsQuarantined !== true + assertionResult._unflakableIsQuarantined !== true && + assertionResult._unflakableIsFailureTestIndependent !== true ).length; - // We retry any type of failure, including quarantined failures. + // We retry any type of failure, including quarantined and test-independent failures. if ( testResult.testResults.some( (assertionResult) => assertionResult.status === FAILED @@ -264,14 +327,25 @@ class UnflakableRunner { testsToRetry.length !== 0 && attempt < attempts; attempt++ ) { + const numTestsToRetry = testsToRetry.reduce( + (count, { testResult }) => + count + + testResult.testResults.filter( + // NB: We retry all failed tests, but quarantined and test-independent failures + // aren't counted in numFailingTests. + (assertion) => assertion.status === "failed" + ).length, + 0 + ); process.stderr.write( chalk.stderr.yellow.bold( - `Retrying ${testsToRetry.reduce( - (count, { testResult }) => count + testResult.numFailingTests, - 0 - )} failed test(s) from ${testsToRetry.length} file(s) -- ${ - attempts - attempt - 1 - } ${attempts - attempt - 1 === 1 ? "retry" : "retries"} remaining` + `Retrying ${numTestsToRetry} failed test${ + numTestsToRetry === 1 ? "" : "s" + } from ${testsToRetry.length} file${ + testsToRetry.length === 1 ? "" : "s" + } -- ${attempts - attempt - 1} ${ + attempts - attempt - 1 === 1 ? "retry" : "retries" + } remaining` ) + "\n" ); @@ -327,7 +401,8 @@ class UnflakableRunner { } } - // Returns an array of tests that should be retried, which includes quarantined failures. + // Returns an array of tests that should be retried, which includes both quarantined and + // test-independent failures. private async runTestsImpl( tests: Test[], watcher: TestWatcher, @@ -355,7 +430,7 @@ class UnflakableRunner { onResult !== undefined && this.unflakableConfig.enabled ? async (test: Test, result: TestResult): Promise => { // NB: We call this first because it modifies `result`. - this.onResult( + await this.onResult( attempt, manifest, repoRoot, @@ -411,6 +486,33 @@ class UnflakableRunner { ), ] : []), + ...(supportsEventEmitters === true + ? [ + eventEmittingTestRunner.on( + "test-case-result", + ([testPath, assertionResult]: [ + string, + UnflakableAssertionResult + ]): void => { + debug( + `on(test-case-result) path=\`${testPath}\` title=%o status=%o`, + [ + ...assertionResult.ancestorTitles, + assertionResult.title, + ], + assertionResult.status + ); + + this.capturedOutput[ + JSON.stringify(testKey(assertionResult, false)) + ] = { + stdout: assertionResult._unflakableCapturedStdout ?? "", + stderr: assertionResult._unflakableCapturedStderr ?? "", + }; + } + ), + ] + : []), ...Object.entries(this.testEventHandlers).flatMap( ([eventName, listeners]) => listeners.map((listener) => { @@ -420,12 +522,12 @@ class UnflakableRunner { ) { return eventEmittingTestRunner.on( "test-file-success", - ([ + async ([ test, result, - ]: TestEvents["test-file-success"]): void | Promise => { + ]: TestEvents["test-file-success"]): Promise => { // NB: We call this first because it modifies `result`. - this.onResult( + await this.onResult( attempt, manifest, repoRoot, diff --git a/packages/jest-plugin/src/test-runner.ts b/packages/jest-plugin/src/test-runner.ts new file mode 100644 index 0000000..1884aaf --- /dev/null +++ b/packages/jest-plugin/src/test-runner.ts @@ -0,0 +1,87 @@ +// Copyright (c) 2023 Developer Innovations, LLC + +import { TestEvents, TestFileEvent, TestResult } from "@jest/test-result"; +import { JestEnvironment } from "@jest/environment"; +import { Config } from "@jest/types"; +import Runtime from "jest-runtime"; +import circus from "jest-circus/runner"; +import { debug as _debug } from "debug"; +import util from "util"; +import { UnflakableAssertionResult } from "./types"; + +const baseStderrWrite = process.stderr.write.bind(process.stderr); +const baseStdoutWrite = process.stdout.write.bind(process.stdout); + +const debug = _debug("unflakable:test-runner"); +// Don't capture our own debug output as test output. +debug.log = (...args: unknown[]): boolean => + baseStderrWrite(util.format(...args) + "\n"); + +const write = + (base: NodeJS.WriteStream["write"], capture: (buf: string) => void) => + ( + buffer: Uint8Array | string, + encodingOrCb?: BufferEncoding | ((err?: Error) => void), + cb?: (err?: Error) => void + ): boolean => { + capture( + typeof buffer === "string" + ? buffer + : typeof encodingOrCb === "string" + ? new util.TextDecoder(encodingOrCb).decode(buffer) + : buffer.toString() + ); + return base(buffer, encodingOrCb as BufferEncoding | undefined, cb); + }; + +export default ( + globalConfig: Config.GlobalConfig, + config: Config.ProjectConfig, + environment: JestEnvironment, + runtime: Runtime, + testPath: string, + sendMessageToJest?: TestFileEvent +): Promise => { + debug(`initialize pid=${process.pid} testPath=\`${testPath}\``); + + let capturedStderr = ""; + let capturedStdout = ""; + + process.stderr.write = write( + baseStderrWrite, + (buf) => (capturedStderr += buf) + ); + process.stdout.write = write( + baseStdoutWrite, + (buf) => (capturedStdout += buf) + ); + + // TODO: Support other test runners? + return circus( + globalConfig, + config, + environment, + runtime, + testPath, + sendMessageToJest !== undefined + ? ( + eventName: T, + args: TestEvents[T] + ): unknown => { + debug( + `sendMessageToJest pid=${process.pid} testPath=\`${testPath}\` eventName=\`${eventName}\`` + ); + + if (eventName === "test-case-result") { + const result = args[1] as UnflakableAssertionResult; + result._unflakableCapturedStderr = capturedStderr; + result._unflakableCapturedStdout = capturedStdout; + capturedStdout = ""; + capturedStderr = ""; + } + + return sendMessageToJest(eventName, args); + } + : undefined + ); +}; diff --git a/packages/jest-plugin/src/types.ts b/packages/jest-plugin/src/types.ts index a1d007b..849a5da 100644 --- a/packages/jest-plugin/src/types.ts +++ b/packages/jest-plugin/src/types.ts @@ -5,8 +5,33 @@ import type { AssertionResult, TestResult, } from "@jest/test-result"; +import { + UnflakableConfig, + UnflakableConfigEnabled, +} from "@unflakable/plugins-common"; + +export type IsFailureTestIndependentFn = (args: { + failure: string; + stderr: string; + stdout: string; + testFilePath: string; + testName: string[]; +}) => boolean | Promise; + +export type UnflakableJestConfigInner = { + isFailureTestIndependent?: RegExp[] | IsFailureTestIndependentFn; + __unstableIsFailureTestIndependent?: undefined; +}; + +export type UnflakableJestConfig = UnflakableConfig & UnflakableJestConfigInner; + +export type UnflakableJestConfigEnabled = UnflakableConfigEnabled & + UnflakableJestConfigInner; export type UnflakableAssertionResult = AssertionResult & { + _unflakableCapturedStderr?: string; + _unflakableCapturedStdout?: string; + _unflakableIsFailureTestIndependent?: boolean; _unflakableIsQuarantined?: boolean; }; @@ -19,6 +44,9 @@ export type UnflakableTestResult = Omit & { export type UnflakableTestResultWithCounts = UnflakableTestResult & { _unflakableNumQuarantinedTests: number; _unflakableNumFlakyTests: number; + // This represents a *subset* of numPassingTests and should not be added to it, or we'll be + // double-counting passes. + _unflakableNumPassingTestsWithIndependentFailures: number; }; export type UnflakableAggregatedResult = Omit< @@ -36,5 +64,11 @@ export type UnflakableAggregatedResultWithCounts = Omit< _unflakableNumQuarantinedTests: number; _unflakableNumQuarantinedSuites: number; + // These represent *subsets* of numPassedTests/numPassedTestSuites and should not be added to + // those values. Rather, they are intended to be consumed by the SummaryReporter to show which + // tests/suites would have failed were it not for the failures being test-independent. + _unflakableNumPassedTestsWithIndependentFailures: number; + _unflakableNumPassedTestSuitesWithIndependentFailures: number; + testResults: UnflakableTestResultWithCounts[]; }; diff --git a/packages/jest-plugin/src/vendored/getSummary.ts b/packages/jest-plugin/src/vendored/getSummary.ts index 51547af..8422414 100644 --- a/packages/jest-plugin/src/vendored/getSummary.ts +++ b/packages/jest-plugin/src/vendored/getSummary.ts @@ -93,6 +93,8 @@ export const getSummary = ( const snapshotsUpdated = snapshotResults.updated; const suitesFailed = aggregatedResults.numFailedTestSuites; const suitesPassed = aggregatedResults.numPassedTestSuites; + const suitesPassedWithIndependentFailures = + aggregatedResults._unflakableNumPassedTestSuitesWithIndependentFailures; const suitesPending = aggregatedResults.numPendingTestSuites; const suitesQuarantined = aggregatedResults._unflakableNumQuarantinedSuites; const suitesRun = suitesFailed + suitesPassed + suitesQuarantined; @@ -100,6 +102,8 @@ export const getSummary = ( const testsFailed = aggregatedResults.numFailedTests; const testsFlaky = aggregatedResults._unflakableNumFlakyTests; const testsPassed = aggregatedResults.numPassedTests; + const testsPassedWithIndependentFailures = + aggregatedResults._unflakableNumPassedTestsWithIndependentFailures; const testsPending = aggregatedResults.numPendingTests; const testsQuarantined = aggregatedResults._unflakableNumQuarantinedTests; const testsTodo = aggregatedResults.numTodoTests; @@ -117,7 +121,15 @@ export const getSummary = ( ? chalk.bold.yellow(`${suitesPending} skipped`) + ", " : "" }${ - suitesPassed > 0 ? chalk.bold.green(`${suitesPassed} passed`) + ", " : "" + suitesPassed > 0 + ? chalk.bold.green( + `${suitesPassed} passed${ + suitesPassedWithIndependentFailures > 0 + ? ` (${suitesPassedWithIndependentFailures} with test-independent failures)` + : "" + }` + ) + ", " + : "" }${ suitesRun !== suitesTotal ? `${suitesRun} of ${suitesTotal}` : suitesTotal } total`; @@ -135,7 +147,15 @@ export const getSummary = ( ? chalk.bold.yellow(`${testsPending} skipped`) + ", " : "") + (testsTodo > 0 ? chalk.bold.magenta(`${testsTodo} todo`) + ", " : "") + - (testsPassed > 0 ? chalk.bold.green(`${testsPassed} passed`) + ", " : "") + + (testsPassed > 0 + ? chalk.bold.green( + `${testsPassed} passed${ + testsPassedWithIndependentFailures > 0 + ? ` (${testsPassedWithIndependentFailures} with test-independent failures)` + : "" + }` + ) + ", " + : "") + `${testsTotal} total`; const snapshots = diff --git a/packages/jest-plugin/test/integration-input/jest.config.js b/packages/jest-plugin/test/integration-input/jest.config.js index 190d8e6..e23590c 100644 --- a/packages/jest-plugin/test/integration-input/jest.config.js +++ b/packages/jest-plugin/test/integration-input/jest.config.js @@ -2,7 +2,7 @@ module.exports = { clearMocks: true, - maxWorkers: 2, + // maxWorkers: 2, // Default changed in Jest 29 (see // https://github.com/facebook/jest/blob/94c06ef0aa9b327f3c400610b861e7308b29ee0d/docs/UpgradingToJest29.md). diff --git a/packages/jest-plugin/test/integration-input/src/fail.test.ts b/packages/jest-plugin/test/integration-input/src/fail.test.ts index d89a4b0..cf69151 100644 --- a/packages/jest-plugin/test/integration-input/src/fail.test.ts +++ b/packages/jest-plugin/test/integration-input/src/fail.test.ts @@ -6,6 +6,8 @@ describe("describe block", () => { // properly. "should ([escape regex]?.*$ fail", () => { + process.stderr.write("fail stderr\n"); + process.stdout.write("fail stdout\n"); if (process.env.TEST_SNAPSHOTS !== undefined) { expect({ foo: true }).toMatchInlineSnapshot(` Object { @@ -13,7 +15,7 @@ describe("describe block", () => { } `); } else { - throw new Error(); + throw new Error("test failed\nnew line"); } } ); diff --git a/packages/jest-plugin/test/integration-input/src/flake.test.ts b/packages/jest-plugin/test/integration-input/src/flake.test.ts index 0b0633b..6acbbbb 100644 --- a/packages/jest-plugin/test/integration-input/src/flake.test.ts +++ b/packages/jest-plugin/test/integration-input/src/flake.test.ts @@ -7,25 +7,26 @@ import fs from "fs/promises"; async (i) => { if (process.env.FLAKY_TEST_TEMP === undefined) { throw new Error("missing FLAKY_TEST_TEMP environment variable"); + } else if (process.env.FLAKE_FAIL_COUNT === undefined) { + throw new Error("missing FLAKE_FAIL_COUNT environment variable"); } const tempFilePath = `${process.env.FLAKY_TEST_TEMP}${i}`; - // We can't maintain in-memory state between test tries, so we write to a temp file to indicate - // that it's not the first attempt. - const exists = await fs - .stat(tempFilePath) - .then(() => true) - .catch(() => false); - await fs.writeFile(tempFilePath, ""); + // We can't maintain in-memory state between test tries, so we write to a temp file. + const attempt = await fs + .readFile(tempFilePath, { encoding: "utf8" }) + .then(Number.parseInt) + .catch(() => 0); + await fs.writeFile(tempFilePath, (attempt + 1).toString()); if (process.env.TEST_SNAPSHOTS !== undefined) { - expect({ exists }).toMatchInlineSnapshot(` + expect({ exists: attempt > 0 }).toMatchInlineSnapshot(` Object { "exists": true, } `); - } else if (!exists) { - throw new Error("first try should fail"); + } else if (attempt < Number.parseInt(process.env.FLAKE_FAIL_COUNT)) { + throw new Error(`first try should fail`); } } ); diff --git a/packages/jest-plugin/test/integration-input/src/mixed.test.ts b/packages/jest-plugin/test/integration-input/src/mixed.test.ts index 16b867f..129635d 100644 --- a/packages/jest-plugin/test/integration-input/src/mixed.test.ts +++ b/packages/jest-plugin/test/integration-input/src/mixed.test.ts @@ -10,14 +10,15 @@ describe("mixed", () => { () => { if (process.env.TEST_SNAPSHOTS !== undefined) { // NB: If we include a description here, Jest treats the snapshot as obsolete if the test - // is skipped (and exits with a non-zero code) since it uses a string equality match instead + // is skipped (and exits with a non-zero code) since it uses a string equality match + // instead // of prefix check: // https://github.com/facebook/jest/blob/54eadb65a9f9ce789df6cf92df82cdbda68c0d4b/packages/jest-snapshot/src/State.ts#L99 expect({ foo: false }).toMatchSnapshot(); } else { // Have the snapshot pass here so that Jest doesn't treat it as obsolete. expect({ foo: true }).toMatchSnapshot(); - throw new Error(); + throw new Error("mixed quarantined test failed"); } } ); @@ -25,7 +26,9 @@ describe("mixed", () => { (process.env.SKIP_FAILURES !== undefined ? it.skip : it)( "mixed: should fail", () => { - throw new Error(); + process.stderr.write("mixed fail stderr\n"); + process.stdout.write("mixed fail stdout\n"); + throw new Error("mixed test failed\nnew line"); } ); diff --git a/packages/jest-plugin/test/integration-input/src/quarantined.test.ts b/packages/jest-plugin/test/integration-input/src/quarantined.test.ts index f8461e1..397bae1 100644 --- a/packages/jest-plugin/test/integration-input/src/quarantined.test.ts +++ b/packages/jest-plugin/test/integration-input/src/quarantined.test.ts @@ -4,7 +4,7 @@ describe("describe block", () => { (process.env.SKIP_QUARANTINED !== undefined ? it.skip : it)( "should be quarantined", () => { - throw new Error(); + throw new Error("quarantined test failed"); } ); }); diff --git a/packages/jest-plugin/test/integration/package.json b/packages/jest-plugin/test/integration/package.json index 494e9ae..c80c811 100644 --- a/packages/jest-plugin/test/integration/package.json +++ b/packages/jest-plugin/test/integration/package.json @@ -9,6 +9,7 @@ "@types/temp": "^0.9.1", "@unflakable/jest-plugin": "workspace:^", "@unflakable/js-api": "workspace:^", + "es6-promisify": "^7.0.0", "escape-string-regexp": "^4.0.0", "expect": "25.1.0 - 29", "jest": "25.1.0 - 29", @@ -16,7 +17,8 @@ "jest-get-type": "25.1.0 - 29", "jest-matcher-utils": "25.1.0 - 29", "mockttp": "^3.7.5", - "temp": "^0.9.4", + "semver": "^7.5.4", + "tmp": "^0.2.1", "typescript": "^4.9.5", "unflakable-test-common": "workspace:^" }, diff --git a/packages/jest-plugin/test/integration/src/basic.test.ts b/packages/jest-plugin/test/integration/src/basic.test.ts index eb98a3f..2f81bec 100644 --- a/packages/jest-plugin/test/integration/src/basic.test.ts +++ b/packages/jest-plugin/test/integration/src/basic.test.ts @@ -30,7 +30,9 @@ integrationTestSuite((mockBackend) => { failedTests: 2, flakyTests: 0, passedSuites: 1, + passedSuitesWithIndependentFailures: 0, passedTests: 2, + passedTestsWithIndependentFailures: 0, quarantinedSuites: 2, quarantinedTests: 4, skippedSuites: 0, @@ -56,7 +58,9 @@ integrationTestSuite((mockBackend) => { failedTests: 0, flakyTests: 2, passedSuites: 1, + passedSuitesWithIndependentFailures: 0, passedTests: 2, + passedTestsWithIndependentFailures: 0, quarantinedSuites: 2, quarantinedTests: 2, skippedSuites: 1, @@ -83,7 +87,9 @@ integrationTestSuite((mockBackend) => { failedTests: 0, flakyTests: 0, passedSuites: 1, + passedSuitesWithIndependentFailures: 0, passedTests: 2, + passedTestsWithIndependentFailures: 0, quarantinedSuites: 3, quarantinedTests: 4, skippedSuites: 1, diff --git a/packages/jest-plugin/test/integration/src/disable-plugin.test.ts b/packages/jest-plugin/test/integration/src/disable-plugin.test.ts index bd42b0e..5673309 100644 --- a/packages/jest-plugin/test/integration/src/disable-plugin.test.ts +++ b/packages/jest-plugin/test/integration/src/disable-plugin.test.ts @@ -22,7 +22,9 @@ integrationTestSuite((mockBackend) => { failedTests: 6, flakyTests: 0, passedSuites: 1, + passedSuitesWithIndependentFailures: 0, passedTests: 2, + passedTestsWithIndependentFailures: 0, quarantinedSuites: 0, quarantinedTests: 0, skippedSuites: 0, @@ -51,7 +53,9 @@ integrationTestSuite((mockBackend) => { failedTests: 6, flakyTests: 0, passedSuites: 1, + passedSuitesWithIndependentFailures: 0, passedTests: 2, + passedTestsWithIndependentFailures: 0, quarantinedSuites: 0, quarantinedTests: 0, skippedSuites: 0, diff --git a/packages/jest-plugin/test/integration/src/long-names.test.ts b/packages/jest-plugin/test/integration/src/long-names.test.ts index e2a4f66..27aa9d8 100644 --- a/packages/jest-plugin/test/integration/src/long-names.test.ts +++ b/packages/jest-plugin/test/integration/src/long-names.test.ts @@ -40,7 +40,9 @@ integrationTestSuite((mockBackend) => { failedTests: 2, flakyTests: 0, passedSuites: 1, + passedSuitesWithIndependentFailures: 0, passedTests: 2, + passedTestsWithIndependentFailures: 0, quarantinedSuites: 2, quarantinedTests: 4, skippedSuites: 0, diff --git a/packages/jest-plugin/test/integration/src/no-quarantine.test.ts b/packages/jest-plugin/test/integration/src/no-quarantine.test.ts index c397b18..97e735f 100644 --- a/packages/jest-plugin/test/integration/src/no-quarantine.test.ts +++ b/packages/jest-plugin/test/integration/src/no-quarantine.test.ts @@ -18,7 +18,9 @@ integrationTestSuite((mockBackend) => { failedTests: 4, flakyTests: 2, passedSuites: 1, + passedSuitesWithIndependentFailures: 0, passedTests: 2, + passedTestsWithIndependentFailures: 0, quarantinedSuites: 0, quarantinedTests: 0, skippedSuites: 0, diff --git a/packages/jest-plugin/test/integration/src/plugin-failures.test.ts b/packages/jest-plugin/test/integration/src/plugin-failures.test.ts index 3b2305c..af57524 100644 --- a/packages/jest-plugin/test/integration/src/plugin-failures.test.ts +++ b/packages/jest-plugin/test/integration/src/plugin-failures.test.ts @@ -22,7 +22,9 @@ integrationTestSuite((mockBackend) => { failedTests: 0, flakyTests: 0, passedSuites: 2, + passedSuitesWithIndependentFailures: 0, passedTests: 2, + passedTestsWithIndependentFailures: 0, quarantinedSuites: 0, quarantinedTests: 0, skippedSuites: 3, @@ -62,7 +64,9 @@ integrationTestSuite((mockBackend) => { failedTests: 4, flakyTests: 2, passedSuites: 1, + passedSuitesWithIndependentFailures: 0, passedTests: 2, + passedTestsWithIndependentFailures: 0, quarantinedSuites: 0, quarantinedTests: 0, skippedSuites: 0, diff --git a/packages/jest-plugin/test/integration/src/retries.test.ts b/packages/jest-plugin/test/integration/src/retries.test.ts index da85f1f..95f57e9 100644 --- a/packages/jest-plugin/test/integration/src/retries.test.ts +++ b/packages/jest-plugin/test/integration/src/retries.test.ts @@ -22,7 +22,9 @@ integrationTestSuite((mockBackend) => { failedTests: 4, flakyTests: 0, passedSuites: 1, + passedSuitesWithIndependentFailures: 0, passedTests: 2, + passedTestsWithIndependentFailures: 0, quarantinedSuites: 1, quarantinedTests: 2, skippedSuites: 0, diff --git a/packages/jest-plugin/test/integration/src/run-test-case.ts b/packages/jest-plugin/test/integration/src/run-test-case.ts index 6b6f112..fa869f3 100644 --- a/packages/jest-plugin/test/integration/src/run-test-case.ts +++ b/packages/jest-plugin/test/integration/src/run-test-case.ts @@ -1,10 +1,9 @@ // Copyright (c) 2022-2023 Developer Innovations, LLC -import * as temp from "temp"; +import { tmpName, TmpNameOptions } from "tmp"; import { CreateTestSuiteRunInlineRequest, TEST_NAME_ENTRY_MAX_LENGTH, - TestRunAttemptRecord, TestRunRecord, TestSuiteManifest, } from "@unflakable/js-api"; @@ -16,7 +15,7 @@ import { CosmiconfigMockParams, } from "unflakable-test-common/dist/config"; import path from "path"; -import { promisify } from "util"; +import { promisify } from "es6-promisify"; import { execFile } from "child_process"; import { MockBackend, @@ -28,6 +27,8 @@ import { AsyncTestError, spawnTestWithTimeout, } from "unflakable-test-common/dist/spawn"; +import * as util from "util"; +import * as fs from "fs/promises"; // Jest times out after 120 seconds, so we bail early here to allow time to print the // captured output before Jest kills the test. @@ -60,6 +61,7 @@ export type SimpleGitMockParams = export type TestCaseParams = { config: Partial | null; + configJs: string | null; envVars: { [key in string]: string | undefined }; expectedApiKey: string; expectedBranch: string | undefined; @@ -68,13 +70,20 @@ export type TestCaseParams = { expectedFlakeTestNameSuffix: string; expectedSuiteId: string; expectedRepoRelativePathPrefix: string; + expectFailuresToBeTestIndependent: boolean; + expectFailuresFirstAttemptToBeTestIndependent: boolean; + expectFlakeToBeTestIndependent: boolean; + expectFlakeFirstAttemptToBeTestIndependent: boolean; expectPluginToBeEnabled: boolean; expectResultsToBeUploaded: boolean; expectQuarantinedTestsToBeQuarantined: boolean; expectQuarantinedTestsToBeSkipped: boolean; + expectQuarantinedTestsToBeTestIndependent: boolean; + expectQuarantinedTestsFirstAttemptToBeTestIndependent: boolean; expectSnapshots: boolean; failToFetchManifest: boolean; failToUploadResults: boolean; + flakeFailCount: number; git: SimpleGitMockParams; quarantineFlake: boolean; skipFailures: boolean; @@ -92,7 +101,9 @@ const TIMESTAMP_REGEX = export type ResultCounts = { passedSuites: number; + passedSuitesWithIndependentFailures: number; passedTests: number; + passedTestsWithIndependentFailures: number; failedSuites: number; failedTests: number; flakyTests: number; @@ -111,13 +122,20 @@ const verifyUploadResults = ( request: CompletedRequest ): void => { const { + expectFailuresFirstAttemptToBeTestIndependent, + expectFailuresToBeTestIndependent, + expectFlakeFirstAttemptToBeTestIndependent, + expectFlakeToBeTestIndependent, + expectQuarantinedTestsFirstAttemptToBeTestIndependent, + expectQuarantinedTestsToBeQuarantined, + expectQuarantinedTestsToBeSkipped, + expectQuarantinedTestsToBeTestIndependent, expectedBranch, expectedCommit, expectedFailureRetries, expectedFlakeTestNameSuffix, - expectQuarantinedTestsToBeQuarantined, - expectQuarantinedTestsToBeSkipped, failToFetchManifest, + flakeFailCount, quarantineFlake, skipFailures, skipFlake, @@ -171,92 +189,70 @@ const verifyUploadResults = ( { filename: specRepoPath(params, "fail"), name: ["describe block", "should ([escape regex]?.*$ fail"], - attempts: Array( - expectedFailureRetries + 1 - ).fill({ - duration_ms: expect.any(Number) as number, - result: "fail", - }), + attempts: Array(expectedFailureRetries + 1) + .fill(0) + .map((_, idx) => ({ + duration_ms: expect.any(Number) as number, + result: "fail", + ...(expectFailuresToBeTestIndependent || + (expectFailuresFirstAttemptToBeTestIndependent && idx === 0) + ? { + failure_reason: "independent", + } + : {}), + })), }, ] as TestRunRecord[])), - ...(skipFlake || - (expectQuarantinedTestsToBeSkipped && - quarantineFlake && - !failToFetchManifest) || - (testNamePattern !== undefined && - `should be flaky 1${expectedFlakeTestNameSuffix}`.match( - testNamePattern - ) === null) - ? [] - : [ - { - filename: specRepoPath(params, "flake"), - name: [ - `should be flaky 1${expectedFlakeTestNameSuffix}`.substring( - 0, - TEST_NAME_ENTRY_MAX_LENGTH - ), - ], - attempts: [ - { - duration_ms: expect.any(Number) as number, - result: - quarantineFlake && - !failToFetchManifest && - expectQuarantinedTestsToBeQuarantined - ? "quarantined" - : "fail", - }, - ...(expectedFailureRetries > 0 - ? [ - { - duration_ms: expect.any(Number) as number, - result: "pass", - }, - ] - : []), - ], - } as TestRunRecord, - ]), - ...(skipFlake || - (expectQuarantinedTestsToBeSkipped && - quarantineFlake && - !failToFetchManifest) || - (testNamePattern !== undefined && - `should be flaky 2${expectedFlakeTestNameSuffix}`.match( - testNamePattern - ) === null) - ? [] - : [ - { - filename: specRepoPath(params, "flake"), - name: [ - `should be flaky 2${expectedFlakeTestNameSuffix}`.substring( - 0, - TEST_NAME_ENTRY_MAX_LENGTH - ), - ], - attempts: [ - { - duration_ms: expect.any(Number) as number, - result: - quarantineFlake && - !failToFetchManifest && - expectQuarantinedTestsToBeQuarantined - ? "quarantined" - : "fail", - }, - ...(expectedFailureRetries > 0 - ? [ - { - duration_ms: expect.any(Number) as number, - result: "pass", - }, - ] - : []), - ], - } as TestRunRecord, - ]), + ...[1, 2].flatMap((i) => + skipFlake || + (expectQuarantinedTestsToBeSkipped && + quarantineFlake && + !failToFetchManifest) || + (testNamePattern !== undefined && + `should be flaky ${i}${expectedFlakeTestNameSuffix}`.match( + testNamePattern + ) === null) + ? [] + : [ + { + filename: specRepoPath(params, "flake"), + name: [ + `should be flaky ${i}${expectedFlakeTestNameSuffix}`.substring( + 0, + TEST_NAME_ENTRY_MAX_LENGTH + ), + ], + attempts: [ + ...Array(flakeFailCount) + .fill(0) + .map((_, idx) => ({ + duration_ms: expect.any(Number) as number, + result: + quarantineFlake && + !failToFetchManifest && + expectQuarantinedTestsToBeQuarantined + ? "quarantined" + : "fail", + ...(expectFlakeToBeTestIndependent || + (expectFlakeFirstAttemptToBeTestIndependent && + idx === 0) + ? { + failure_reason: "independent", + } + : {}), + })), + ...(expectedFailureRetries > 0 + ? [ + { + duration_ms: expect.any(Number) as number, + result: "pass", + }, + ] + : []), + ], + } as TestRunRecord, + ] + ), ...(skipQuarantined || (expectQuarantinedTestsToBeSkipped && !failToFetchManifest) || (testNamePattern !== undefined && @@ -266,16 +262,23 @@ const verifyUploadResults = ( { filename: specRepoPath(params, "mixed"), name: ["mixed", "mixed: should be quarantined"], - attempts: Array( - expectedFailureRetries + 1 - ).fill({ - duration_ms: expect.any(Number) as number, - result: - failToFetchManifest || - !expectQuarantinedTestsToBeQuarantined - ? "fail" - : "quarantined", - }), + attempts: Array(expectedFailureRetries + 1) + .fill(0) + .map((_, idx) => ({ + duration_ms: expect.any(Number) as number, + result: + failToFetchManifest || + !expectQuarantinedTestsToBeQuarantined + ? "fail" + : "quarantined", + ...(expectQuarantinedTestsToBeTestIndependent || + (expectQuarantinedTestsFirstAttemptToBeTestIndependent && + idx === 0) + ? { + failure_reason: "independent", + } + : {}), + })), }, ] as TestRunRecord[])), ...(skipFailures || @@ -286,12 +289,18 @@ const verifyUploadResults = ( { filename: specRepoPath(params, "mixed"), name: ["mixed", "mixed: should fail"], - attempts: Array( - expectedFailureRetries + 1 - ).fill({ - duration_ms: expect.any(Number) as number, - result: "fail", - }), + attempts: Array(expectedFailureRetries + 1) + .fill(0) + .map((_, idx) => ({ + duration_ms: expect.any(Number) as number, + result: "fail", + ...(expectFailuresToBeTestIndependent || + (expectFailuresFirstAttemptToBeTestIndependent && idx === 0) + ? { + failure_reason: "independent", + } + : {}), + })), }, ] as TestRunRecord[])), ...(testNamePattern === undefined || @@ -334,16 +343,23 @@ const verifyUploadResults = ( { filename: specRepoPath(params, "quarantined"), name: ["describe block", "should be quarantined"], - attempts: Array( - expectedFailureRetries + 1 - ).fill({ - duration_ms: expect.any(Number) as number, - result: - failToFetchManifest || - !expectQuarantinedTestsToBeQuarantined - ? "fail" - : "quarantined", - }), + attempts: Array(expectedFailureRetries + 1) + .fill(0) + .map((_, idx) => ({ + duration_ms: expect.any(Number) as number, + result: + failToFetchManifest || + !expectQuarantinedTestsToBeQuarantined + ? "fail" + : "quarantined", + ...(expectQuarantinedTestsToBeTestIndependent || + (expectQuarantinedTestsFirstAttemptToBeTestIndependent && + idx === 0) + ? { + failure_reason: "independent", + } + : {}), + })), }, ] as TestRunRecord[])), ].filter( @@ -456,8 +472,14 @@ export const runTestCase = async ( expectedResults: ResultCounts, mockBackend: MockBackend ): Promise => { - const { git, skipFailures, skipFlake, skipQuarantined, testNamePattern } = - params; + const { + flakeFailCount, + git, + skipFailures, + skipFlake, + skipQuarantined, + testNamePattern, + } = params; const asyncTestError: AsyncTestError = { error: undefined }; @@ -474,30 +496,49 @@ export const runTestCase = async ( } ); + const configPath = + params.configJs !== null + ? (await promisify(tmpName)({ + prefix: "unflakable-jest-integration-config", + })) + ".js" + : null; + if (configPath !== null) { + await fs.writeFile( + configPath, + Buffer.from(params.configJs as string, "utf8") + ); + } + const integrationInputPath = path.join("..", "integration-input"); - const configMockParams: CosmiconfigMockParams = { - searchFrom: path.resolve(integrationInputPath), - searchResult: - params.config !== null - ? { - config: params.config, - filepath: path.join( - "MOCK_BASE", - "packages", - "jest-plugin", - "test", - "integration-input", - "package.json" - ), - } - : null, - }; + const configMockParams: CosmiconfigMockParams = + configPath !== null + ? { + expectedSearchFrom: path.resolve(integrationInputPath), + pathToLoad: configPath, + } + : { + expectedSearchFrom: path.resolve(integrationInputPath), + searchResult: + params.config !== null + ? { + config: params.config, + filepath: path.join( + "MOCK_BASE", + "packages", + "jest-plugin", + "test", + "integration-input", + "package.json" + ), + } + : null, + }; // We don't directly invoke `jest` because we need to pass `--require` to Node.JS in order to // mock cosmiconfig for testing. Instead, we resolve the binary to an absolute path using `yarn // bin` and then invoke node directly. const jestBin = ( - await promisify(execFile)("yarn", ["bin", "jest"], { + await util.promisify(execFile)("yarn", ["bin", "jest"], { cwd: integrationInputPath, // yarn.CMD isn't executable without a shell on Windows. shell: process.platform === "win32", @@ -514,13 +555,12 @@ export const runTestCase = async ( // Uncomment to enable debugger. // "--inspect", jestBin, - // --no-cache disables the cache that stores the past timings, which makes the output - // non-deterministic since it gets bolded if it exceeds the expected time. - "--no-cache", "--reporters", "@unflakable/jest-plugin/dist/reporter", "--runner", "@unflakable/jest-plugin/dist/runner", + "--testRunner", + "@unflakable/jest-plugin/dist/test-runner", ...(skipFailures ? ["--testPathIgnorePatterns", "/src/invalid\\.test\\.ts"] : []), @@ -533,7 +573,9 @@ export const runTestCase = async ( ...params.envVars, DEBUG: process.env.TEST_DEBUG, // The flaky test needs external state to know when it's being retried so that it can pass. - FLAKY_TEST_TEMP: temp.path(), + FLAKY_TEST_TEMP: await promisify(tmpName)({ + prefix: "unflakable-jest-integration-flaky-test", + }), PATH: process.env.PATH, UNFLAKABLE_API_BASE_URL: `http://localhost:${mockBackend.apiServerPort}`, [CONFIG_MOCK_ENV_VAR]: JSON.stringify(configMockParams), @@ -541,6 +583,7 @@ export const runTestCase = async ( ...(skipFailures ? { SKIP_FAILURES: "1" } : {}), ...(skipFlake ? { SKIP_FLAKE: "1" } : {}), ...(skipQuarantined ? { SKIP_QUARANTINED: "1" } : {}), + FLAKE_FAIL_COUNT: flakeFailCount.toString(), // Windows requires these environment variables to be propagated. ...(process.platform === "win32" ? { diff --git a/packages/jest-plugin/test/integration/src/skip-tests.test.ts b/packages/jest-plugin/test/integration/src/skip-tests.test.ts index 1302612..b122fbf 100644 --- a/packages/jest-plugin/test/integration/src/skip-tests.test.ts +++ b/packages/jest-plugin/test/integration/src/skip-tests.test.ts @@ -18,7 +18,9 @@ integrationTestSuite((mockBackend) => { failedTests: 2, flakyTests: 2, passedSuites: 1, + passedSuitesWithIndependentFailures: 0, passedTests: 2, + passedTestsWithIndependentFailures: 0, quarantinedSuites: 0, quarantinedTests: 0, skippedSuites: 1, @@ -51,7 +53,9 @@ integrationTestSuite((mockBackend) => { failedTests: 2, flakyTests: 2, passedSuites: 1, + passedSuitesWithIndependentFailures: 0, passedTests: 2, + passedTestsWithIndependentFailures: 0, quarantinedSuites: 0, quarantinedTests: 0, skippedSuites: 1, @@ -85,7 +89,9 @@ integrationTestSuite((mockBackend) => { failedTests: 2, flakyTests: 0, passedSuites: 1, + passedSuitesWithIndependentFailures: 0, passedTests: 2, + passedTestsWithIndependentFailures: 0, quarantinedSuites: 0, quarantinedTests: 0, skippedSuites: 2, @@ -117,7 +123,9 @@ integrationTestSuite((mockBackend) => { failedTests: 0, flakyTests: 0, passedSuites: 0, + passedSuitesWithIndependentFailures: 0, passedTests: 0, + passedTestsWithIndependentFailures: 0, quarantinedSuites: 0, quarantinedTests: 0, skippedSuites: 5, diff --git a/packages/jest-plugin/test/integration/src/test-independence.test.ts b/packages/jest-plugin/test/integration/src/test-independence.test.ts new file mode 100644 index 0000000..9905c71 --- /dev/null +++ b/packages/jest-plugin/test/integration/src/test-independence.test.ts @@ -0,0 +1,404 @@ +// Copyright (c) 2023 Developer Innovations, LLC + +import jestPackage from "jest/package.json"; +import { + defaultExpectedResults, + integrationTest, + integrationTestSuite, +} from "./test-wrappers"; +import semverLt from "semver/functions/lt"; + +integrationTestSuite((mockBackend) => { + // Test independence requires Jest 28+, so we skip these tests for earlier versions. + const itOrSkip = semverLt(jestPackage.version, "28.0.0") ? it.skip : it; + + itOrSkip("test-independent flake should pass", (done) => { + integrationTest( + { + params: { + configJs: ` +module.exports = { + __unstableIsFailureTestIndependent: ({ failure }) => + failure.includes("Error: first try should fail"), +}; +`, + expectFlakeToBeTestIndependent: true, + }, + expectedExitCode: 1, + expectedResults: { + failedSuites: 3, + failedTests: 2, + flakyTests: 0, + passedSuites: 2, + passedSuitesWithIndependentFailures: 1, + passedTests: 4, + passedTestsWithIndependentFailures: 2, + quarantinedSuites: 1, + quarantinedTests: 2, + skippedSuites: 0, + skippedTests: 0, + passedSnapshots: 1, + failedSnapshots: 0, + totalSnapshots: 1, + }, + }, + mockBackend, + done + ); + }); + + itOrSkip( + "test-independent flake without failures should exit successfully", + (done) => { + integrationTest( + { + params: { + configJs: ` +module.exports = { + __unstableIsFailureTestIndependent: /Error: first try should fail/, +}; +`, + expectFlakeToBeTestIndependent: true, + expectQuarantinedTestsToBeSkipped: true, + skipFailures: true, + skipQuarantined: true, + }, + expectedExitCode: 0, + expectedResults: { + failedSuites: 0, + failedTests: 0, + flakyTests: 0, + passedSuites: 3, + passedSuitesWithIndependentFailures: 1, + passedTests: 4, + passedTestsWithIndependentFailures: 2, + quarantinedSuites: 0, + quarantinedTests: 0, + skippedSuites: 2, + skippedTests: 4, + passedSnapshots: 0, + failedSnapshots: 0, + totalSnapshots: 0, + }, + }, + mockBackend, + done + ); + } + ); + + itOrSkip( + "test-independent fail followed by regular fail should fail", + (done) => { + integrationTest( + { + params: { + configJs: ` +const fs = require("fs/promises"); +const path = require("path"); +module.exports = { + __unstableIsFailureTestIndependent: async ({ testFilePath, failure }) => { + if (failure.includes("Error: test failed") + || failure.includes("Error: mixed test failed") + ) { + const tempFilePath = + \`\${process.env.FLAKY_TEST_TEMP}-test-independent-\${path.basename(testFilePath)}\`; + + const attempt = await fs + .readFile(tempFilePath, { encoding: "utf8" }) + .then(Number.parseInt) + .catch(() => 0); + await fs.writeFile(tempFilePath, (attempt + 1).toString()); + + return attempt === 0; + } else { + return false; + } + } +}; +`, + expectFailuresFirstAttemptToBeTestIndependent: true, + }, + expectedExitCode: 1, + expectedResults: defaultExpectedResults, + }, + mockBackend, + done + ); + } + ); + + itOrSkip( + "quarantined test-independent fail followed by regular fail should be quarantined", + (done) => { + integrationTest( + { + params: { + configJs: ` +const fs = require("fs/promises"); +const path = require("path"); +module.exports = { + __unstableIsFailureTestIndependent: async ({ testFilePath, failure }) => { + if (failure.includes("Error: quarantined test failed") + || failure.includes("Error: mixed quarantined test failed") + ) { + const tempFilePath = + \`\${process.env.FLAKY_TEST_TEMP}-test-independent-\${path.basename(testFilePath)}\`; + + const attempt = await fs + .readFile(tempFilePath, { encoding: "utf8" }) + .then(Number.parseInt) + .catch(() => 0); + await fs.writeFile(tempFilePath, (attempt + 1).toString()); + + return attempt === 0; + } else { + return false; + } + } +}; +`, + expectQuarantinedTestsFirstAttemptToBeTestIndependent: true, + }, + expectedExitCode: 1, + expectedResults: defaultExpectedResults, + }, + mockBackend, + done + ); + } + ); + + itOrSkip( + "test-independent fail then fail then pass should be flaky", + (done) => { + integrationTest( + { + params: { + configJs: ` +const fs = require("fs/promises"); +const path = require("path"); +module.exports = { + __unstableIsFailureTestIndependent: async ({ testFilePath, testName, failure }) => { + if (failure.includes("Error: first try should fail")) { + const tempFilePath = + \`\${process.env.FLAKY_TEST_TEMP}-test-independent-\${testName[0][testName[0].length - 1]}\`; + + const attempt = await fs + .readFile(tempFilePath, { encoding: "utf8" }) + .then(Number.parseInt) + .catch(() => 0); + await fs.writeFile(tempFilePath, (attempt + 1).toString()); + + return attempt === 0; + } else { + return false; + } + } +}; +`, + expectFlakeFirstAttemptToBeTestIndependent: true, + flakeFailCount: 2, + }, + expectedExitCode: 1, + expectedResults: defaultExpectedResults, + }, + mockBackend, + done + ); + } + ); + + itOrSkip( + "test-independent quarantined fail then fail then pass should be quarantined", + (done) => { + integrationTest( + { + params: { + configJs: ` +const fs = require("fs/promises"); +const path = require("path"); +module.exports = { + __unstableIsFailureTestIndependent: async ({ testFilePath, testName, failure }) => { + if (failure.includes("Error: first try should fail")) { + const tempFilePath = + \`\${process.env.FLAKY_TEST_TEMP}-test-independent-\${testName[0][testName[0].length - 1]}\`; + + const attempt = await fs + .readFile(tempFilePath, { encoding: "utf8" }) + .then(Number.parseInt) + .catch(() => 0); + await fs.writeFile(tempFilePath, (attempt + 1).toString()); + + return attempt === 0; + } else { + return false; + } + } +}; +`, + expectFlakeFirstAttemptToBeTestIndependent: true, + flakeFailCount: 2, + quarantineFlake: true, + }, + expectedExitCode: 1, + expectedResults: { + failedSuites: 3, + failedTests: 2, + flakyTests: 0, + passedSuites: 1, + passedSuitesWithIndependentFailures: 0, + passedTests: 2, + passedTestsWithIndependentFailures: 0, + quarantinedSuites: 2, + quarantinedTests: 4, + skippedSuites: 0, + skippedTests: 0, + passedSnapshots: 1, + failedSnapshots: 0, + totalSnapshots: 1, + }, + }, + mockBackend, + done + ); + } + ); + + itOrSkip("test-independent quarantined flake should pass", (done) => { + integrationTest( + { + params: { + configJs: ` +module.exports = { + __unstableIsFailureTestIndependent: ({ failure }) => + failure.includes("Error: first try should fail"), +}; +`, + expectFlakeToBeTestIndependent: true, + quarantineFlake: true, + }, + expectedExitCode: 1, + expectedResults: { + failedSuites: 3, + failedTests: 2, + flakyTests: 0, + passedSuites: 2, + passedSuitesWithIndependentFailures: 1, + passedTests: 4, + passedTestsWithIndependentFailures: 2, + quarantinedSuites: 1, + quarantinedTests: 2, + skippedSuites: 0, + skippedTests: 0, + passedSnapshots: 1, + failedSnapshots: 0, + totalSnapshots: 1, + }, + }, + mockBackend, + done + ); + }); + + itOrSkip( + "repeated test-independent failures without pass should fail", + (done) => { + integrationTest( + { + params: { + configJs: ` +module.exports = { + __unstableIsFailureTestIndependent: "Error: (?:mixed )?test failed", +}; +`, + expectFailuresToBeTestIndependent: true, + }, + expectedExitCode: 1, + expectedResults: defaultExpectedResults, + }, + mockBackend, + done + ); + } + ); + + itOrSkip( + "repeated quarantined test-independent failures without pass should be quarantined", + (done) => { + integrationTest( + { + params: { + configJs: ` +module.exports = { + __unstableIsFailureTestIndependent: /Error: (?:mixed )?quarantined test failed/, +}; +`, + expectQuarantinedTestsToBeTestIndependent: true, + }, + expectedExitCode: 1, + expectedResults: defaultExpectedResults, + }, + mockBackend, + done + ); + } + ); + + itOrSkip("multi-line regex should match", (done) => { + integrationTest( + { + params: { + configJs: ` +module.exports = { + __unstableIsFailureTestIndependent: "Error: (?:mixed )?test failed\\nnew line", +}; +`, + expectFailuresToBeTestIndependent: true, + }, + expectedExitCode: 1, + expectedResults: defaultExpectedResults, + }, + mockBackend, + done + ); + }); + + itOrSkip("regex should match stderr", (done) => { + integrationTest( + { + params: { + configJs: ` +module.exports = { + __unstableIsFailureTestIndependent: "(?:^|\\n)(?:mixed )?fail stderr\\n", +}; +`, + expectFailuresToBeTestIndependent: true, + }, + expectedExitCode: 1, + expectedResults: defaultExpectedResults, + }, + mockBackend, + done + ); + }); + + itOrSkip("regex should match stdout", (done) => { + integrationTest( + { + params: { + configJs: ` +module.exports = { + __unstableIsFailureTestIndependent: "(?:^|\\n)(?:mixed )?fail stdout\\n", +}; +`, + expectFailuresToBeTestIndependent: true, + }, + expectedExitCode: 1, + expectedResults: defaultExpectedResults, + }, + mockBackend, + done + ); + }); +}); diff --git a/packages/jest-plugin/test/integration/src/test-name-pattern.test.ts b/packages/jest-plugin/test/integration/src/test-name-pattern.test.ts index cd39f5e..a1d6a7a 100644 --- a/packages/jest-plugin/test/integration/src/test-name-pattern.test.ts +++ b/packages/jest-plugin/test/integration/src/test-name-pattern.test.ts @@ -19,7 +19,9 @@ integrationTestSuite((mockBackend) => { failedTests: 2, flakyTests: 2, passedSuites: 0, + passedSuitesWithIndependentFailures: 0, passedTests: 0, + passedTestsWithIndependentFailures: 0, quarantinedSuites: 0, quarantinedTests: 0, skippedSuites: 2, @@ -47,7 +49,9 @@ integrationTestSuite((mockBackend) => { failedTests: 0, flakyTests: 0, passedSuites: 0, + passedSuitesWithIndependentFailures: 0, passedTests: 0, + passedTestsWithIndependentFailures: 0, quarantinedSuites: 0, quarantinedTests: 0, skippedSuites: 5, @@ -90,7 +94,9 @@ integrationTestSuite((mockBackend) => { failedTests: 4, flakyTests: 0, passedSuites: 0, + passedSuitesWithIndependentFailures: 0, passedTests: 0, + passedTestsWithIndependentFailures: 0, quarantinedSuites: 0, quarantinedTests: 0, skippedSuites: 2, diff --git a/packages/jest-plugin/test/integration/src/test-wrappers.ts b/packages/jest-plugin/test/integration/src/test-wrappers.ts index baf2dd2..70c53f7 100644 --- a/packages/jest-plugin/test/integration/src/test-wrappers.ts +++ b/packages/jest-plugin/test/integration/src/test-wrappers.ts @@ -18,7 +18,9 @@ export const defaultExpectedResults: ResultCounts = { failedTests: 2, flakyTests: 2, passedSuites: 1, + passedSuitesWithIndependentFailures: 0, passedTests: 2, + passedTestsWithIndependentFailures: 0, quarantinedSuites: 1, quarantinedTests: 2, skippedSuites: 0, @@ -36,6 +38,7 @@ export const integrationTest = ( void runTestCase( { config: null, + configJs: null, expectedApiKey: "MOCK_API_KEY", expectedBranch: "MOCK_BRANCH", expectedCommit: "MOCK_COMMIT", @@ -43,13 +46,20 @@ export const integrationTest = ( expectedFlakeTestNameSuffix: "", expectedRepoRelativePathPrefix: "test/integration-input/", expectedSuiteId: "MOCK_SUITE_ID", + expectFailuresToBeTestIndependent: false, + expectFailuresFirstAttemptToBeTestIndependent: false, + expectFlakeToBeTestIndependent: false, + expectFlakeFirstAttemptToBeTestIndependent: false, expectPluginToBeEnabled: true, expectResultsToBeUploaded: true, expectQuarantinedTestsToBeQuarantined: true, expectQuarantinedTestsToBeSkipped: false, + expectQuarantinedTestsToBeTestIndependent: false, + expectQuarantinedTestsFirstAttemptToBeTestIndependent: false, expectSnapshots: false, failToFetchManifest: false, failToUploadResults: false, + flakeFailCount: 1, git: { abbreviatedRefs: { HEAD: "MOCK_BRANCH", diff --git a/packages/jest-plugin/test/integration/src/verify-output.ts b/packages/jest-plugin/test/integration/src/verify-output.ts index accc00d..d0029de 100644 --- a/packages/jest-plugin/test/integration/src/verify-output.ts +++ b/packages/jest-plugin/test/integration/src/verify-output.ts @@ -37,7 +37,8 @@ const specResultRegexMatch = ( const testResultRegexMatch = ( result: TestAttemptResult | "skipped", testName: string, - indent?: number + indent?: number, + isTestIndependent?: boolean ): RegExp => new RegExp( `^${" ".repeat(indent ?? 4)}${escapeStringRegexp( @@ -57,6 +58,10 @@ const testResultRegexMatch = ( )} \u001b\\[2m${result === "skipped" ? "skipped " : ""}${escapeStringRegexp( testName )}${ + isTestIndependent === true + ? escapeStringRegexp("\u001b[31m [test independent]\u001b[39m") + : "" + }${ result === "quarantined" ? escapeStringRegexp("\u001b[33m [quarantined]\u001b[39m") : "" @@ -67,15 +72,22 @@ const testResultRegexMatch = ( export const verifyOutput = ( { + expectFailuresToBeTestIndependent, + expectFailuresFirstAttemptToBeTestIndependent, + expectFlakeToBeTestIndependent, + expectFlakeFirstAttemptToBeTestIndependent, expectPluginToBeEnabled, expectQuarantinedTestsToBeQuarantined, expectQuarantinedTestsToBeSkipped, + expectQuarantinedTestsToBeTestIndependent, + expectQuarantinedTestsFirstAttemptToBeTestIndependent, expectResultsToBeUploaded, expectedFailureRetries, expectedFlakeTestNameSuffix, expectedSuiteId, failToFetchManifest, failToUploadResults, + flakeFailCount, quarantineFlake, skipFailures, skipFlake, @@ -108,18 +120,45 @@ export const verifyOutput = ( : 0 ); + const numFailAttempts = + !skipFailures && + (testNamePattern === undefined || + "describe block should ([escape regex]?.*$ fail".match( + testNamePattern + ) !== null) + ? expectPluginToBeEnabled + ? 1 + expectedFailureRetries + : 1 + : 0; expect(stderrLines).toContainEqualTimes( expect.stringMatching(specResultRegexMatch("fail", "src/", "fail.test.ts")), - !skipFailures && + numFailAttempts + ); + + // Test-independent failure. + expect(stderrLines).toContainEqualTimes( + expect.stringMatching( + testResultRegexMatch( + "fail", + "should ([escape regex]?.*$ fail", + 4, + true // isTestIndependent + ) + ), + expectPluginToBeEnabled && + !skipFailures && (testNamePattern === undefined || "describe block should ([escape regex]?.*$ fail".match( testNamePattern ) !== null) - ? expectPluginToBeEnabled + ? expectFailuresToBeTestIndependent ? 1 + expectedFailureRetries - : 1 + : expectFailuresFirstAttemptToBeTestIndependent + ? 1 + : 0 : 0 ); + // Non-test-independent failures. expect(stderrLines).toContainEqualTimes( expect.stringMatching( testResultRegexMatch("fail", "should ([escape regex]?.*$ fail") @@ -130,7 +169,11 @@ export const verifyOutput = ( testNamePattern ) !== null) ? expectPluginToBeEnabled - ? 1 + expectedFailureRetries + ? expectFailuresToBeTestIndependent + ? 0 + : expectFailuresFirstAttemptToBeTestIndependent + ? expectedFailureRetries + : 1 + expectedFailureRetries : 1 : 0 ); @@ -143,21 +186,31 @@ export const verifyOutput = ( !expectQuarantinedTestsToBeSkipped) && (testNamePattern === undefined || flakyTest1Name.match(testNamePattern) !== null); + + // This test should fail then pass (though we're not verifying the order here). + // Test-independent failure. expect(stderrLines).toContainEqualTimes( expect.stringMatching( - specResultRegexMatch( + testResultRegexMatch( quarantineFlake && !failToFetchManifest && expectQuarantinedTestsToBeQuarantined ? "quarantined" : "fail", - "src/", - "flake.test.ts" + flakyTest1Name, + 2, + true ) ), - flakyTest1ShouldRun ? 1 : 0 + expectPluginToBeEnabled && flakyTest1ShouldRun + ? expectFlakeToBeTestIndependent + ? flakeFailCount + : expectFlakeFirstAttemptToBeTestIndependent + ? 1 + : 0 + : 0 ); - // This test should fail then pass (though we're not verifying the order here). + // Non-test-independent failures. expect(stderrLines).toContainEqualTimes( expect.stringMatching( testResultRegexMatch( @@ -170,8 +223,17 @@ export const verifyOutput = ( 2 ) ), - flakyTest1ShouldRun ? 1 : 0 + flakyTest1ShouldRun + ? expectPluginToBeEnabled + ? expectFlakeToBeTestIndependent + ? 0 // These all ran above + : expectFlakeFirstAttemptToBeTestIndependent + ? Math.max(0, flakeFailCount - 1) + : flakeFailCount + : 1 // Plugin disabled + : 0 ); + expect(stderrLines).toContainEqualTimes( expect.stringMatching(testResultRegexMatch("pass", flakyTest1Name, 2)), expectPluginToBeEnabled && expectedFailureRetries > 0 && flakyTest1ShouldRun @@ -196,7 +258,8 @@ export const verifyOutput = ( ? "quarantined" : "fail", flakyTest2Name, - 2 + 2, + expectFlakeToBeTestIndependent ) ), flakyTest2ShouldRun ? 1 : 0 @@ -209,6 +272,31 @@ export const verifyOutput = ( : 0 ); + expect(stderrLines).toContainEqualTimes( + expect.stringMatching( + specResultRegexMatch( + quarantineFlake && + !failToFetchManifest && + expectQuarantinedTestsToBeQuarantined + ? "quarantined" + : "fail", + "src/", + "flake.test.ts" + ) + ), + flakyTest1ShouldRun || flakyTest2ShouldRun ? flakeFailCount : 0 + ); + expect(stderrLines).toContainEqualTimes( + expect.stringMatching( + specResultRegexMatch("pass", "src/", "flake.test.ts") + ), + expectPluginToBeEnabled && + (flakyTest1ShouldRun || flakyTest2ShouldRun) && + expectedFailureRetries > 0 + ? 1 + : 0 + ); + expect(stderrLines).toContainEqualTimes( expect.stringMatching( specResultRegexMatch("fail", "src/", "invalid.test.ts") @@ -216,6 +304,15 @@ export const verifyOutput = ( !skipFailures ? 1 : 0 ); + const numQuarantinedAttempts = + !skipQuarantined && + (!expectQuarantinedTestsToBeSkipped || failToFetchManifest) && + (testNamePattern === undefined || + "describe block should be quarantined".match(testNamePattern) !== null) + ? expectPluginToBeEnabled + ? 1 + expectedFailureRetries + : 1 + : 0; expect(stderrLines).toContainEqualTimes( expect.stringMatching( specResultRegexMatch( @@ -229,15 +326,37 @@ export const verifyOutput = ( "quarantined.test.ts" ) ), - !skipQuarantined && - (!expectQuarantinedTestsToBeSkipped || failToFetchManifest) && + numQuarantinedAttempts + ); + + // Test-independent failure. + expect(stderrLines).toContainEqualTimes( + expect.stringMatching( + testResultRegexMatch( + expectPluginToBeEnabled && + !failToFetchManifest && + expectQuarantinedTestsToBeQuarantined + ? "quarantined" + : "fail", + "should be quarantined", + 4, + true + ) + ), + expectPluginToBeEnabled && + !skipQuarantined && (testNamePattern === undefined || - "describe block should be quarantined".match(testNamePattern) !== null) - ? expectPluginToBeEnabled + "describe block should be quarantined".match(testNamePattern) !== + null) && + !expectQuarantinedTestsToBeSkipped + ? expectQuarantinedTestsToBeTestIndependent ? 1 + expectedFailureRetries - : 1 + : expectQuarantinedTestsFirstAttemptToBeTestIndependent + ? 1 + : 0 : 0 ); + // Non-test-independent failures. expect(stderrLines).toContainEqualTimes( expect.stringMatching( testResultRegexMatch( @@ -255,7 +374,11 @@ export const verifyOutput = ( null) && !expectQuarantinedTestsToBeSkipped ? expectPluginToBeEnabled - ? 1 + expectedFailureRetries + ? expectQuarantinedTestsToBeTestIndependent + ? 0 + : expectQuarantinedTestsFirstAttemptToBeTestIndependent + ? expectedFailureRetries + : 1 + expectedFailureRetries : 1 : 0 ); @@ -274,20 +397,67 @@ export const verifyOutput = ( "mixed mixed: should pass".match(testNamePattern) !== null; // Mixed file containing both a failed test and a quarantined one. - expect(stderrLines).toContainEqualTimes( - expect.stringMatching( - specResultRegexMatch("fail", "src/", "mixed.test.ts") - ), + const numMixedFailingAttempts = ((!expectPluginToBeEnabled || failToFetchManifest || !expectQuarantinedTestsToBeQuarantined) && mixedQuarantinedTestShouldRun) || - mixedFailTestShouldRun + mixedFailTestShouldRun ? expectPluginToBeEnabled ? 1 + expectedFailureRetries : 1 + : 0; + expect(stderrLines).toContainEqualTimes( + expect.stringMatching( + specResultRegexMatch("fail", "src/", "mixed.test.ts") + ), + numMixedFailingAttempts + ); + const numMixedQuarantinedAttempts = + expectPluginToBeEnabled && + !failToFetchManifest && + !mixedFailTestShouldRun && + mixedQuarantinedTestShouldRun && + expectQuarantinedTestsToBeQuarantined + ? 1 + expectedFailureRetries + : 0; + expect(stderrLines).toContainEqualTimes( + expect.stringMatching( + specResultRegexMatch("quarantined", "src/", "mixed.test.ts") + ), + numMixedQuarantinedAttempts + ); + const hasMixedPassingAttempt = skipFailures && skipQuarantined; + expect(stderrLines).toContainEqualTimes( + expect.stringMatching( + specResultRegexMatch("pass", "src/", "mixed.test.ts") + ), + hasMixedPassingAttempt ? 1 : 0 + ); + + // Test-independent failure. + expect(stderrLines).toContainEqualTimes( + expect.stringMatching( + testResultRegexMatch( + expectPluginToBeEnabled && + !failToFetchManifest && + expectQuarantinedTestsToBeQuarantined + ? "quarantined" + : "fail", + "mixed: should be quarantined", + 4, + true // isTestIndependent + ) + ), + expectPluginToBeEnabled && mixedQuarantinedTestShouldRun + ? expectQuarantinedTestsToBeTestIndependent + ? 1 + expectedFailureRetries + : expectQuarantinedTestsFirstAttemptToBeTestIndependent + ? 1 + : 0 : 0 ); + // Non-test-independent failures. expect(stderrLines).toContainEqualTimes( expect.stringMatching( testResultRegexMatch( @@ -301,15 +471,43 @@ export const verifyOutput = ( ), mixedQuarantinedTestShouldRun ? expectPluginToBeEnabled - ? 1 + expectedFailureRetries + ? expectQuarantinedTestsToBeTestIndependent + ? 0 + : expectQuarantinedTestsFirstAttemptToBeTestIndependent + ? expectedFailureRetries + : 1 + expectedFailureRetries : 1 : 0 ); + + // Test-independent failure. + expect(stderrLines).toContainEqualTimes( + expect.stringMatching( + testResultRegexMatch( + "fail", + "mixed: should fail", + 4, + true // isTestIndependent + ) + ), + expectPluginToBeEnabled && mixedFailTestShouldRun + ? expectFailuresToBeTestIndependent + ? 1 + expectedFailureRetries + : expectFailuresFirstAttemptToBeTestIndependent + ? 1 + : 0 + : 0 + ); + // Non-test-independent failures. expect(stderrLines).toContainEqualTimes( expect.stringMatching(testResultRegexMatch("fail", "mixed: should fail")), mixedFailTestShouldRun ? expectPluginToBeEnabled - ? 1 + expectedFailureRetries + ? expectFailuresToBeTestIndependent + ? 0 + : expectFailuresFirstAttemptToBeTestIndependent + ? expectedFailureRetries + : 1 + expectedFailureRetries : 1 : 0 ); @@ -336,6 +534,41 @@ export const verifyOutput = ( : 0 ); + for (let attempt = 1; attempt < expectedFailureRetries; attempt++) { + const numTestsToRetry = + (!skipFailures && numFailAttempts > attempt ? 1 : 0) + + (flakeFailCount + 1 > attempt + ? (flakyTest1ShouldRun ? 1 : 0) + (flakyTest2ShouldRun ? 1 : 0) + : 0) + + (numQuarantinedAttempts > attempt ? 1 : 0) + + (numMixedFailingAttempts + numMixedQuarantinedAttempts > attempt + ? (mixedFailTestShouldRun ? 1 : 0) + + (mixedQuarantinedTestShouldRun ? 1 : 0) + : 0); + const numTestFilesToRetry = + (!skipFailures && numFailAttempts > attempt ? 1 : 0) + + ((flakyTest1ShouldRun || flakyTest2ShouldRun) && + flakeFailCount + 1 > attempt + ? 1 + : 0) + + (numQuarantinedAttempts > attempt ? 1 : 0) + + ((mixedFailTestShouldRun || mixedQuarantinedTestShouldRun) && + numMixedFailingAttempts + numMixedQuarantinedAttempts > attempt + ? 1 + : 0); + expect(numTestsToRetry).toBeGreaterThanOrEqual(numTestFilesToRetry); + expect(stderrLines).toContainEqualTimes( + `\u001b[33m\u001b[1mRetrying ${numTestsToRetry} failed test${ + numTestsToRetry === 1 ? "" : "s" + } from ${numTestFilesToRetry} file${ + numTestFilesToRetry === 1 ? "" : "s" + } -- ${expectedFailureRetries - attempt} ${ + expectedFailureRetries - attempt === 1 ? "retry" : "retries" + } remaining\u001b[22m\u001b[39m`, + expectPluginToBeEnabled && numTestsToRetry > 0 ? 1 : 0 + ); + } + // Test our SummaryReporter customization. expect(stderrLines).toContain( `\u001b[1mTest Suites: \u001b[22m${ @@ -352,7 +585,11 @@ export const verifyOutput = ( : "" }${ expectedResults.passedSuites !== 0 - ? `\u001b[1m\u001b[32m${expectedResults.passedSuites} passed\u001b[39m\u001b[22m, ` + ? `\u001b[1m\u001b[32m${expectedResults.passedSuites} passed${ + expectedResults.passedSuitesWithIndependentFailures > 0 + ? ` (${expectedResults.passedSuitesWithIndependentFailures} with test-independent failures)` + : "" + }\u001b[39m\u001b[22m, ` : "" }${ expectedResults.skippedSuites !== 0 @@ -391,7 +628,11 @@ export const verifyOutput = ( : "" }${ expectedResults.passedTests !== 0 - ? `\u001b[1m\u001b[32m${expectedResults.passedTests} passed\u001b[39m\u001b[22m, ` + ? `\u001b[1m\u001b[32m${expectedResults.passedTests} passed${ + expectedResults.passedTestsWithIndependentFailures > 0 + ? ` (${expectedResults.passedTestsWithIndependentFailures} with test-independent failures)` + : "" + }\u001b[39m\u001b[22m, ` : "" }${ expectedResults.failedTests + @@ -422,7 +663,9 @@ export const verifyOutput = ( expect(stderrLines).toContainEqual( expect.stringMatching( new RegExp( - `${escapeStringRegexp("\u001b[1mTime:\u001b[22m ")}[0-9.]+ s` + `${escapeStringRegexp( + "\u001b[1mTime:\u001b[22m " + )}(?:\u001b\\[1m\u001b\\[33m[0-9.]+ s\u001b\\[39m\u001b\\[22m|[0-9.]+ s)(?:, estimated [0-9.]+ s)?` ) ) ); diff --git a/packages/jest-plugin/tsconfig.json b/packages/jest-plugin/tsconfig.json index ec572ba..97b22c5 100644 --- a/packages/jest-plugin/tsconfig.json +++ b/packages/jest-plugin/tsconfig.json @@ -7,8 +7,8 @@ "module": "esnext", // Ensure Rollup build fails if there are type errors. "noEmitOnError": true, - // Removes DOM types. - "lib": ["ES2019"], + // Remove DOM types and support 2-argument Error constructor that takes a cause. + "lib": ["ES2022"], // Avoids conflicting global definitions from, e.g., jasmine. "types": ["node", "jest"], // Some versions of Jest (e.g., 28.0.0) have internally broken types. @@ -16,10 +16,12 @@ }, "include": [ ".eslintrc.js", + "jest.config.js", + "jest-circus.d.ts", + "rollup.config.mjs", "src", "test/.eslintrc.js", "test/babel.config.js", - "rollup.config.mjs", "window.d.ts" ] } diff --git a/packages/js-api/src/index.ts b/packages/js-api/src/index.ts index 778720c..a16a155 100644 --- a/packages/js-api/src/index.ts +++ b/packages/js-api/src/index.ts @@ -50,10 +50,12 @@ export type TestSuiteManifest = { }; export type TestAttemptResult = "pass" | "fail" | "quarantined"; +export type TestAttemptFailureReason = "independent"; export type TestRunAttemptRecord = { start_time?: string; end_time?: string; duration_ms?: number; + failure_reason?: TestAttemptFailureReason; result: TestAttemptResult; }; export type TestRunRecord = { diff --git a/packages/plugins-common/src/config.ts b/packages/plugins-common/src/config.ts index db11559..d277fdd 100644 --- a/packages/plugins-common/src/config.ts +++ b/packages/plugins-common/src/config.ts @@ -4,6 +4,7 @@ import { CosmiconfigResult } from "cosmiconfig/dist/types"; import { cosmiconfig, cosmiconfigSync } from "cosmiconfig"; import _debug from "debug"; import { EnvVar, suiteIdOverride, uploadResultsOverride } from "./env"; +import * as util from "util"; const debug = _debug("unflakable:plugins-common:config"); @@ -29,7 +30,7 @@ export type UnflakableConfig = testSuiteId: string | undefined; } & UnflakableConfigInner); -type UnflakableConfigFile = { +export type UnflakableConfigFile = { enabled: boolean; testSuiteId: string | undefined; } & UnflakableConfigInner; @@ -44,9 +45,10 @@ const defaultConfig: UnflakableConfigFile = { uploadResults: true, }; -const validateConfig = ( - configResult: NonNullable -): UnflakableConfigFile => { +const validateConfig = ( + configResult: NonNullable, + validateExtraConfig: (config: CosmiconfigResult) => [T, string[]] +): UnflakableConfigFile & T => { debug(`Loaded config from ${configResult.filepath}`); // NB: typeof null is "object" if (typeof configResult.config !== "object" || configResult.config === null) { @@ -57,11 +59,13 @@ const validateConfig = ( ); } + const [extraConfig, validExtraConfigKeys] = validateExtraConfig(configResult); + return Object.entries(configResult.config as { [s: string]: unknown }).reduce( - (result: UnflakableConfigFile, [key, value]: [string, unknown]) => { + (result: UnflakableConfigFile & T, [key, value]: [string, unknown]) => { const throwUnexpected = (): never => { throw new Error( - `Unexpected value \`${JSON.stringify(value)}\` for ${JSON.stringify( + `Unexpected value \`${util.format(value)}\` for ${JSON.stringify( key )} found in ${configResult.filepath}` ); @@ -139,12 +143,16 @@ const validateConfig = ( return throwUnexpected(); } default: - throw new Error( - `Unknown Unflakable config option \`${key}\` found in ${configResult.filepath}` - ); + if (!validExtraConfigKeys.includes(key)) { + throw new Error( + `Unknown Unflakable config option \`${key}\` found in ${configResult.filepath}` + ); + } + + return result; } }, - { ...defaultConfig } + { ...defaultConfig, ...extraConfig } ); }; @@ -247,9 +255,10 @@ export const setCosmiconfigSync = (config: typeof cosmiconfigSync): void => { ).__unflakableCosmiconfigSync = config; }; -const loadConfigFile = async ( - searchFrom: string -): Promise => { +const loadConfigFile = async ( + searchFrom: string, + validateExtraConfig: (config: CosmiconfigResult) => [T, string[]] +): Promise => { const configExplorer = ( ( globalThis as { @@ -262,22 +271,31 @@ const loadConfigFile = async ( debug(`Searching for config from directory \`${searchFrom}\` upward`); const configResult = await configExplorer.search(searchFrom); if (configResult !== null) { - return validateConfig(configResult); + return validateConfig(configResult, validateExtraConfig); } else { debug("No config file found; using defaults"); - return { ...defaultConfig }; + return { + ...defaultConfig, + ...((validateExtraConfig !== undefined + ? validateExtraConfig(configResult) + : {}) as T), + }; } }; -export const loadConfig = ( +export const loadConfig = ( searchFrom: string, + validateExtraConfig: (config: CosmiconfigResult) => [T, string[]], testSuiteId?: string -): Promise => - loadConfigFile(searchFrom).then((config) => +): Promise => + loadConfigFile(searchFrom, validateExtraConfig).then((config) => mergeConfigWithEnv(config, testSuiteId) ); -const loadConfigFileSync = (searchFrom: string): UnflakableConfigFile => { +const loadConfigFileSync = ( + searchFrom: string, + validateExtraConfig: (config: CosmiconfigResult) => [T, string[]] +): UnflakableConfigFile & T => { const configExplorer = ( ( globalThis as { @@ -290,15 +308,23 @@ const loadConfigFileSync = (searchFrom: string): UnflakableConfigFile => { debug(`Searching for config from directory \`${searchFrom}\` upward`); const configResult = configExplorer.search(searchFrom); if (configResult !== null) { - return validateConfig(configResult); + return validateConfig(configResult, validateExtraConfig); } else { debug("No config file found; using defaults"); - return { ...defaultConfig }; + return { + ...defaultConfig, + ...((validateExtraConfig !== undefined + ? validateExtraConfig(configResult) + : {}) as T), + }; } }; -export const loadConfigSync = (searchFrom: string): UnflakableConfig => - mergeConfigWithEnv(loadConfigFileSync(searchFrom)); +export const loadConfigSync = ( + searchFrom: string, + validateExtraConfig: (config: CosmiconfigResult) => [T, string[]] +): UnflakableConfig & T => + mergeConfigWithEnv(loadConfigFileSync(searchFrom, validateExtraConfig)); export const loadApiKey = (): string => { if (apiKey.value !== undefined && apiKey.value !== "") { diff --git a/packages/plugins-common/src/index.ts b/packages/plugins-common/src/index.ts index 851f379..ee6fa9e 100644 --- a/packages/plugins-common/src/index.ts +++ b/packages/plugins-common/src/index.ts @@ -6,6 +6,7 @@ export { QuarantineMode, UnflakableConfig, UnflakableConfigEnabled, + UnflakableConfigFile, loadApiKey, loadConfig, loadConfigSync, diff --git a/packages/test-common/src/config.ts b/packages/test-common/src/config.ts index 4ac20ed..c39d9a6 100644 --- a/packages/test-common/src/config.ts +++ b/packages/test-common/src/config.ts @@ -6,7 +6,12 @@ import { setCosmiconfig, setCosmiconfigSync, } from "@unflakable/plugins-common"; -import type { cosmiconfig, cosmiconfigSync, Options } from "cosmiconfig"; +import { + cosmiconfig, + cosmiconfigSync, + Options, + OptionsSync, +} from "cosmiconfig"; import { default as expect } from "expect"; const debug = _debug("unflakable:test-common:config"); @@ -16,12 +21,17 @@ const throwUnimplemented = (): never => { }; export type CosmiconfigMockParams = { - searchFrom: string; - searchResult: { - config: Partial; - filepath: string; - } | null; -}; + expectedSearchFrom: string; +} & ( + | { pathToLoad: string; searchResult?: undefined } + | { + pathToLoad?: undefined; + searchResult: { + config: Partial; + filepath: string; + } | null; + } +); export const CONFIG_MOCK_ENV_VAR = "__UNFLAKABLE_TEST_CONFIG_MOCK_PARAMS"; @@ -55,8 +65,12 @@ export const registerCosmiconfigMock = (): void => { search: ( searchFrom?: string ): ReturnType["search"]> => { - expect(searchFrom).toBe(params.searchFrom); - return Promise.resolve(params.searchResult); + expect(searchFrom).toBe(params.expectedSearchFrom); + if (params.pathToLoad !== undefined) { + return cosmiconfig(moduleName, options).load(params.pathToLoad); + } else { + return Promise.resolve(params.searchResult); + } }, }; } @@ -65,7 +79,7 @@ export const registerCosmiconfigMock = (): void => { setCosmiconfigSync( ( moduleName: string, - options?: Options + options?: OptionsSync ): ReturnType => { expect(moduleName).toBe("unflakable"); expect(options?.searchPlaces).toContain("package.json"); @@ -81,8 +95,12 @@ export const registerCosmiconfigMock = (): void => { search: ( searchFrom?: string ): ReturnType["search"]> => { - expect(searchFrom).toBe(params.searchFrom); - return params.searchResult; + expect(searchFrom).toBe(params.expectedSearchFrom); + if (params.pathToLoad !== undefined) { + return cosmiconfigSync(moduleName, options).load(params.pathToLoad); + } else { + return params.searchResult; + } }, }; } diff --git a/yarn.lock b/yarn.lock index dc6274a..5ea928e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3271,10 +3271,14 @@ __metadata: deep-equal: ^2.0.5 escape-string-regexp: ^4.0.0 exit: ^0.1.2 + jest: 25.1.0 - 29 + jest-circus: 25.1.0 - 29 + jest-environment-node: 25.1.0 - 29 jest-runner: 25.1.0 - 29 jest-util: 25.1.0 - 29 rimraf: ^5.0.1 rollup: ^3.21.1 + semver: ^7.5.4 simple-git: ^3.16.0 typescript: ^4.9.5 peerDependencies: @@ -7341,7 +7345,7 @@ __metadata: languageName: node linkType: hard -"jest-circus@npm:^29.5.0": +"jest-circus@npm:25.1.0 - 29, jest-circus@npm:^29.5.0": version: 29.5.0 resolution: "jest-circus@npm:29.5.0" dependencies: @@ -7543,6 +7547,7 @@ __metadata: "@types/temp": ^0.9.1 "@unflakable/jest-plugin": "workspace:^" "@unflakable/js-api": "workspace:^" + es6-promisify: ^7.0.0 escape-string-regexp: ^4.0.0 expect: 25.1.0 - 29 jest: 25.1.0 - 29 @@ -7550,7 +7555,8 @@ __metadata: jest-get-type: 25.1.0 - 29 jest-matcher-utils: 25.1.0 - 29 mockttp: ^3.7.5 - temp: ^0.9.4 + semver: ^7.5.4 + tmp: ^0.2.1 typescript: ^4.9.5 unflakable-test-common: "workspace:^" languageName: unknown @@ -8500,7 +8506,7 @@ __metadata: languageName: node linkType: hard -"mkdirp@npm:^0.5.1, mkdirp@npm:^0.5.5": +"mkdirp@npm:^0.5.5": version: 0.5.6 resolution: "mkdirp@npm:0.5.6" dependencies: @@ -9713,17 +9719,6 @@ __metadata: languageName: node linkType: hard -"rimraf@npm:~2.6.2": - version: 2.6.3 - resolution: "rimraf@npm:2.6.3" - dependencies: - glob: ^7.1.3 - bin: - rimraf: ./bin.js - checksum: 3ea587b981a19016297edb96d1ffe48af7e6af69660e3b371dbfc73722a73a0b0e9be5c88089fbeeb866c389c1098e07f64929c7414290504b855f54f901ab10 - languageName: node - linkType: hard - "rollup-plugin-dts@npm:^5.3.0": version: 5.3.0 resolution: "rollup-plugin-dts@npm:5.3.0" @@ -9878,7 +9873,7 @@ __metadata: languageName: node linkType: hard -"semver@npm:^7.3.2, semver@npm:^7.3.4, semver@npm:^7.3.5, semver@npm:^7.3.7, semver@npm:^7.5.1, semver@npm:^7.5.3": +"semver@npm:^7.3.2, semver@npm:^7.3.4, semver@npm:^7.3.5, semver@npm:^7.3.7, semver@npm:^7.5.1, semver@npm:^7.5.3, semver@npm:^7.5.4": version: 7.5.4 resolution: "semver@npm:7.5.4" dependencies: @@ -10447,16 +10442,6 @@ __metadata: languageName: node linkType: hard -"temp@npm:^0.9.4": - version: 0.9.4 - resolution: "temp@npm:0.9.4" - dependencies: - mkdirp: ^0.5.1 - rimraf: ~2.6.2 - checksum: 8709d4d63278bd309ca0e49e80a268308dea543a949e71acd427b3314cd9417da9a2cc73425dd9c21c6780334dbffd67e05e7be5aaa73e9affe8479afc6f20e3 - languageName: node - linkType: hard - "term-size@npm:2.1.0": version: 2.1.0 resolution: "term-size@npm:2.1.0" @@ -10542,7 +10527,7 @@ __metadata: languageName: node linkType: hard -"tmp@npm:~0.2.1": +"tmp@npm:^0.2.1, tmp@npm:~0.2.1": version: 0.2.1 resolution: "tmp@npm:0.2.1" dependencies: From d87911f82d430b877e98ccd8d53f5ed0453b9bd7 Mon Sep 17 00:00:00 2001 From: "David A. Ramos" Date: Sat, 5 Aug 2023 14:27:13 -0700 Subject: [PATCH 19/53] [js-api] Retry all 5xx errors S3 and other AWS services occasionally return transient 5xx errors, and we should retry all of those. --- packages/js-api/src/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/js-api/src/index.ts b/packages/js-api/src/index.ts index a16a155..b2cc254 100644 --- a/packages/js-api/src/index.ts +++ b/packages/js-api/src/index.ts @@ -15,7 +15,7 @@ const fetch = (url: string, init?: RequestInit): Promise => { return retry( () => nodeFetch(url, init).then((response) => { - if (response.status === 503) { + if (response.status >= 500 && response.status <= 599) { throw new Error( `Server returned ${response.status} ${response.statusText}` ); From 6261a76ca7f5ec54df588c914a2d7eb50ca50923 Mon Sep 17 00:00:00 2001 From: "David A. Ramos" Date: Sat, 26 Aug 2023 14:54:34 -0700 Subject: [PATCH 20/53] [cypress] Add comment about cypress-io/cypress#27390 --- packages/cypress-plugin/src/plugin.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/cypress-plugin/src/plugin.ts b/packages/cypress-plugin/src/plugin.ts index b244d07..60e1596 100644 --- a/packages/cypress-plugin/src/plugin.ts +++ b/packages/cypress-plugin/src/plugin.ts @@ -88,6 +88,7 @@ const marshalAttempt = ( return null; } + // NB: These types are broken in 12.17+ due to https://github.com/cypress-io/cypress/issues/27390. return { start_time: new Date(attempt.startedAt).toISOString(), // NB: there's no explicit end time for each attempt, Cypress does set the duration. From 7156946c3e03e605591c08193bc4de174bb3cf3b Mon Sep 17 00:00:00 2001 From: "David A. Ramos" Date: Sat, 26 Aug 2023 15:48:16 -0700 Subject: [PATCH 21/53] [cypress] Fix tests for Cypress 12.17.4 (Webpack 5) --- .../test/integration-input-manual/cypress-config.mjs | 10 ++++++++++ .../test/integration-input-manual/cypress.config.js | 10 ++++++++++ .../cypress/support/e2e-webpack5.js | 10 ++++++++++ .../test/integration-input-manual/package.json | 1 + yarn.lock | 1 + 5 files changed, 32 insertions(+) create mode 100644 packages/cypress-plugin/test/integration-input-manual/cypress/support/e2e-webpack5.js diff --git a/packages/cypress-plugin/test/integration-input-manual/cypress-config.mjs b/packages/cypress-plugin/test/integration-input-manual/cypress-config.mjs index db5d6b2..952022a 100644 --- a/packages/cypress-plugin/test/integration-input-manual/cypress-config.mjs +++ b/packages/cypress-plugin/test/integration-input-manual/cypress-config.mjs @@ -6,6 +6,8 @@ import webpackConfig from "./config/webpack.js"; import { registerSimpleGitMock } from "unflakable-test-common/dist/git.js"; import { registerCosmiconfigMock } from "unflakable-test-common/dist/config.js"; import { registerUnflakable } from "@unflakable/cypress-plugin"; +import semverGte from "semver/functions/gte.js"; +import path from "path"; /** * @type {Cypress.ConfigOptions} @@ -43,6 +45,14 @@ export default { tasks.registerTasks(on); devtools.openDevToolsOnLaunch(on); + // Versions prior to 12.17.4 use Webpack 4, which doesn't support the package.json "exports" + // field (see https://github.com/cypress-io/cypress/issues/23826). Webpack 5 both supports and + // enforces this field, so we have to use a different require path to manually import the + // skip-tests module. + if (semverGte(config.version, "12.17.4")) { + config.supportFile = path.resolve("./cypress/support/e2e-webpack5.js"); + } + return registerUnflakable(on, config); }, // supportFile: "cypress/support/e2e.js", diff --git a/packages/cypress-plugin/test/integration-input-manual/cypress.config.js b/packages/cypress-plugin/test/integration-input-manual/cypress.config.js index bb3bc5a..9e0a4e8 100644 --- a/packages/cypress-plugin/test/integration-input-manual/cypress.config.js +++ b/packages/cypress-plugin/test/integration-input-manual/cypress.config.js @@ -7,8 +7,10 @@ const { registerSimpleGitMock } = require("unflakable-test-common/dist/git"); const { registerCosmiconfigMock, } = require("unflakable-test-common/dist/config"); +const semverGte = require("semver/functions/gte"); const { registerUnflakable } = require("@unflakable/cypress-plugin"); +const path = require("path"); module.exports = { /** @@ -46,6 +48,14 @@ module.exports = { registerTasks(on); openDevToolsOnLaunch(on); + // Versions prior to 12.17.4 use Webpack 4, which doesn't support the package.json "exports" + // field (see https://github.com/cypress-io/cypress/issues/23826). Webpack 5 both supports and + // enforces this field, so we have to use a different require path to manually import the + // skip-tests module. + if (semverGte(config.version, "12.17.4")) { + config.supportFile = path.resolve("./cypress/support/e2e-webpack5.js"); + } + return registerUnflakable(on, config); }, // supportFile: "cypress/support/e2e.js", diff --git a/packages/cypress-plugin/test/integration-input-manual/cypress/support/e2e-webpack5.js b/packages/cypress-plugin/test/integration-input-manual/cypress/support/e2e-webpack5.js new file mode 100644 index 0000000..088357d --- /dev/null +++ b/packages/cypress-plugin/test/integration-input-manual/cypress/support/e2e-webpack5.js @@ -0,0 +1,10 @@ +// Copyright (c) 2023 Developer Innovations, LLC + +// Import commands.js using CJS syntax: +require("./commands.js"); + +const { + registerMochaInstrumentation, +} = require("@unflakable/cypress-plugin/skip-tests"); + +registerMochaInstrumentation(); diff --git a/packages/cypress-plugin/test/integration-input-manual/package.json b/packages/cypress-plugin/test/integration-input-manual/package.json index 0591f0d..d97f955 100644 --- a/packages/cypress-plugin/test/integration-input-manual/package.json +++ b/packages/cypress-plugin/test/integration-input-manual/package.json @@ -16,6 +16,7 @@ "process": "^0.11.10", "react": "^18.2.0", "react-dom": "^18.2.0", + "semver": "^7.5.4", "ts-loader": "^9.4.3", "typescript": "^4.9.5", "unflakable-test-common": "workspace:^", diff --git a/yarn.lock b/yarn.lock index 5ea928e..ffb7e94 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4777,6 +4777,7 @@ __metadata: process: ^0.11.10 react: ^18.2.0 react-dom: ^18.2.0 + semver: ^7.5.4 ts-loader: ^9.4.3 typescript: ^4.9.5 unflakable-test-common: "workspace:^" From 774a1d91cf053c4de0f7f014ee20098be4396d26 Mon Sep 17 00:00:00 2001 From: "David A. Ramos" Date: Sat, 26 Aug 2023 17:52:05 -0700 Subject: [PATCH 22/53] [cypress] Treat Chrome DevTools errors as test-independent --- packages/cypress-plugin/test/integration/unflakable.js | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/cypress-plugin/test/integration/unflakable.js b/packages/cypress-plugin/test/integration/unflakable.js index ccb959b..1adc55c 100644 --- a/packages/cypress-plugin/test/integration/unflakable.js +++ b/packages/cypress-plugin/test/integration/unflakable.js @@ -8,5 +8,6 @@ module.exports = { // NB: This requires DEBUG="cypress:server:run" (at a minimum). /attempting to close the browser tab(?:(?!resetting server state).)*$/s, /Still waiting to connect to Edge, retrying in 1 second.*(?:Error: Test timed out after|All promises were rejected)/s, + /There was an error reconnecting to the Chrome DevTools protocol\. Please restart the browser\./, ], }; From c3cc5d1316f3b1a5bd33d16dd445ba2d7cfcd14c Mon Sep 17 00:00:00 2001 From: "David A. Ramos" Date: Wed, 30 Aug 2023 17:12:08 -0700 Subject: [PATCH 23/53] [cypress] Treat "Timed out waiting for the browser" as test-independent --- .../cypress-plugin/test/integration/src/verify-output.ts | 4 ++-- packages/cypress-plugin/test/integration/unflakable.js | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/packages/cypress-plugin/test/integration/src/verify-output.ts b/packages/cypress-plugin/test/integration/src/verify-output.ts index 6512caf..1605924 100644 --- a/packages/cypress-plugin/test/integration/src/verify-output.ts +++ b/packages/cypress-plugin/test/integration/src/verify-output.ts @@ -1345,9 +1345,9 @@ export const verifyOutput = ( ).toStrictEqual(expectedSpecs); // Make sure there are no unexpected specs. - expect(expectedSpecs).toStrictEqual( + expect( parsedOutput.specOutputs.map((spec) => spec.filename).sort() - ); + ).toStrictEqual(expectedSpecs); verifySpecOutputs(params, parsedOutput.specOutputs); } diff --git a/packages/cypress-plugin/test/integration/unflakable.js b/packages/cypress-plugin/test/integration/unflakable.js index 1adc55c..d24c20f 100644 --- a/packages/cypress-plugin/test/integration/unflakable.js +++ b/packages/cypress-plugin/test/integration/unflakable.js @@ -9,5 +9,11 @@ module.exports = { /attempting to close the browser tab(?:(?!resetting server state).)*$/s, /Still waiting to connect to Edge, retrying in 1 second.*(?:Error: Test timed out after|All promises were rejected)/s, /There was an error reconnecting to the Chrome DevTools protocol\. Please restart the browser\./, + // When this error occurs, Cypress ends up printing the "Running: " header multiple times, + // which the integration test parses as if that spec were in fact invoked multiple times. We + // don't want the test itself to ignore multiple spec invocations since that could indicate a + // bug. Instead, we treat it as a test independent failure iff this error message is in the + // output. Otherwise, we'll still treat it as a true failure. + /Timed out waiting for the browser to connect. Retrying\.\.\./, ], }; From 1885917a4cb21cc6bcd79c36054b06213460880c Mon Sep 17 00:00:00 2001 From: "David A. Ramos" Date: Mon, 4 Sep 2023 14:26:21 -0700 Subject: [PATCH 24/53] Bump actions/checkout to v4 See https://github.com/actions/checkout/issues/1448. --- .github/workflows/ci.yaml | 10 +++++----- .github/workflows/cypress-realworld-app.yaml | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 72abb77..fba26b6 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -20,7 +20,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-node@v3 with: @@ -128,7 +128,7 @@ jobs: - "12.16" - "12.17" steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-node@v3 with: @@ -233,7 +233,7 @@ jobs: - "12.16" - "12.17" steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-node@v3 with: @@ -357,7 +357,7 @@ jobs: CYPRESS_INSTALL_BINARY: "0" steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-node@v3 with: @@ -473,7 +473,7 @@ jobs: CYPRESS_INSTALL_BINARY: "0" steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-node@v3 with: diff --git a/.github/workflows/cypress-realworld-app.yaml b/.github/workflows/cypress-realworld-app.yaml index 0bb2ac2..eab82bc 100644 --- a/.github/workflows/cypress-realworld-app.yaml +++ b/.github/workflows/cypress-realworld-app.yaml @@ -20,7 +20,7 @@ jobs: options: --user 1001 steps: - name: Check out latest cypress-io/cypress-realworld-app - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: cypress-io/cypress-realworld-app From 84224be5e54b774b74c4535f38aafa40fb0f6840 Mon Sep 17 00:00:00 2001 From: "David A. Ramos" Date: Thu, 31 Aug 2023 15:02:02 -0700 Subject: [PATCH 25/53] [cypress] Support Cypress 13 --- .github/workflows/ci.yaml | 8 +- README.md | 2 +- packages/cypress-plugin/README.md | 2 +- packages/cypress-plugin/mocha.d.ts | 2 - packages/cypress-plugin/package.json | 5 +- packages/cypress-plugin/src/main.ts | 2 +- packages/cypress-plugin/src/plugin.ts | 287 ++++++------- packages/cypress-plugin/src/reporter.ts | 218 +++++++--- .../test/integration-input-esm/package.json | 2 +- .../cypress/e2e/hook-fail.cy.js | 4 + .../integration-input-manual/package.json | 2 +- .../cypress/component/hook-fail.cy.ts | 8 +- .../test/integration-input/package.json | 2 +- .../test/integration/package.json | 3 +- .../integration/src/hook-failures.test.ts | 6 +- .../test/integration/src/run-test-case.ts | 22 +- .../test/integration/src/verify-output.ts | 391 +++++++++--------- yarn.lock | 16 +- 18 files changed, 573 insertions(+), 409 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index fba26b6..1bff501 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -127,6 +127,8 @@ jobs: - "12.15" - "12.16" - "12.17" + - "13.0" + - "13.1" steps: - uses: actions/checkout@v4 @@ -169,7 +171,7 @@ jobs: env: CYPRESS_INSTALL_BINARY: "0" run: | - yarn set resolution "cypress@npm:10 - 12" ${{ matrix.cypress }} + yarn set resolution "cypress@npm:11.2 - 13" ${{ matrix.cypress }} grep --after-context=1 "^\".*cypress.*" yarn.lock - name: Install Cypress binary @@ -232,6 +234,8 @@ jobs: - "12.15" - "12.16" - "12.17" + - "13.0" + - "13.1" steps: - uses: actions/checkout@v4 @@ -278,7 +282,7 @@ jobs: env: CYPRESS_INSTALL_BINARY: "0" run: | - yarn set resolution "cypress@npm:10 - 12" ${{ matrix.cypress }} + yarn set resolution "cypress@npm:11.2 - 13" ${{ matrix.cypress }} Select-String -Pattern '^".*cypress.*' -Path yarn.lock -Context 0,1 - name: Install Cypress binary diff --git a/README.md b/README.md index 0b3915a..59e9c79 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ complete usage instructions. This plugin maintains compatibility with the Cypress and Node.js versions listed below: -[![11.2.0+ | 12.0.0+](https://img.shields.io/badge/Cypress-11.2.0%2B%20%7C%2012.0.0%2B-17202C?logo=cypress&labelColor=white&logoColor=17202C&style=flat)](https://cypress.io) +[![11.2.0+ | 12.0.0+ | 13.0.0+](https://img.shields.io/badge/Cypress-11.2.0%2B%20%7C%2012.0.0%2B%20%7C%2013.0.0%2B-17202C?logo=cypress&labelColor=white&logoColor=17202C&style=flat)](https://cypress.io) [![16 | 18 | 20](https://img.shields.io/badge/Node.js-16%20%7C%2018%20%7C%2020-339933?logo=node.js&labelColor=white&logoColor=339933&style=flat)](https://nodejs.org) ## Jest Plugin diff --git a/packages/cypress-plugin/README.md b/packages/cypress-plugin/README.md index 961ceac..2d64bc8 100644 --- a/packages/cypress-plugin/README.md +++ b/packages/cypress-plugin/README.md @@ -19,7 +19,7 @@ complete usage instructions. This plugin maintains compatibility with the Cypress and Node.js versions listed below: -[![11.2.0+ | 12.0.0+](https://img.shields.io/badge/Cypress-11.2.0%2B%20%7C%2012.0.0%2B-17202C?logo=cypress&labelColor=white&logoColor=17202C&style=flat)](https://cypress.io) +[![11.2.0+ | 12.0.0+ | 13.0.0+](https://img.shields.io/badge/Cypress-11.2.0%2B%20%7C%2012.0.0%2B%20%7C%2013.0.0%2B-17202C?logo=cypress&labelColor=white&logoColor=17202C&style=flat)](https://cypress.io) [![16 | 18 | 20](https://img.shields.io/badge/Node.js-16%20%7C%2018%20%7C%2020-339933?logo=node.js&labelColor=white&logoColor=339933&style=flat)](https://nodejs.org) ## Contributing diff --git a/packages/cypress-plugin/mocha.d.ts b/packages/cypress-plugin/mocha.d.ts index d81ed0d..428fca1 100644 --- a/packages/cypress-plugin/mocha.d.ts +++ b/packages/cypress-plugin/mocha.d.ts @@ -20,8 +20,6 @@ declare namespace Mocha { // Additional fields that Cypress adds to Mocha.Test. export type CypressTestProps = { - id: string; - order: number; wallClockStartedAt: string; }; diff --git a/packages/cypress-plugin/package.json b/packages/cypress-plugin/package.json index dc62879..0e4fb64 100644 --- a/packages/cypress-plugin/package.json +++ b/packages/cypress-plugin/package.json @@ -56,6 +56,7 @@ "debug": "^4.3.3", "deep-equal": "^2.0.5", "es6-promisify": "^7.0.0", + "escape-string-regexp": "^4.0.0", "lodash": "^4.17.21", "log-symbols": "^4.1.0", "mocha": "=7.0.1", @@ -80,7 +81,7 @@ "@types/yargs": "^17.0.24", "@unflakable/plugins-common": "workspace:", "cross-env": "^7.0.3", - "cypress": "10 - 12", + "cypress": "11.2 - 13", "jest": "^29.5.0", "jest-environment-node": "^29.5.0", "rimraf": "^5.0.1", @@ -91,7 +92,7 @@ "widest-line": "3.1.0" }, "peerDependencies": { - "cypress": "11.2 - 12" + "cypress": "11.2 - 13" }, "scripts": { "build": "yarn clean && tsc --noEmit && tsc --noEmit -p src && rollup --config", diff --git a/packages/cypress-plugin/src/main.ts b/packages/cypress-plugin/src/main.ts index c1deb1a..ccf5ad2 100755 --- a/packages/cypress-plugin/src/main.ts +++ b/packages/cypress-plugin/src/main.ts @@ -393,7 +393,7 @@ const main = async (): Promise => { } : runOptions ); - if (results.status === "finished") { + if (results.status !== "failed") { exitDefault(results, unflakableConfig); } else { exitFailure(results); diff --git a/packages/cypress-plugin/src/plugin.ts b/packages/cypress-plugin/src/plugin.ts index 60e1596..158e548 100644 --- a/packages/cypress-plugin/src/plugin.ts +++ b/packages/cypress-plugin/src/plugin.ts @@ -90,7 +90,10 @@ const marshalAttempt = ( // NB: These types are broken in 12.17+ due to https://github.com/cypress-io/cypress/issues/27390. return { - start_time: new Date(attempt.startedAt).toISOString(), + start_time: + attempt.startedAt !== undefined + ? new Date(attempt.startedAt).toISOString() + : undefined, // NB: there's no explicit end time for each attempt, Cypress does set the duration. duration_ms: attempt.duration, result, @@ -100,9 +103,7 @@ const marshalAttempt = ( // Adapted from: // https://github.com/cypress-io/cypress/blob/19e091d0bc2d1f4e6a6e62d2f81ea6a2f60d531a/packages/server/lib/util/print-run.ts#L397C15-L440 const displayResults = ( - spec: Cypress.Spec & { - relativeToCommonRoot: string; - }, + relativeToCommonRoot: string, results: CypressCommandLine.RunResult ): void => { const reporterStats = reporterStatsOrDefault(results); @@ -154,7 +155,7 @@ const displayResults = ( ["Duration:", humanTime.long(results.stats.wallClockDuration ?? 0)], [ "Spec Ran:", - formatPath(spec.relativeToCommonRoot, getWidth(table, 1), resultColor), + formatPath(relativeToCommonRoot, getWidth(table, 1), resultColor), ], ] as (HorizontalTableRow | undefined)[] ) @@ -223,136 +224,17 @@ const formatFooterSummary = ( ]; }; -// Adapted from: -// https://github.com/cypress-io/cypress/blob/19e091d0bc2d1f4e6a6e62d2f81ea6a2f60d531a/packages/server/lib/util/print-run.ts#L299-L395 -const renderSummaryTable = ( - results: CypressCommandLine.CypressRunResult -): void => { - const runs = results.runs ?? []; - - console.log(""); - - terminal.divider("="); - - console.log(""); - - terminal.header("Run Finished", { - color: ["reset"], - }); - - if (runs.length > 0) { - const colAligns: HorizontalAlignment[] = [ - "left", - "left", - "right", - "right", - "right", - "right", - "right", - "right", - "right", - "right", - ]; - const colWidths = [3, 29, 11, 7, 9, 9, 7, 7, 9, 9]; - - const table1 = terminal.table({ - colAligns, - colWidths, - type: "noBorder", - head: [ - "", - gray("Spec"), - "", - gray("Tests"), - gray("Passing"), - gray("Failing"), - gray("Flaky"), - gray("Quar."), - gray("Pending"), - gray("Skipped"), - ], - }); - - const table2 = terminal.table({ - colAligns, - colWidths, - type: "border", - }); - - const table3 = terminal.table({ - colAligns, - colWidths, - type: "noBorder", - head: formatFooterSummary(results), - }); - - runs.forEach((run): number => { - const reporterStats = reporterStatsOrDefault(run); - - const ms = duration.format(run.stats.wallClockDuration ?? 0); - - const formattedSpec = formatPath( - run.spec.relativeToCommonRoot, - getWidth(table2, 1) - ); - - if (run.skippedSpec) { - return table2.push([ - "-", - formattedSpec, - color("SKIPPED", "gray"), - "-", - "-", - "-", - "-", - "-", - "-", - "-", - ]); - } - - return table2.push([ - formatSymbolSummary( - Math.max( - reporterStats.failures + (reporterStats.unquarantinedFlakes ?? 0), - (reporterStats.unquarantinedSkipped ?? 0) > 0 ? 1 : 0 - ) - ), - formattedSpec, - color(ms, "gray"), - colorIf(run.stats.tests, "reset"), - colorIf(reporterStats.passes, "green"), - colorIf(reporterStats.failures, "red"), - colorIf(reporterStats.unquarantinedFlakes ?? 0, "yellow"), - colorIf( - (reporterStats.quarantinedFailures ?? 0) + - (reporterStats.quarantinedFlakes ?? 0) + - (reporterStats.quarantinedPending ?? 0), - "magenta" - ), - colorIf(reporterStats.unquarantinedPending ?? 0, "cyan"), - colorIf( - (reporterStats.quarantinedSkipped ?? 0) + - (reporterStats.unquarantinedSkipped ?? 0), - "blue" - ), - ]); - }); - - console.log(""); - console.log(""); - console.log(terminal.renderTables(table1, table2, table3)); - console.log(""); - } -}; - export class UnflakableCypressPlugin { private readonly apiKey: string; private readonly manifest: TestSuiteManifest | null; private readonly repoRoot: string; private readonly unflakableConfig: UnflakableConfig; - private specs: Cypress.Spec[] | null = null; + private specs: + | (Cypress.Spec & { + relativeToCommonRoot: string; + })[] + | null = null; private specIndex = 0; private supportFilePath: string | null = null; @@ -545,7 +427,12 @@ ${ const debug = baseDebug.extend("beforeRun"); debug("Received beforeRun event"); - this.specs = specs ?? []; + this.specs = + (specs as + | (Cypress.Spec & { + relativeToCommonRoot: string; + })[] + | undefined) ?? []; displayRunStarting({ // Some of these public types seem wrong since Cypress passes the same values to the // `before:run` event as it does to displayRunStarting(): @@ -585,8 +472,8 @@ ${ }); } - if (results.status === "finished") { - renderSummaryTable(results); + if (results.status !== "failed") { + this.renderSummaryTable(results); const testRuns = results.runs.flatMap(({ spec, tests }) => // Contrary to Cypress's TypeScript typing, tests can be `null` when the specs fail to @@ -657,12 +544,136 @@ ${ // This can be set by Cypress Cloud. if (!results.skippedSpec) { - displayResults( - spec as Cypress.Spec & { - relativeToCommonRoot: string; - }, - results - ); + displayResults(this.relativeToCommonRoot(spec), results); + } + }; + + private relativeToCommonRoot = (spec: Cypress.Spec): string => + (this.specs ?? []).find((fullSpec) => fullSpec.absolute === spec.absolute) + ?.relativeToCommonRoot ?? spec.name; + + // Adapted from: + // https://github.com/cypress-io/cypress/blob/19e091d0bc2d1f4e6a6e62d2f81ea6a2f60d531a/packages/server/lib/util/print-run.ts#L299-L395 + private renderSummaryTable = ( + results: CypressCommandLine.CypressRunResult + ): void => { + const runs = results.runs ?? []; + + console.log(""); + + terminal.divider("="); + + console.log(""); + + terminal.header("Run Finished", { + color: ["reset"], + }); + + if (runs.length > 0) { + const colAligns: HorizontalAlignment[] = [ + "left", + "left", + "right", + "right", + "right", + "right", + "right", + "right", + "right", + "right", + ]; + const colWidths = [3, 29, 11, 7, 9, 9, 7, 7, 9, 9]; + + const table1 = terminal.table({ + colAligns, + colWidths, + type: "noBorder", + head: [ + "", + gray("Spec"), + "", + gray("Tests"), + gray("Passing"), + gray("Failing"), + gray("Flaky"), + gray("Quar."), + gray("Pending"), + gray("Skipped"), + ], + }); + + const table2 = terminal.table({ + colAligns, + colWidths, + type: "border", + }); + + const table3 = terminal.table({ + colAligns, + colWidths, + type: "noBorder", + head: formatFooterSummary(results), + }); + + runs.forEach((run): number => { + const reporterStats = reporterStatsOrDefault(run); + + const ms = duration.format( + run.stats.wallClockDuration ?? run.stats.duration ?? 0 + ); + + const formattedSpec = formatPath( + this.relativeToCommonRoot(run.spec), + getWidth(table2, 1) + ); + + if (run.skippedSpec) { + return table2.push([ + "-", + formattedSpec, + color("SKIPPED", "gray"), + "-", + "-", + "-", + "-", + "-", + "-", + "-", + ]); + } + + return table2.push([ + formatSymbolSummary( + Math.max( + reporterStats.failures + (reporterStats.unquarantinedFlakes ?? 0), + (reporterStats.unquarantinedSkipped ?? 0) > 0 ? 1 : 0 + ) + ), + formattedSpec, + color(ms, "gray"), + colorIf(run.stats.tests, "reset"), + colorIf(reporterStats.passes, "green"), + colorIf(reporterStats.failures, "red"), + colorIf(reporterStats.unquarantinedFlakes ?? 0, "yellow"), + colorIf( + (reporterStats.quarantinedFailures ?? 0) + + (reporterStats.quarantinedFlakes ?? 0) + + (reporterStats.quarantinedPending ?? 0), + "magenta" + ), + colorIf(reporterStats.unquarantinedPending ?? 0, "cyan"), + colorIf( + (reporterStats.quarantinedSkipped ?? 0) + + (reporterStats.unquarantinedSkipped ?? 0), + "blue" + ), + ]); + }); + + console.log(""); + console.log(""); + console.log(terminal.renderTables(table1, table2, table3)); + console.log(""); } }; } diff --git a/packages/cypress-plugin/src/reporter.ts b/packages/cypress-plugin/src/reporter.ts index 8e3551a..3475e69 100644 --- a/packages/cypress-plugin/src/reporter.ts +++ b/packages/cypress-plugin/src/reporter.ts @@ -22,6 +22,7 @@ import deepEqual from "deep-equal"; import { printWarning } from "./utils"; import { ReporterStats } from "./reporter-common"; import styles, { ForegroundColor } from "ansi-styles"; +import escapeStringRegexp from "escape-string-regexp"; const debug = _debug("unflakable:reporter"); @@ -80,6 +81,30 @@ const stringifyDiffObjs = (err: Mocha.Error): void => { } }; +// Mocha test fail events for hook failures have the hook name in the title. However, we need to +// extract the title of the affected test, which we do using a regex below. +const extractTestTitleFromFailedHookTitle = ( + failedHookTitle: string, + currentHookName: string +): string => { + // Mocha concatenates the hook name to the test title here: + // https://github.com/mochajs/mocha/blob/0be3f78491bbbcdc4dcea660ee7bfd557a225d9c/lib/runner.js#L331 + const matches = RegExp( + `^${escapeStringRegexp(currentHookName)} for "(.*)"$` + ).exec(failedHookTitle); + if (matches !== null) { + debug( + `extracted test title "${matches[1]}" from failed hook title "${failedHookTitle}"` + ); + return matches[1]; + } else { + debug( + `could not extract test title from failed hook title "${failedHookTitle}" with current hook name "${currentHookName}"` + ); + return failedHookTitle; + } +}; + const mergeTestFailure = ( src: { err?: Mocha.Error }, dest: { err?: Mocha.Error } @@ -93,42 +118,45 @@ const mergeTestFailure = ( // We already recorded this test failure, but it has multiple exceptions, so we need to track // those. if (dest.err !== undefined) { - dest.err.multiple = [ - src.err, - ...(src.err.multiple ?? []), - ...(dest.err.multiple ?? []), - ]; + if (!(dest.err.multiple ?? []).some((err) => deepEqual(src.err, err))) { + dest.err.multiple = [ + src.err, + ...(src.err.multiple ?? []), + ...(dest.err.multiple ?? []), + ]; + } } else { dest.err = src.err; } } }; +const addHookNameToError = (hookName: string, err: Mocha.Error): void => { + const hookMsg = `${hookName} failed:\n`; + if (err.message !== undefined) { + // This is consumed by formatErrorMsgAndStack(). + if (err.stack !== undefined) { + const indexIntoStackOrMsg = err.stack.indexOf(err.message); + if (indexIntoStackOrMsg !== -1) { + err.stack = + err.stack.slice(0, indexIntoStackOrMsg) + + hookMsg + + err.stack.slice(indexIntoStackOrMsg); + } + } + err.message = hookMsg + err.message; + } +}; + // Returns true if a new test was added, or false if the test failure was merged with an existing // test. const pushOrMergeFailedTest = ( failedTests: TestWithError[], - test: T & { hookName?: string } + test: T, + titlePath: TestTitle ): boolean => { const currentRetry = currentTestRetry(test); - if (test.err !== undefined && test.hookName !== undefined) { - const hookMsg = `"${test.hookName}" hook failed:\n`; - if (test.err.message !== undefined) { - // This is consumed by formatErrorMsgAndStack(). - if (test.err.stack !== undefined) { - const indexIntoStackOrMsg = test.err.stack.indexOf(test.err.message); - if (indexIntoStackOrMsg !== -1) { - test.err.stack = - test.err.stack.slice(0, indexIntoStackOrMsg) + - hookMsg + - test.err.stack.slice(indexIntoStackOrMsg); - } - } - test.err.message = hookMsg + test.err.message; - } - } - // NB: There's an edge case involving tests that have multiple errors. Ordinarily, onTestFail() // doesn't get called until the final retry fails. However, if multiple errors are thrown by // a single test (see https://github.com/mochajs/mocha/pull/4033), onTestFail() will get called @@ -139,14 +167,14 @@ const pushOrMergeFailedTest = ( // guaranteed (especially if tests run in parallel) due to the async nature of Cypress. .find( (existingTestFailure) => - existingTestFailure.test.id === test.id && + deepEqual(existingTestFailure.titlePath, titlePath) && currentTestRetry(existingTestFailure.test) === currentRetry ); if (existingTestFailure === undefined) { // At this point, it's too early to determine whether the test is a flake or a failure, which // depends on whether any of the retries will pass. - failedTests.push({ test, err: test.err }); + failedTests.push({ test, titlePath, err: test.err }); return true; } else { mergeTestFailure(test, existingTestFailure); @@ -260,17 +288,18 @@ const reportFailedAttempt = ({ ); }; +type TestTitle = string[]; + // Cypress mutates the runnable in response to Mocha events, which can overwrite our modifications // to the test's `err` field (i.e., its `multiple` field). We store a separate reference to the // error to prevent this. type TestWithError = { // Omit the `err` field from the type so that we don't accidentally use it. test: Omit; + titlePath: TestTitle; err: Mocha.Error | undefined; }; -type TestTitle = string[]; - // This reporter is a quarantine- and retry-aware TypeScript adaptation of Mocha's spec reporter: // https://github.com/mochajs/mocha/blob/ccee5f1b37bb405b81814daa35c63801cad20b4d/lib/reporters/spec.js export default class UnflakableSpecReporter extends reporters.Base { @@ -280,7 +309,9 @@ export default class UnflakableSpecReporter extends reporters.Base { private readonly manifest: TestSuiteManifest | null; private readonly posixTestFilename: string | null; - private indents = 0; + private currentSuiteTitles: string[] = []; + private currentHookName: string | null = null; + private lastHookName: string | null = null; // Used for computing the set of skipped tests from each suite. private nonSkippedTestJsonTitlePaths: Set = new Set(); @@ -337,6 +368,10 @@ export default class UnflakableSpecReporter extends reporters.Base { runner.on(Runner.constants.EVENT_SUITE_BEGIN, this.onSuiteBegin.bind(this)); runner.on(Runner.constants.EVENT_SUITE_END, this.onSuiteEnd.bind(this)); + runner.on(Runner.constants.EVENT_HOOK_BEGIN, this.onHookBegin.bind(this)); + runner.on(Runner.constants.EVENT_HOOK_END, this.onHookEnd.bind(this)); + + runner.on(Runner.constants.EVENT_TEST_BEGIN, this.onTestBegin.bind(this)); runner.on(Runner.constants.EVENT_TEST_FAIL, this.onTestFail.bind(this)); runner.on( Runner.constants.EVENT_TEST_PENDING, @@ -359,7 +394,8 @@ export default class UnflakableSpecReporter extends reporters.Base { ? styles.color[c].open + str + styles.color[c].close : str; - private indent = (): string => Array(this.indents).join(" "); + private indent = (): string => + Array(this.currentSuiteTitles.length).join(" "); private isQuarantined = (titlePath: TestTitle): boolean => { // For ignore_failure (and skip_tests if somehow the quarantined tests still executed), ignore @@ -399,15 +435,16 @@ export default class UnflakableSpecReporter extends reporters.Base { // in its parent suite to find the original test title. const suiteTest = (lastTestAttempt.test.parent?.tests as CypressTest[] | undefined)?.find( - (suiteTest) => suiteTest.id === lastTestAttempt.test.id + (suiteTest) => + deepEqual(suiteTest.titlePath(), lastTestAttempt.titlePath) ) ?? lastTestAttempt.test; const testTitle = formatFailedTestTitle(suiteTest.titlePath()); const failedAttempts: FailedTestAttempt[] = [ ...this.retriedTests - .filter( - (retriedTest) => retriedTest.test.id === lastTestAttempt.test.id + .filter((retriedTest) => + deepEqual(retriedTest.titlePath, lastTestAttempt.titlePath) ) .map((retriedTest) => ({ currentRetry: currentTestRetry(retriedTest.test), @@ -661,7 +698,7 @@ export default class UnflakableSpecReporter extends reporters.Base { this.specTests.push(test.titlePath()); }); - this.indents++; + this.currentSuiteTitles.push(suite.title); console.log(reporters.Base.color("suite", this.indent() + suite.title)); }; @@ -672,8 +709,8 @@ export default class UnflakableSpecReporter extends reporters.Base { }\` root=${String(suite.root)}` ); - this.indents--; - if (this.indents === 1) { + this.currentSuiteTitles.pop(); + if (this.currentSuiteTitles.length === 1) { console.log(); } }; @@ -686,6 +723,7 @@ export default class UnflakableSpecReporter extends reporters.Base { // test.currentRetry() < test.retries(). In this case, onTestFail() will get called with the // second exception, followed by onTestEnd() getting called with the first exception. After // some testing, it doesn't seem like there can ever be more than two exceptions. + // For hook failures, this gets called with this.currentHookName still set. private onTestFail = (test: CypressTest, _err?: Mocha.Error): void => { debug( `onTestFail [${test.state ?? "no state"}] ${test.title}: ${ @@ -693,23 +731,30 @@ export default class UnflakableSpecReporter extends reporters.Base { }` ); - // If the failure occurred in a before/after hook, `test`'s name includes the name of the hook. - // However, we want quarantine to be based on the test itself, so we look up the test in its - // parent suite to find the original test title. - const suiteTest = - (test.parent?.tests as CypressTest[] | undefined)?.find( - (suiteTest) => suiteTest.id === test.id - ) ?? test; + if (test.err !== undefined && this.currentHookName !== null) { + addHookNameToError(this.currentHookName, test.err); + } + + const titlePath = + this.currentHookName !== null + ? [ + ...test.titlePath().slice(0, -1), + extractTestTitleFromFailedHookTitle( + test.title, + this.currentHookName + ), + ] + : test.titlePath(); // Case (2) above, or case (1) when Cypress won't retry due to a before/after all hook failing. // Unfortunately, we can't determine at this point whether the test will be retried or not, so // we assume that it will, and then handle the edge case in onTestEnd(). if (currentTestRetry(test) < testRetries(test)) { - this.retriedTests.push({ test, err: test.err }); - } else if (this.isQuarantined(suiteTest.titlePath())) { - pushOrMergeFailedTest(this.quarantinedFailures, test); + pushOrMergeFailedTest(this.retriedTests, test, titlePath); + } else if (this.isQuarantined(titlePath)) { + pushOrMergeFailedTest(this.quarantinedFailures, test, titlePath); } else { - pushOrMergeFailedTest(this.unquarantinedFailures, test); + pushOrMergeFailedTest(this.unquarantinedFailures, test, titlePath); } // NB: We don't print any output in this function since, for non-final-attempts, it only gets @@ -723,9 +768,17 @@ export default class UnflakableSpecReporter extends reporters.Base { const isFlaky = currentRetry > 0; const isQuarantined = isFlaky && this.isQuarantined(test.titlePath()); if (isFlaky && isQuarantined) { - this.quarantinedFlakes.push({ test, err: test.err }); + this.quarantinedFlakes.push({ + test, + titlePath: test.titlePath(), + err: test.err, + }); } else if (isFlaky) { - this.unquarantinedFlakes.push({ test, err: test.err }); + this.unquarantinedFlakes.push({ + test, + titlePath: test.titlePath(), + err: test.err, + }); } // Cypress overrides the behavior of the Mocha spec reporter (see @@ -759,9 +812,9 @@ export default class UnflakableSpecReporter extends reporters.Base { const isQuarantined = this.isQuarantined(test.titlePath()); if (isQuarantined) { - pushOrMergeFailedTest(this.quarantinedFailures, test); + pushOrMergeFailedTest(this.quarantinedFailures, test, test.titlePath()); } else { - pushOrMergeFailedTest(this.unquarantinedFailures, test); + pushOrMergeFailedTest(this.unquarantinedFailures, test, test.titlePath()); } const quarantinedFmt = ` ${reporters.Base.symbols.err} ${test.title} [failed, quarantined]`; @@ -780,9 +833,45 @@ export default class UnflakableSpecReporter extends reporters.Base { ); }; + private onHookBegin = (hook: CypressTest & { hookName: string }): void => { + debug( + `onHookBegin [${hook.state ?? "no state"}] ${hook.title}: ${ + hook.err?.message ?? "" + }` + ); + + this.currentHookName = hook.title; + }; + + private onHookEnd = (test: CypressTest): void => { + debug( + `onHookEnd [${test.state ?? "no state"}] ${test.title}: ${ + test.err?.message ?? "" + }` + ); + + this.lastHookName = this.currentHookName; + this.currentHookName = null; + }; + + private onTestBegin = (test: CypressTest): void => { + debug( + `onTestBegin [${test.state ?? "no state"}] ${test.title}: ${ + test.err?.message ?? "" + }` + ); + + // It's possible the previous hook failed without invoking onHookEnd(). Once a test starts + // running, we stop caring about previous hooks. + this.lastHookName = null; + this.currentHookName = null; + }; + // This handler gets called after the final attempt of each test, so it's where we determine the // test's overall outcome and whether it's quarantined. // NB: An `err` argument is never passed to this event handler. + // For hook failures, this gets called with this.currentHookName still set. The test title never + // has the hook name in it here. private onTestEnd = (test: CypressTest): void => { debug( `onTestEnd [${test.state ?? "no state"}] ${test.title}: ${ @@ -795,6 +884,10 @@ export default class UnflakableSpecReporter extends reporters.Base { if (test.state === "passed") { this.reportTestPassed(test); } else if (test.state === "failed") { + if (test.err !== undefined && this.currentHookName !== null) { + addHookNameToError(this.currentHookName, test.err); + } + const currentRetry = currentTestRetry(test); // Edge case: a test failed and won't be retried due to a before()/after() hook failing. In // this case, we already added the test to this.retriedTests in onTestFail() (where it was @@ -804,7 +897,7 @@ export default class UnflakableSpecReporter extends reporters.Base { if (currentRetry < testRetries(test)) { const existingTestFailureIdx = this.retriedTests.findIndex( (existingTestFailure) => - existingTestFailure.test.id === test.id && + deepEqual(existingTestFailure.titlePath, test.titlePath()) && currentTestRetry(existingTestFailure.test) === currentRetry ); if (existingTestFailureIdx !== -1) { @@ -875,6 +968,10 @@ export default class UnflakableSpecReporter extends reporters.Base { }; // NB: An `err` argument is never passed to this event handler. + // For hook failures, this gets called without this.currentHookName set because onHookEnd() gets + // called first. Instead, we use this.lastHookName. The test.hookName attribute is also set, which + // lets us determine if the error was caused by a hook. However, this doesn't include the full + // user-supplied hook title, so we instead rely on currentHookName/lastHookName. private onTestRetry = ( test: MochaEventTest & { hookName?: string } ): void => { @@ -884,11 +981,26 @@ export default class UnflakableSpecReporter extends reporters.Base { }` ); + const hookName = + test.hookName !== undefined + ? this.currentHookName ?? this.lastHookName + : null; + + const titlePath = [ + // The root suite has an empty title that needs to be skipped. + ...this.currentSuiteTitles.slice(1), + test.title, + ]; + + if (test.err !== undefined && hookName !== null) { + addHookNameToError(hookName, test.err); + } + // Edge case: onTestRetry() gets called twice if a test fails and its afterEach() hook also // fails. However, we don't want to print the failed attempt twice in that case. if ( - pushOrMergeFailedTest(this.retriedTests, test) || - test.hookName === undefined + pushOrMergeFailedTest(this.retriedTests, test, titlePath) || + hookName === null ) { console.log( this.indent() + diff --git a/packages/cypress-plugin/test/integration-input-esm/package.json b/packages/cypress-plugin/test/integration-input-esm/package.json index a455fa8..6a5dbfe 100644 --- a/packages/cypress-plugin/test/integration-input-esm/package.json +++ b/packages/cypress-plugin/test/integration-input-esm/package.json @@ -7,7 +7,7 @@ "@types/react": "^18.2.7", "@types/react-dom": "^18.2.4", "@unflakable/cypress-plugin": "workspace:^", - "cypress": "10 - 12", + "cypress": "11.2 - 13", "mocha": "=7.0.1", "mocha-junit-reporter": "^2.2.0", "process": "^0.11.10", diff --git a/packages/cypress-plugin/test/integration-input-manual/cypress/e2e/hook-fail.cy.js b/packages/cypress-plugin/test/integration-input-manual/cypress/e2e/hook-fail.cy.js index 3eca195..297f0bb 100644 --- a/packages/cypress-plugin/test/integration-input-manual/cypress/e2e/hook-fail.cy.js +++ b/packages/cypress-plugin/test/integration-input-manual/cypress/e2e/hook-fail.cy.js @@ -3,6 +3,7 @@ describe("describe block", () => { if (Cypress.env("SKIP_BEFORE_HOOK") === undefined) { before( + "before hook with title", /** * @param {Mocha.Done} done */ @@ -21,6 +22,7 @@ describe("describe block", () => { if (Cypress.env("SKIP_BEFORE_EACH_HOOK") === undefined) { beforeEach( + "beforeEach hook with title", /** * @param {Mocha.Done} done */ @@ -49,6 +51,7 @@ describe("describe block", () => { if (Cypress.env("SKIP_AFTER_EACH_HOOK") === undefined) { afterEach( + "afterEach hook with title", /** * @param {Mocha.Done} done */ @@ -67,6 +70,7 @@ describe("describe block", () => { if (Cypress.env("SKIP_AFTER_HOOK") === undefined) { after( + "after hook with title", /** * @param {Mocha.Done} done */ diff --git a/packages/cypress-plugin/test/integration-input-manual/package.json b/packages/cypress-plugin/test/integration-input-manual/package.json index d97f955..49515a8 100644 --- a/packages/cypress-plugin/test/integration-input-manual/package.json +++ b/packages/cypress-plugin/test/integration-input-manual/package.json @@ -9,7 +9,7 @@ "@types/react": "^18.2.7", "@types/react-dom": "^18.2.4", "@unflakable/cypress-plugin": "workspace:^", - "cypress": "10 - 12", + "cypress": "11.2 - 13", "cypress-multi-reporters": "^1.6.3", "mocha": "=7.0.1", "mocha-junit-reporter": "^2.2.0", diff --git a/packages/cypress-plugin/test/integration-input/cypress/component/hook-fail.cy.ts b/packages/cypress-plugin/test/integration-input/cypress/component/hook-fail.cy.ts index f37bae2..08c00f3 100644 --- a/packages/cypress-plugin/test/integration-input/cypress/component/hook-fail.cy.ts +++ b/packages/cypress-plugin/test/integration-input/cypress/component/hook-fail.cy.ts @@ -2,7 +2,7 @@ describe("describe block", () => { if (Cypress.env("SKIP_BEFORE_HOOK") === undefined) { - before((done: Mocha.Done) => { + before("before hook with title", (done: Mocha.Done) => { process.nextTick(() => { throw new Error("before Error #1"); }); @@ -15,7 +15,7 @@ describe("describe block", () => { } if (Cypress.env("SKIP_BEFORE_EACH_HOOK") === undefined) { - beforeEach((done: Mocha.Done) => { + beforeEach("beforeEach hook with title", (done: Mocha.Done) => { process.nextTick(() => { throw new Error("beforeEach Error #1"); }); @@ -37,7 +37,7 @@ describe("describe block", () => { it("should be skipped", () => undefined); if (Cypress.env("SKIP_AFTER_EACH_HOOK") === undefined) { - afterEach((done: Mocha.Done) => { + afterEach("afterEach hook with title", (done: Mocha.Done) => { process.nextTick(() => { throw new Error("afterEach Error #1"); }); @@ -50,7 +50,7 @@ describe("describe block", () => { } if (Cypress.env("SKIP_AFTER_HOOK") === undefined) { - after((done: Mocha.Done) => { + after("after hook with title", (done: Mocha.Done) => { process.nextTick(() => { throw new Error("after Error #1"); }); diff --git a/packages/cypress-plugin/test/integration-input/package.json b/packages/cypress-plugin/test/integration-input/package.json index 8722d91..6c81b9e 100644 --- a/packages/cypress-plugin/test/integration-input/package.json +++ b/packages/cypress-plugin/test/integration-input/package.json @@ -6,7 +6,7 @@ "@types/react": "^18.2.7", "@types/react-dom": "^18.2.4", "@unflakable/cypress-plugin": "workspace:^", - "cypress": "10 - 12", + "cypress": "11.2 - 13", "mocha": "=7.0.1", "mocha-junit-reporter": "^2.2.0", "process": "^0.11.10", diff --git a/packages/cypress-plugin/test/integration/package.json b/packages/cypress-plugin/test/integration/package.json index a36693b..6382d0b 100644 --- a/packages/cypress-plugin/test/integration/package.json +++ b/packages/cypress-plugin/test/integration/package.json @@ -6,13 +6,14 @@ "@unflakable/jest-plugin": "workspace:^", "@unflakable/js-api": "workspace:^", "@unflakable/plugins-common": "workspace:^", - "cypress": "10 - 12", + "cypress": "11.2 - 13", "debug": "^4.3.3", "escape-string-regexp": "^4.0.0", "jest": "^29.5.0", "jest-environment-node": "^29.5.0", "jest-expect-message": "^1.1.3", "mockttp": "^3.7.5", + "semver": "^7.5.4", "ts-jest": "^29.1.0", "typescript": "^4.9.5", "unflakable-test-common": "workspace:^" diff --git a/packages/cypress-plugin/test/integration/src/hook-failures.test.ts b/packages/cypress-plugin/test/integration/src/hook-failures.test.ts index 013f65b..06df22f 100644 --- a/packages/cypress-plugin/test/integration/src/hook-failures.test.ts +++ b/packages/cypress-plugin/test/integration/src/hook-failures.test.ts @@ -3,7 +3,7 @@ import { integrationTest, integrationTestSuite } from "./test-wrappers"; integrationTestSuite((mockBackend) => { - it("run should succeed when before() fails both tests are quarantined", (done) => + it("run should succeed when before() fails and both tests are quarantined", (done) => integrationTest( { params: { @@ -27,7 +27,7 @@ integrationTestSuite((mockBackend) => { done )); - it("run should succeed when beforeEach() fails both tests are quarantined", (done) => + it("run should succeed when beforeEach() fails and both tests are quarantined", (done) => integrationTest( { params: { @@ -53,7 +53,7 @@ integrationTestSuite((mockBackend) => { done )); - it("run should succeed when afterEach() fails both tests are quarantined", (done) => + it("run should succeed when afterEach() fails and both tests are quarantined", (done) => integrationTest( { params: { diff --git a/packages/cypress-plugin/test/integration/src/run-test-case.ts b/packages/cypress-plugin/test/integration/src/run-test-case.ts index 35ac129..5e9b235 100644 --- a/packages/cypress-plugin/test/integration/src/run-test-case.ts +++ b/packages/cypress-plugin/test/integration/src/run-test-case.ts @@ -33,6 +33,8 @@ import { AsyncTestError, spawnTestWithTimeout, } from "unflakable-test-common/dist/spawn"; +import cypressPackage from "cypress/package.json"; +import semverLt from "semver/functions/lt"; // Jest times out after 120 seconds, so we bail early here to allow time to print the // captured output before Jest kills the test. @@ -175,6 +177,12 @@ type ExpectedRunRecord = { attemptResults: TestAttemptResult[]; }; +// Cypress 13 removed per-attempt timing information. See: +// - https://github.com/cypress-io/cypress/issues/27732 +// - https://github.com/cypress-io/cypress/pull/27230 +// - https://github.com/cypress-io/cypress/issues/27390 +const expectPerAttemptTiming = semverLt(cypressPackage.version, "13.0.0"); + const expectSpecRuns = ( params: TestCaseParams, specNameStub: string, @@ -183,11 +191,15 @@ const expectSpecRuns = ( params.specNameStubs === undefined || params.specNameStubs.includes(specNameStub) ? expectedRunRecords.map(({ name, attemptResults }) => ({ - attempts: attemptResults.map((result) => ({ - start_time: expectExt.stringMatching(TIMESTAMP_REGEX), - duration_ms: expectExt.toBeAnInteger(), - result, - })), + attempts: attemptResults.map((result) => + expectPerAttemptTiming + ? { + start_time: expectExt.stringMatching(TIMESTAMP_REGEX), + duration_ms: expectExt.toBeAnInteger(), + result, + } + : { result } + ), filename: specRepoPath(params, specNameStub), name, })) diff --git a/packages/cypress-plugin/test/integration/src/verify-output.ts b/packages/cypress-plugin/test/integration/src/verify-output.ts index 1605924..92c79c7 100644 --- a/packages/cypress-plugin/test/integration/src/verify-output.ts +++ b/packages/cypress-plugin/test/integration/src/verify-output.ts @@ -842,212 +842,231 @@ const verifySpecOutputs = ( : "fail" : "skip"; - const hookFailTestFailure = { - attempts: Array.from( + const verifyHookFailSpecOutput = (errorLines: string[]): void => { + const hookFailTestFailure = { + attempts: Array.from( + { + length: + skipBeforeHook && + (!skipBeforeEachHook || !skipAfterEachHook || hookAndTestErrors) + ? expectedRetries + 1 + : 1, + }, + (_, idx) => ({ + titlePath: ["describe block", "should fail due to hook"], + attempt: + skipBeforeHook && + (!skipBeforeEachHook || !skipAfterEachHook || hookAndTestErrors) + ? { + attemptNum: idx + 1, + totalAttempts: expectedRetries + 1, + } + : undefined, + errorLines, + }) + ), + }; + const hookSkipTestFailure = { + attempts: [ + { + titlePath: ["describe block", "should be skipped"], + attempt: undefined, + errorLines: expectExt.arrayContaining([ + ...(!skipAfterHook + ? [ + '\x1B[0m\x1B[31m Error: "after all" hook: after hook with title failed:', + ] + : []), + `> after Error #1`, + ]), + }, + ], + }; + + verifySpecOutput( + params, + specOutputs, + "hook-fail", { - length: - skipBeforeHook && - (!skipBeforeEachHook || !skipAfterEachHook || hookAndTestErrors) - ? expectedRetries + 1 - : 1, - }, - (_, idx) => ({ - titlePath: ["describe block", "should fail due to hook"], - attempt: - skipBeforeHook && - (!skipBeforeEachHook || !skipAfterEachHook || hookAndTestErrors) - ? { - attemptNum: idx + 1, - totalAttempts: expectedRetries + 1, - } - : undefined, - errorLines: expectExt.arrayContaining([ - ...(hookAndTestErrors + ...EMPTY_REPORTER_OUTPUT_MATCH, + suitesAndTestAttempts: [ + "\x1B[0m describe block\x1B[0m", + ...(hookFailResult === "pass" ? [ expectExt.stringMatching( // eslint-disable-next-line no-control-regex - /^(?:\x1B\[0m)?\x1B\[31m {5}Error: test error\x1B\[0m\x1B\[90m$/ - ), - ] - : []), - ...(!skipBeforeHook || !skipBeforeEachHook || !skipAfterEachHook - ? [ - expectExt.stringMatching( new RegExp( - `^(?:\x1B\\[0m)?\x1B\\[31m {5}Error: "${ - !skipBeforeHook - ? "before all" - : !skipBeforeEachHook - ? "before each" - : "after each" - }" hook failed:$` + `^ {2}\\x1B\\[32m {2}${PASS_SYMBOL}\\x1B\\[0m\\x1B\\[90m should fail due to hook\\x1B\\[0m\\x1B\\[(?:33|90)m \\([0-9]+.+?\\)\\x1B\\[0m$` ) ), + ] + : Array.from( + { + length: + skipBeforeHook && + (!skipBeforeEachHook || + !skipAfterEachHook || + hookAndTestErrors) + ? expectedRetries + 1 + : 1, + }, + (_, idx) => + // NB: The default Mocha reporter prints `"" hook for ""` as + // a sort of fake test name, but it uses the same test ID as the associated test. + // Dealing with multiple names for a single test leads to inconsistencies and also + // makes it hard to quarantine tests that fail due to hook failures. Instead, we + // just treat hook failures as failures of the associated test, and Cypress's error + // message already mentions the hook that failed. See: + // https://github.com/mochajs/mocha/blob/0be3f78491bbbcdc4dcea660ee7bfd557a225d9c/lib/runner.js#L332 + (hookFailResult === "quarantined" && + (!skipBeforeHook || idx === expectedRetries) + ? ` \x1B[35m ${FAIL_SYMBOL} should fail due to hook [failed, quarantined]\x1B[39m` + : ` \x1B[31m ${FAIL_SYMBOL} should fail due to hook\x1B[0m`) + + (skipBeforeHook && expectedRetries > 0 + ? `\x1B[33m (attempt ${idx + 1} of ${ + expectedRetries + 1 + })\x1B[0m` + : "") + )), + ...(hookSkipResult === "pass" + ? [ expectExt.stringMatching( + // eslint-disable-next-line no-control-regex new RegExp( - `^ *> ${ - !skipBeforeHook - ? "before" - : !skipBeforeEachHook - ? "beforeEach" - : "afterEach" - } Error #1$` + `^ {2}\\x1B\\[32m {2}${PASS_SYMBOL}\\x1B\\[0m\\x1B\\[90m should be skipped\\x1B\\[0m\\x1B\\[(?:33|90)m \\([0-9]+.+?\\)\\x1B\\[0m$` ) ), - ...(multipleHookErrors - ? [ - expectExt.stringMatching( - new RegExp( - `^(?:\x1B\\[31m)?(?: {5})?(?:Error: )?${ - !skipBeforeHook - ? "before" - : !skipBeforeEachHook - ? "beforeEach" - : "afterEach" - } Error #2 \\(and Mocha's done\\(\\) called multiple times\\)$` - ) - ), - ] - : []), + ] + : hookSkipResult === "fail" + ? [` \x1B[31m ${FAIL_SYMBOL} should be skipped\x1B[0m`] + : hookSkipResult === "quarantined" + ? [ + ` \x1B[35m ${FAIL_SYMBOL} should be skipped [failed, quarantined]\x1B[39m`, ] : []), - ]), - }) - ), - }; - const hookSkipTestFailure = { - attempts: [ - { - titlePath: ["describe block", "should be skipped"], - attempt: undefined, - errorLines: expectExt.arrayContaining([ - ...(!skipAfterHook - ? ['\x1B[0m\x1B[31m Error: "after all" hook failed:'] - : []), - `> after Error #1`, - ]), + ], + passing: + (hookFailResult === "pass" ? 1 : 0) + + (hookSkipResult === "pass" ? 1 : 0), + failures: { + count: + (hookFailResult === "fail" ? 1 : 0) + + (hookSkipResult === "fail" ? 1 : 0), + tests: [ + ...(hookFailResult === "fail" ? [hookFailTestFailure] : []), + ...(hookSkipResult === "fail" ? [hookSkipTestFailure] : []), + ], + }, + quarantinedFailures: { + count: + (hookFailResult === "quarantined" ? 1 : 0) + + (hookSkipResult === "quarantined" ? 1 : 0), + tests: [ + ...(hookFailResult === "quarantined" ? [hookFailTestFailure] : []), + ...(hookSkipResult === "quarantined" ? [hookSkipTestFailure] : []), + ], + }, + skipped: { + count: hookSkipResult === "skip" ? 1 : 0, + tests: + hookSkipResult === "skip" + ? [ + { + titlePath: ["describe block", "should be skipped"], + isQuarantined: quarantineHookSkip, + }, + ] + : [], + }, }, - ], - }; - - verifySpecOutput( - params, - specOutputs, - "hook-fail", - { - ...EMPTY_REPORTER_OUTPUT_MATCH, - suitesAndTestAttempts: [ - "\x1B[0m describe block\x1B[0m", - ...(hookFailResult === "pass" - ? [ - expectExt.stringMatching( - // eslint-disable-next-line no-control-regex - new RegExp( - `^ {2}\\x1B\\[32m {2}${PASS_SYMBOL}\\x1B\\[0m\\x1B\\[90m should fail due to hook\\x1B\\[0m\\x1B\\[(?:33|90)m \\([0-9]+.+?\\)\\x1B\\[0m$` - ) - ), - ] - : Array.from( - { - length: - skipBeforeHook && - (!skipBeforeEachHook || - !skipAfterEachHook || - hookAndTestErrors) - ? expectedRetries + 1 - : 1, - }, - (_, idx) => - // NB: The default Mocha reporter prints `"" hook for ""` as a - // sort of fake test name, but it uses the same test ID as the associated test. - // Dealing with multiple names for a single test leads to inconsistencies and also - // makes it hard to quarantine tests that fail due to hook failures. Instead, we just - // treat hook failures as failures of the associated test, and Cypress's error - // message already mentions the hook that failed. See: - // https://github.com/mochajs/mocha/blob/0be3f78491bbbcdc4dcea660ee7bfd557a225d9c/lib/runner.js#L332 - (hookFailResult === "quarantined" && - (!skipBeforeHook || idx === expectedRetries) - ? ` \x1B[35m ${FAIL_SYMBOL} should fail due to hook [failed, quarantined]\x1B[39m` - : ` \x1B[31m ${FAIL_SYMBOL} should fail due to hook\x1B[0m`) + - (skipBeforeHook && expectedRetries > 0 - ? `\x1B[33m (attempt ${idx + 1} of ${ - expectedRetries + 1 - })\x1B[0m` - : "") - )), - ...(hookSkipResult === "pass" - ? [ - expectExt.stringMatching( - // eslint-disable-next-line no-control-regex - new RegExp( - `^ {2}\\x1B\\[32m {2}${PASS_SYMBOL}\\x1B\\[0m\\x1B\\[90m should be skipped\\x1B\\[0m\\x1B\\[(?:33|90)m \\([0-9]+.+?\\)\\x1B\\[0m$` - ) - ), - ] - : hookSkipResult === "fail" - ? [` \x1B[31m ${FAIL_SYMBOL} should be skipped\x1B[0m`] - : hookSkipResult === "quarantined" - ? [ - ` \x1B[35m ${FAIL_SYMBOL} should be skipped [failed, quarantined]\x1B[39m`, - ] - : []), - ], - passing: - (hookFailResult === "pass" ? 1 : 0) + - (hookSkipResult === "pass" ? 1 : 0), - failures: { - count: + { + ...EMPTY_RESULTS, + color: + hookFailResult !== "fail" && + hookSkipResult !== "fail" && + (hookSkipResult !== "skip" || quarantineHookSkip) + ? "pass" + : "fail", + numTests: 2, + numFailing: (hookFailResult === "fail" ? 1 : 0) + (hookSkipResult === "fail" ? 1 : 0), - tests: [ - ...(hookFailResult === "fail" ? [hookFailTestFailure] : []), - ...(hookSkipResult === "fail" ? [hookSkipTestFailure] : []), - ], - }, - quarantinedFailures: { - count: + numPassing: + (hookFailResult === "pass" ? 1 : 0) + + (hookSkipResult === "pass" ? 1 : 0), + numQuarantined: (hookFailResult === "quarantined" ? 1 : 0) + (hookSkipResult === "quarantined" ? 1 : 0), - tests: [ - ...(hookFailResult === "quarantined" ? [hookFailTestFailure] : []), - ...(hookSkipResult === "quarantined" ? [hookSkipTestFailure] : []), - ], - }, - skipped: { - count: hookSkipResult === "skip" ? 1 : 0, - tests: - hookSkipResult === "skip" - ? [ - { - titlePath: ["describe block", "should be skipped"], - isQuarantined: quarantineHookSkip, - }, - ] - : [], - }, - }, - { - ...EMPTY_RESULTS, - color: - hookFailResult !== "fail" && - hookSkipResult !== "fail" && - (hookSkipResult !== "skip" || quarantineHookSkip) - ? "pass" - : "fail", - numTests: 2, - numFailing: - (hookFailResult === "fail" ? 1 : 0) + - (hookSkipResult === "fail" ? 1 : 0), - numPassing: - (hookFailResult === "pass" ? 1 : 0) + - (hookSkipResult === "pass" ? 1 : 0), - numQuarantined: - (hookFailResult === "quarantined" ? 1 : 0) + - (hookSkipResult === "quarantined" ? 1 : 0), - numSkipped: hookSkipResult === "skip" ? 1 : 0, - } + numSkipped: hookSkipResult === "skip" ? 1 : 0, + } + ); + }; + + verifyHookFailSpecOutput( + expectExt.arrayContaining([ + ...(hookAndTestErrors + ? [ + expectExt.stringMatching( + // eslint-disable-next-line no-control-regex + /^(?:\x1B\[0m)?\x1B\[31m {5}Error: test error\x1B\[0m\x1B\[90m$/ + ), + ] + : []), + ...(!skipBeforeHook || !skipBeforeEachHook || !skipAfterEachHook + ? [ + expectExt.stringMatching( + new RegExp( + `^(?:\x1B\\[0m)?\x1B\\[31m {5}Error: ${ + !skipBeforeHook + ? '"before all" hook: before hook with title' + : !skipBeforeEachHook + ? '"before each" hook: beforeEach hook with title' + : '"after each" hook: afterEach hook with title' + } failed:$` + ) + ), + expectExt.stringMatching( + new RegExp( + `^ *> ${ + !skipBeforeHook + ? "before" + : !skipBeforeEachHook + ? "beforeEach" + : "afterEach" + } Error #1$` + ) + ), + ...(multipleHookErrors + ? [ + expectExt.stringMatching( + new RegExp( + `^(?:\x1B\\[31m)?(?: {5})?(?:Error: )?${ + !skipBeforeHook + ? "before" + : !skipBeforeEachHook + ? "beforeEach" + : "afterEach" + } Error #2 \\(and Mocha's done\\(\\) called multiple times\\)$` + ) + ), + ] + : []), + ] + : []), + ]) ); + if (!skipBeforeHook || !skipBeforeEachHook || !skipAfterEachHook) { + verifyHookFailSpecOutput( + // The error should contain the hook name and not the plain test name. This should catch bugs + // in which we fail to merge errors originating in hooks when we both onTestFail() and either + // onTestRetry()/onTestEnd() are called. + expectExt.not.arrayContaining([ + expectExt.stringMatching("should fail due to hook failed:"), + ]) + ); + } + verifySpecOutput( params, specOutputs, diff --git a/yarn.lock b/yarn.lock index ffb7e94..d805a5d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3216,12 +3216,13 @@ __metadata: cli-table3: 0.5.1 cosmiconfig: ^7.0.1 cross-env: ^7.0.3 - cypress: 10 - 12 + cypress: 11.2 - 13 cypress-multi-reporters: ^1.6.3 dayjs: ^1.10.4 debug: ^4.3.3 deep-equal: ^2.0.5 es6-promisify: ^7.0.0 + escape-string-regexp: ^4.0.0 jest: ^29.5.0 jest-environment-node: ^29.5.0 lodash: ^4.17.21 @@ -3240,7 +3241,7 @@ __metadata: widest-line: 3.1.0 yargs: ^17.7.2 peerDependencies: - cypress: 11.2 - 12 + cypress: 11.2 - 13 bin: cypress-unflakable: ./dist/main.js languageName: unknown @@ -4750,7 +4751,7 @@ __metadata: "@types/react": ^18.2.7 "@types/react-dom": ^18.2.4 "@unflakable/cypress-plugin": "workspace:^" - cypress: 10 - 12 + cypress: 11.2 - 13 mocha: =7.0.1 mocha-junit-reporter: ^2.2.0 process: ^0.11.10 @@ -4770,7 +4771,7 @@ __metadata: "@types/react": ^18.2.7 "@types/react-dom": ^18.2.4 "@unflakable/cypress-plugin": "workspace:^" - cypress: 10 - 12 + cypress: 11.2 - 13 cypress-multi-reporters: ^1.6.3 mocha: =7.0.1 mocha-junit-reporter: ^2.2.0 @@ -4792,7 +4793,7 @@ __metadata: "@types/react": ^18.2.7 "@types/react-dom": ^18.2.4 "@unflakable/cypress-plugin": "workspace:^" - cypress: 10 - 12 + cypress: 11.2 - 13 mocha: =7.0.1 mocha-junit-reporter: ^2.2.0 process: ^0.11.10 @@ -4813,13 +4814,14 @@ __metadata: "@unflakable/jest-plugin": "workspace:^" "@unflakable/js-api": "workspace:^" "@unflakable/plugins-common": "workspace:^" - cypress: 10 - 12 + cypress: 11.2 - 13 debug: ^4.3.3 escape-string-regexp: ^4.0.0 jest: ^29.5.0 jest-environment-node: ^29.5.0 jest-expect-message: ^1.1.3 mockttp: ^3.7.5 + semver: ^7.5.4 ts-jest: ^29.1.0 typescript: ^4.9.5 unflakable-test-common: "workspace:^" @@ -4838,7 +4840,7 @@ __metadata: languageName: node linkType: hard -"cypress@npm:10 - 12": +"cypress@npm:11.2 - 13": version: 12.14.0 resolution: "cypress@npm:12.14.0" dependencies: From 656b8abecbaaed8c1a9809168339d6ea401941ce Mon Sep 17 00:00:00 2001 From: "David A. Ramos" Date: Mon, 4 Sep 2023 18:16:40 -0700 Subject: [PATCH 26/53] [cypress] Bump @unflakable/cypress-plugin to 0.2.1 --- packages/cypress-plugin/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cypress-plugin/package.json b/packages/cypress-plugin/package.json index 0e4fb64..604db86 100644 --- a/packages/cypress-plugin/package.json +++ b/packages/cypress-plugin/package.json @@ -8,7 +8,7 @@ "bugs": "https://github.com/unflakable/unflakable-javascript/issues", "homepage": "https://unflakable.com", "license": "MIT", - "version": "0.2.0", + "version": "0.2.1", "exports": { ".": { "types": "./dist/index.d.ts", From 4ad3ece860cd0f19196012e7cd917f3066b57f53 Mon Sep 17 00:00:00 2001 From: "David A. Ramos" Date: Mon, 4 Sep 2023 18:18:04 -0700 Subject: [PATCH 27/53] Remove Twitter badges from READMEs --- README.md | 2 -- packages/cypress-plugin/README.md | 1 - packages/jest-plugin/README.md | 1 - packages/js-api/README.md | 1 - packages/plugins-common/README.md | 2 -- 5 files changed, 7 deletions(-) diff --git a/README.md b/README.md index 59e9c79..6ac8e99 100644 --- a/README.md +++ b/README.md @@ -4,8 +4,6 @@

-[![Twitter](https://img.shields.io/twitter/url?label=%40unflakable&style=social&url=https%3A%2F%2Ftwitter.com%2Funflakable)](https://twitter.com/unflakable) - # Official Unflakable Plugins for JavaScript This repository contains the official [Unflakable](https://unflakable.com) plugins for JavaScript diff --git a/packages/cypress-plugin/README.md b/packages/cypress-plugin/README.md index 2d64bc8..3818fe5 100644 --- a/packages/cypress-plugin/README.md +++ b/packages/cypress-plugin/README.md @@ -5,7 +5,6 @@

[![npm version](https://img.shields.io/npm/v/@unflakable/cypress-plugin.svg)](https://www.npmjs.com/package/@unflakable/cypress-plugin) -[![Twitter](https://img.shields.io/twitter/url?label=%40unflakable&style=social&url=https%3A%2F%2Ftwitter.com%2Funflakable)](https://twitter.com/unflakable) # Unflakable Plugin for Cypress diff --git a/packages/jest-plugin/README.md b/packages/jest-plugin/README.md index d8a7de2..38f4fd4 100644 --- a/packages/jest-plugin/README.md +++ b/packages/jest-plugin/README.md @@ -5,7 +5,6 @@

[![npm version](https://img.shields.io/npm/v/@unflakable/jest-plugin.svg)](https://www.npmjs.com/package/@unflakable/jest-plugin) -[![Twitter](https://img.shields.io/twitter/url?label=%40unflakable&style=social&url=https%3A%2F%2Ftwitter.com%2Funflakable)](https://twitter.com/unflakable) # Unflakable Plugin for Jest diff --git a/packages/js-api/README.md b/packages/js-api/README.md index 6479c9f..f106ae7 100644 --- a/packages/js-api/README.md +++ b/packages/js-api/README.md @@ -5,7 +5,6 @@

[![npm version](https://img.shields.io/npm/v/@unflakable/js-api.svg)](https://www.npmjs.com/package/@unflakable/js-api) -[![Twitter](https://img.shields.io/twitter/url?label=%40unflakable&style=social&url=https%3A%2F%2Ftwitter.com%2Funflakable)](https://twitter.com/unflakable) # Unflakable JavaScript API diff --git a/packages/plugins-common/README.md b/packages/plugins-common/README.md index 16369e5..f9b835d 100644 --- a/packages/plugins-common/README.md +++ b/packages/plugins-common/README.md @@ -4,8 +4,6 @@

-[![Twitter](https://img.shields.io/twitter/url?label=%40unflakable&style=social&url=https%3A%2F%2Ftwitter.com%2Funflakable)](https://twitter.com/unflakable) - # Unflakable JavaScript Plugins Common Library This package contains code to be shared between all the Unflakable JavaScript plugins. From 3d9679053b99b75cc65d5c1b58b5042ae0987232 Mon Sep 17 00:00:00 2001 From: "David A. Ramos" Date: Tue, 5 Sep 2023 13:38:58 -0700 Subject: [PATCH 28/53] [cypress] Increase CI timeout for Cypress on Linux to 80 minutes --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 1bff501..a7fbc3f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -103,7 +103,7 @@ jobs: cypress_linux_integration_tests: name: "Cypress ${{ matrix.cypress }} Linux Node ${{ matrix.node }} Integration Tests" runs-on: ubuntu-latest - timeout-minutes: 60 + timeout-minutes: 80 needs: # Don't incur the cost of the test matrix if the basic build fails. - check From 2b92f3a16d1427d10c058480fb13922c8fdac90f Mon Sep 17 00:00:00 2001 From: "David A. Ramos" Date: Tue, 5 Sep 2023 13:49:40 -0700 Subject: [PATCH 29/53] Bump @unflakable/js-api to 0.3.1 --- packages/js-api/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/js-api/package.json b/packages/js-api/package.json index d4347b7..d3a805e 100644 --- a/packages/js-api/package.json +++ b/packages/js-api/package.json @@ -8,7 +8,7 @@ "bugs": "https://github.com/unflakable/unflakable-javascript/issues", "homepage": "https://unflakable.com", "license": "MIT", - "version": "0.3.0", + "version": "0.3.1", "main": "dist/index.js", "types": "dist/index.d.ts", "exports": { From 6a2808a549d4d273e1e0b70ad4bc4d6a7591abf5 Mon Sep 17 00:00:00 2001 From: "David A. Ramos" Date: Thu, 7 Sep 2023 19:35:35 -0700 Subject: [PATCH 30/53] [cypress] Increase Windows CI timeout to 120 minutes Also treat failure to connect to Chrome DevTools as test-independent, which is potentially new error text in Cypress 13. --- .github/workflows/ci.yaml | 2 +- packages/cypress-plugin/test/integration/unflakable.js | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a7fbc3f..7b043e5 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -210,7 +210,7 @@ jobs: name: "Cypress ${{ matrix.cypress }} Windows Node ${{ matrix.node }} Integration Tests" runs-on: windows-2019 # Cypress on Windows is slowwww... - timeout-minutes: 90 + timeout-minutes: 120 needs: # Don't incur the cost of the test matrix if the basic build fails. - check diff --git a/packages/cypress-plugin/test/integration/unflakable.js b/packages/cypress-plugin/test/integration/unflakable.js index d24c20f..96914fb 100644 --- a/packages/cypress-plugin/test/integration/unflakable.js +++ b/packages/cypress-plugin/test/integration/unflakable.js @@ -9,6 +9,7 @@ module.exports = { /attempting to close the browser tab(?:(?!resetting server state).)*$/s, /Still waiting to connect to Edge, retrying in 1 second.*(?:Error: Test timed out after|All promises were rejected)/s, /There was an error reconnecting to the Chrome DevTools protocol\. Please restart the browser\./, + /Cypress failed to make a connection to the Chrome DevTools Protocol after retrying/, // When this error occurs, Cypress ends up printing the "Running: " header multiple times, // which the integration test parses as if that spec were in fact invoked multiple times. We // don't want the test itself to ignore multiple spec invocations since that could indicate a From deec7d332b0d98c442a2c4e2e755e600bbf27495 Mon Sep 17 00:00:00 2001 From: "David A. Ramos" Date: Fri, 8 Sep 2023 18:49:29 -0700 Subject: [PATCH 31/53] [cypress] Bump CI container for Real World App to Node 18 See https://github.com/cypress-io/cypress-realworld-app/pull/1396. --- .github/workflows/cypress-realworld-app.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cypress-realworld-app.yaml b/.github/workflows/cypress-realworld-app.yaml index eab82bc..63c169c 100644 --- a/.github/workflows/cypress-realworld-app.yaml +++ b/.github/workflows/cypress-realworld-app.yaml @@ -16,7 +16,7 @@ jobs: timeout-minutes: 15 runs-on: ubuntu-latest container: - image: cypress/browsers:node16.16.0-chrome105-ff104-edge + image: cypress/browsers:node-18.16.0-chrome-114.0.5735.133-1-ff-114.0.2-edge-114.0.1823.51-1 options: --user 1001 steps: - name: Check out latest cypress-io/cypress-realworld-app @@ -62,7 +62,7 @@ jobs: # tarballs. Otherwise, Yarn may install the previously cached version from an earlier build # with the same package version number. run: | - curl -Lo jq https://github.com/jqlang/jq/releases/download/jq-1.6/jq-linux64 + wget -O jq https://github.com/jqlang/jq/releases/download/jq-1.6/jq-linux64 chmod +x jq yarn cache clean @unflakable/js-api From 7a33438820a2e895dad39a3372401dbbdf97bde3 Mon Sep 17 00:00:00 2001 From: "David A. Ramos" Date: Sun, 10 Sep 2023 20:44:54 -0700 Subject: [PATCH 32/53] [cypress] Increase integration test timeout to 4 minutes --- packages/cypress-plugin/test/integration/jest.config.js | 2 +- packages/cypress-plugin/test/integration/src/run-test-case.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/cypress-plugin/test/integration/jest.config.js b/packages/cypress-plugin/test/integration/jest.config.js index eeb9323..5184100 100644 --- a/packages/cypress-plugin/test/integration/jest.config.js +++ b/packages/cypress-plugin/test/integration/jest.config.js @@ -13,6 +13,6 @@ module.exports = { setupFilesAfterEnv: ["jest-expect-message", "./src/matchers.ts"], testEnvironment: "node", // NB: This should be greater than TEST_TIMEOUT_MS used by the watchdog in runTestCase(). - testTimeout: 120000, + testTimeout: 240000, verbose: true, }; diff --git a/packages/cypress-plugin/test/integration/src/run-test-case.ts b/packages/cypress-plugin/test/integration/src/run-test-case.ts index 5e9b235..414f076 100644 --- a/packages/cypress-plugin/test/integration/src/run-test-case.ts +++ b/packages/cypress-plugin/test/integration/src/run-test-case.ts @@ -36,9 +36,9 @@ import { import cypressPackage from "cypress/package.json"; import semverLt from "semver/functions/lt"; -// Jest times out after 120 seconds, so we bail early here to allow time to print the +// Jest times out after 240 seconds, so we bail early here to allow time to print the // captured output before Jest kills the test. -const TEST_TIMEOUT_MS = 110000; +const TEST_TIMEOUT_MS = 230000; const userAgentRegex = new RegExp( "unflakable-js-api/(?:[-0-9.]|alpha|beta)+ unflakable-cypress-plugin/(?:[-0-9.]|alpha|beta)+ \\(Cypress [0-9]+\\.[0-9]+\\.[0-9]+; Node v[0-9]+\\.[0-9]+\\.[0-9]\\)" From 4dc4c6fe91fc7375624b3c015e3f35d38b1605af Mon Sep 17 00:00:00 2001 From: "David A. Ramos" Date: Sun, 10 Sep 2023 18:45:39 -0700 Subject: [PATCH 33/53] Bump mockttp to 3.9.2 This eliminates the transitive dependency on vm2, which is affected by CVE-2023-37466/GHSA-cchq-frgv-rjh5. Please note that these dependencies are only used for this repository's integration tests and are not part of the plugins packaged for end users. --- .../test/integration/package.json | 2 +- .../jest-plugin/test/integration/package.json | 2 +- packages/test-common/package.json | 2 +- yarn.lock | 322 +++++++----------- 4 files changed, 123 insertions(+), 205 deletions(-) diff --git a/packages/cypress-plugin/test/integration/package.json b/packages/cypress-plugin/test/integration/package.json index 6382d0b..36a0a97 100644 --- a/packages/cypress-plugin/test/integration/package.json +++ b/packages/cypress-plugin/test/integration/package.json @@ -12,7 +12,7 @@ "jest": "^29.5.0", "jest-environment-node": "^29.5.0", "jest-expect-message": "^1.1.3", - "mockttp": "^3.7.5", + "mockttp": "^3.9.2", "semver": "^7.5.4", "ts-jest": "^29.1.0", "typescript": "^4.9.5", diff --git a/packages/jest-plugin/test/integration/package.json b/packages/jest-plugin/test/integration/package.json index c80c811..26d5f5a 100644 --- a/packages/jest-plugin/test/integration/package.json +++ b/packages/jest-plugin/test/integration/package.json @@ -16,7 +16,7 @@ "jest-environment-node": "25.1.0 - 29", "jest-get-type": "25.1.0 - 29", "jest-matcher-utils": "25.1.0 - 29", - "mockttp": "^3.7.5", + "mockttp": "^3.9.2", "semver": "^7.5.4", "tmp": "^0.2.1", "typescript": "^4.9.5", diff --git a/packages/test-common/package.json b/packages/test-common/package.json index f1b4508..548d460 100644 --- a/packages/test-common/package.json +++ b/packages/test-common/package.json @@ -6,7 +6,7 @@ "debug": "^4.3.3", "deep-equal": "^2.0.5", "expect": "25.1.0 - 29", - "mockttp": "^3.7.5", + "mockttp": "^3.9.2", "simple-git": "^3.16.0", "tree-kill": "^1.2.2" }, diff --git a/yarn.lock b/yarn.lock index d805a5d..d4d21d6 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2653,13 +2653,6 @@ __metadata: languageName: node linkType: hard -"@tootallnate/once@npm:1": - version: 1.1.2 - resolution: "@tootallnate/once@npm:1.1.2" - checksum: e1fb1bbbc12089a0cb9433dc290f97bddd062deadb6178ce9bcb93bb7c1aecde5e60184bc7065aec42fe1663622a213493c48bbd4972d931aae48315f18e1be9 - languageName: node - linkType: hard - "@tootallnate/once@npm:2": version: 2.0.0 resolution: "@tootallnate/once@npm:2.0.0" @@ -2667,6 +2660,13 @@ __metadata: languageName: node linkType: hard +"@tootallnate/quickjs-emscripten@npm:^0.23.0": + version: 0.23.0 + resolution: "@tootallnate/quickjs-emscripten@npm:0.23.0" + checksum: c350a2947ffb80b22e14ff35099fd582d1340d65723384a0fd0515e905e2534459ad2f301a43279a37308a27c99273c932e64649abd57d0bb3ca8c557150eccc + languageName: node + linkType: hard + "@tsconfig/node10@npm:^1.0.7": version: 1.0.8 resolution: "@tsconfig/node10@npm:1.0.8" @@ -3511,14 +3511,14 @@ __metadata: languageName: node linkType: hard -"acorn-walk@npm:^8.1.1, acorn-walk@npm:^8.2.0": +"acorn-walk@npm:^8.1.1": version: 8.2.0 resolution: "acorn-walk@npm:8.2.0" checksum: 1715e76c01dd7b2d4ca472f9c58968516a4899378a63ad5b6c2d668bba8da21a71976c14ec5f5b75f887b6317c4ae0b897ab141c831d741dc76024d8745f1ad1 languageName: node linkType: hard -"acorn@npm:^8.4.1, acorn@npm:^8.5.0, acorn@npm:^8.7.0, acorn@npm:^8.7.1, acorn@npm:^8.8.0": +"acorn@npm:^8.4.1, acorn@npm:^8.5.0, acorn@npm:^8.7.1, acorn@npm:^8.8.0": version: 8.8.2 resolution: "acorn@npm:8.8.2" bin: @@ -3536,6 +3536,15 @@ __metadata: languageName: node linkType: hard +"agent-base@npm:^7.0.2, agent-base@npm:^7.1.0": + version: 7.1.0 + resolution: "agent-base@npm:7.1.0" + dependencies: + debug: ^4.3.4 + checksum: f7828f991470a0cc22cb579c86a18cbae83d8a3cbed39992ab34fc7217c4d126017f1c74d0ab66be87f71455318a8ea3e757d6a37881b8d0f2a2c6aa55e5418f + languageName: node + linkType: hard + "agentkeepalive@npm:^4.2.1": version: 4.2.1 resolution: "agentkeepalive@npm:4.2.1" @@ -3808,7 +3817,7 @@ __metadata: languageName: node linkType: hard -"ast-types@npm:^0.13.2": +"ast-types@npm:^0.13.4": version: 0.13.4 resolution: "ast-types@npm:0.13.4" dependencies: @@ -4024,6 +4033,13 @@ __metadata: languageName: node linkType: hard +"basic-ftp@npm:^5.0.2": + version: 5.0.3 + resolution: "basic-ftp@npm:5.0.3" + checksum: 8b04e88eb85a64de9311721bb0707c9cd70453eefdd854cab85438e6f46fb6c597ddad57ed1acf0a9ede3c677b14e657f51051688a5f23d6f3ea7b5d9073b850 + languageName: node + linkType: hard + "bcrypt-pbkdf@npm:^1.0.0": version: 1.0.2 resolution: "bcrypt-pbkdf@npm:1.0.2" @@ -4820,7 +4836,7 @@ __metadata: jest: ^29.5.0 jest-environment-node: ^29.5.0 jest-expect-message: ^1.1.3 - mockttp: ^3.7.5 + mockttp: ^3.9.2 semver: ^7.5.4 ts-jest: ^29.1.0 typescript: ^4.9.5 @@ -4901,10 +4917,10 @@ __metadata: languageName: node linkType: hard -"data-uri-to-buffer@npm:3": - version: 3.0.1 - resolution: "data-uri-to-buffer@npm:3.0.1" - checksum: c59c3009686a78c071806b72f4810856ec28222f0f4e252aa495ec027ed9732298ceea99c50328cf59b151dd34cbc3ad6150bbb43e41fc56fa19f48c99e9fc30 +"data-uri-to-buffer@npm:^5.0.1": + version: 5.0.1 + resolution: "data-uri-to-buffer@npm:5.0.1" + checksum: 10958f89c0047b84bd86d572b6b77c9bf238ebe7b55a9a9ab04c90fbf5ab1881783b72e31dc0febdffd30ec914930244f2f728e3629bb8911d922baba129426f languageName: node linkType: hard @@ -4991,7 +5007,7 @@ __metadata: languageName: node linkType: hard -"deep-is@npm:^0.1.3, deep-is@npm:~0.1.3": +"deep-is@npm:^0.1.3": version: 0.1.4 resolution: "deep-is@npm:0.1.4" checksum: edb65dd0d7d1b9c40b2f50219aef30e116cedd6fc79290e740972c132c09106d2e80aa0bc8826673dd5a00222d4179c84b36a790eef63a4c4bca75a37ef90804 @@ -5022,15 +5038,14 @@ __metadata: languageName: node linkType: hard -"degenerator@npm:^3.0.2": - version: 3.0.4 - resolution: "degenerator@npm:3.0.4" +"degenerator@npm:^5.0.0": + version: 5.0.1 + resolution: "degenerator@npm:5.0.1" dependencies: - ast-types: ^0.13.2 - escodegen: ^1.8.1 - esprima: ^4.0.0 - vm2: ^3.9.17 - checksum: 99c27c9456095e32c4f6e01091d2b5c249f246b574487c52bca571e1e586b02d4b74a0ea7f22f30cc953c914383d02e2038d7d476a22f2704a8c1e88b671007d + ast-types: ^0.13.4 + escodegen: ^2.1.0 + esprima: ^4.0.1 + checksum: a64fa39cdf6c2edd75188157d32338ee9de7193d7dbb2aeb4acb1eb30fa4a15ed80ba8dae9bd4d7b085472cf174a5baf81adb761aaa8e326771392c922084152 languageName: node linkType: hard @@ -5416,14 +5431,13 @@ __metadata: languageName: node linkType: hard -"escodegen@npm:^1.8.1": - version: 1.14.3 - resolution: "escodegen@npm:1.14.3" +"escodegen@npm:^2.1.0": + version: 2.1.0 + resolution: "escodegen@npm:2.1.0" dependencies: esprima: ^4.0.1 - estraverse: ^4.2.0 + estraverse: ^5.2.0 esutils: ^2.0.2 - optionator: ^0.8.1 source-map: ~0.6.1 dependenciesMeta: source-map: @@ -5431,7 +5445,7 @@ __metadata: bin: escodegen: bin/escodegen.js esgenerate: bin/esgenerate.js - checksum: 381cdc4767ecdb221206bbbab021b467bbc2a6f5c9a99c9e6353040080bdd3dfe73d7604ad89a47aca6ea7d58bc635f6bd3fbc8da9a1998e9ddfa8372362ccd0 + checksum: 096696407e161305cd05aebb95134ad176708bc5cb13d0dcc89a5fcbb959b8ed757e7f2591a5f8036f8f4952d4a724de0df14cd419e29212729fa6df5ce16bf6 languageName: node linkType: hard @@ -5646,7 +5660,7 @@ __metadata: languageName: node linkType: hard -"estraverse@npm:^4.1.1, estraverse@npm:^4.2.0": +"estraverse@npm:^4.1.1": version: 4.3.0 resolution: "estraverse@npm:4.3.0" checksum: a6299491f9940bb246124a8d44b7b7a413a8336f5436f9837aaa9330209bd9ee8af7e91a654a3545aee9c54b3308e78ee360cef1d777d37cfef77d2fa33b5827 @@ -5883,7 +5897,7 @@ __metadata: languageName: node linkType: hard -"fast-levenshtein@npm:^2.0.6, fast-levenshtein@npm:~2.0.6": +"fast-levenshtein@npm:^2.0.6": version: 2.0.6 resolution: "fast-levenshtein@npm:2.0.6" checksum: 92cfec0a8dfafd9c7a15fba8f2cc29cd0b62b85f056d99ce448bbcd9f708e18ab2764bda4dd5158364f4145a7c72788538994f0d1787b956ef0d1062b0f7c24c @@ -5935,13 +5949,6 @@ __metadata: languageName: node linkType: hard -"file-uri-to-path@npm:2": - version: 2.0.0 - resolution: "file-uri-to-path@npm:2.0.0" - checksum: 4a71a99ddaa6ae7ae7bffe2948c34da59982ed465d930a0af9cb59fcc10fcd93366cc356ec3337c18373fde5df7ac52afda4558f155febd1799d135552207edb - languageName: node - linkType: hard - "fill-range@npm:^7.0.1": version: 7.0.1 resolution: "fill-range@npm:7.0.1" @@ -6177,16 +6184,6 @@ __metadata: languageName: node linkType: hard -"ftp@npm:^0.3.10": - version: 0.3.10 - resolution: "ftp@npm:0.3.10" - dependencies: - readable-stream: 1.1.x - xregexp: 2.0.0 - checksum: ddd313c1d44eb7429f3a7d77a0155dc8fe86a4c64dca58f395632333ce4b4e74c61413c6e0ef66ea3f3d32d905952fbb6d028c7117d522f793eb1fa282e17357 - languageName: node - linkType: hard - "function-bind@npm:^1.1.1": version: 1.1.1 resolution: "function-bind@npm:1.1.1" @@ -6295,17 +6292,15 @@ __metadata: languageName: node linkType: hard -"get-uri@npm:3": - version: 3.0.2 - resolution: "get-uri@npm:3.0.2" +"get-uri@npm:^6.0.1": + version: 6.0.1 + resolution: "get-uri@npm:6.0.1" dependencies: - "@tootallnate/once": 1 - data-uri-to-buffer: 3 - debug: 4 - file-uri-to-path: 2 + basic-ftp: ^5.0.2 + data-uri-to-buffer: ^5.0.1 + debug: ^4.3.4 fs-extra: ^8.1.0 - ftp: ^0.3.10 - checksum: 5325b2906b08ca37529ca421cf52bc50376e75c6a945e0a8064e3f76b4bb67b8ab1e316a2fc7a307c8c606ab36d030720f39a57c97b027ff1134335e12102946 + checksum: a8aec70e1c67386fbe67f66e344ecd671a19f4cfc8e0f0e14d070563af5123d540e77fbceb6e26566f29846fac864d2862699ab134d307f85c85e7d72ce23d14 languageName: node linkType: hard @@ -6671,17 +6666,6 @@ __metadata: languageName: node linkType: hard -"http-proxy-agent@npm:^4.0.1": - version: 4.0.1 - resolution: "http-proxy-agent@npm:4.0.1" - dependencies: - "@tootallnate/once": 1 - agent-base: 6 - debug: 4 - checksum: c6a5da5a1929416b6bbdf77b1aca13888013fe7eb9d59fc292e25d18e041bb154a8dfada58e223fc7b76b9b2d155a87e92e608235201f77d34aa258707963a82 - languageName: node - linkType: hard - "http-proxy-agent@npm:^5.0.0": version: 5.0.0 resolution: "http-proxy-agent@npm:5.0.0" @@ -6693,6 +6677,16 @@ __metadata: languageName: node linkType: hard +"http-proxy-agent@npm:^7.0.0": + version: 7.0.0 + resolution: "http-proxy-agent@npm:7.0.0" + dependencies: + agent-base: ^7.1.0 + debug: ^4.3.4 + checksum: 48d4fac997917e15f45094852b63b62a46d0c8a4f0b9c6c23ca26d27b8df8d178bed88389e604745e748bd9a01f5023e25093722777f0593c3f052009ff438b6 + languageName: node + linkType: hard + "http-signature@npm:~1.3.6": version: 1.3.6 resolution: "http-signature@npm:1.3.6" @@ -6714,7 +6708,7 @@ __metadata: languageName: node linkType: hard -"https-proxy-agent@npm:5, https-proxy-agent@npm:^5.0.0, https-proxy-agent@npm:^5.0.1": +"https-proxy-agent@npm:^5.0.0, https-proxy-agent@npm:^5.0.1": version: 5.0.1 resolution: "https-proxy-agent@npm:5.0.1" dependencies: @@ -6724,6 +6718,16 @@ __metadata: languageName: node linkType: hard +"https-proxy-agent@npm:^7.0.2": + version: 7.0.2 + resolution: "https-proxy-agent@npm:7.0.2" + dependencies: + agent-base: ^7.0.2 + debug: 4 + checksum: 088969a0dd476ea7a0ed0a2cf1283013682b08f874c3bc6696c83fa061d2c157d29ef0ad3eb70a2046010bb7665573b2388d10fdcb3e410a66995e5248444292 + languageName: node + linkType: hard + "human-signals@npm:^1.1.1": version: 1.1.1 resolution: "human-signals@npm:1.1.1" @@ -6832,7 +6836,7 @@ __metadata: languageName: node linkType: hard -"inherits@npm:2, inherits@npm:2.0.4, inherits@npm:^2.0.1, inherits@npm:^2.0.3, inherits@npm:~2.0.1, inherits@npm:~2.0.3": +"inherits@npm:2, inherits@npm:2.0.4, inherits@npm:^2.0.1, inherits@npm:^2.0.3, inherits@npm:~2.0.3": version: 2.0.4 resolution: "inherits@npm:2.0.4" checksum: 4a48a733847879d6cf6691860a6b1e3f0f4754176e4d71494c41f3475553768b10f84b5ce1d40fbd0e34e6bfbb864ee35858ad4dd2cf31e02fc4a154b724d7f1 @@ -6857,10 +6861,10 @@ __metadata: languageName: node linkType: hard -"ip@npm:^1.1.5": - version: 1.1.5 - resolution: "ip@npm:1.1.5" - checksum: 30133981f082a060a32644f6a7746e9ba7ac9e2bc07ecc8bbdda3ee8ca9bec1190724c390e45a1ee7695e7edfd2a8f7dda2c104ec5f7ac5068c00648504c7e5a +"ip@npm:^1.1.8": + version: 1.1.8 + resolution: "ip@npm:1.1.8" + checksum: a2ade53eb339fb0cbe9e69a44caab10d6e3784662285eb5d2677117ee4facc33a64679051c35e0dfdb1a3983a51ce2f5d2cb36446d52e10d01881789b76e28fb languageName: node linkType: hard @@ -7222,13 +7226,6 @@ __metadata: languageName: node linkType: hard -"isarray@npm:0.0.1": - version: 0.0.1 - resolution: "isarray@npm:0.0.1" - checksum: 49191f1425681df4a18c2f0f93db3adb85573bcdd6a4482539d98eac9e705d8961317b01175627e860516a2fc45f8f9302db26e5a380a97a520e272e2a40a8d4 - languageName: node - linkType: hard - "isarray@npm:^2.0.5": version: 2.0.5 resolution: "isarray@npm:2.0.5" @@ -7557,7 +7554,7 @@ __metadata: jest-environment-node: 25.1.0 - 29 jest-get-type: 25.1.0 - 29 jest-matcher-utils: 25.1.0 - 29 - mockttp: ^3.7.5 + mockttp: ^3.9.2 semver: ^7.5.4 tmp: ^0.2.1 typescript: ^4.9.5 @@ -8034,16 +8031,6 @@ __metadata: languageName: node linkType: hard -"levn@npm:~0.3.0": - version: 0.3.0 - resolution: "levn@npm:0.3.0" - dependencies: - prelude-ls: ~1.1.2 - type-check: ~0.3.2 - checksum: 0d084a524231a8246bb10fec48cdbb35282099f6954838604f3c7fc66f2e16fa66fd9cc2f3f20a541a113c4dafdf181e822c887c8a319c9195444e6c64ac395e - languageName: node - linkType: hard - "lines-and-columns@npm:^1.1.6": version: 1.2.4 resolution: "lines-and-columns@npm:1.2.4" @@ -8579,9 +8566,9 @@ __metadata: languageName: node linkType: hard -"mockttp@npm:^3.7.5": - version: 3.7.5 - resolution: "mockttp@npm:3.7.5" +"mockttp@npm:^3.9.2": + version: 3.9.2 + resolution: "mockttp@npm:3.9.2" dependencies: "@graphql-tools/schema": ^8.5.0 "@graphql-tools/utils": ^8.8.0 @@ -8612,19 +8599,19 @@ __metadata: lru-cache: ^7.14.0 native-duplexpair: ^1.0.0 node-forge: ^1.2.1 - pac-proxy-agent: ^5.0.0 + pac-proxy-agent: ^7.0.0 parse-multipart-data: ^1.4.0 performance-now: ^2.1.0 portfinder: 1.0.28 read-tls-client-hello: ^1.0.0 - semver: ^5.7.1 + semver: ^7.5.3 socks-proxy-agent: ^7.0.0 typed-error: ^3.0.2 uuid: ^8.3.2 ws: ^8.8.0 bin: mockttp: dist/admin/admin-bin.js - checksum: 0fe524add6bf879385584db3e98c9b3d6e16d7fc8ae175da23b2236ee61f0674f623099fa06c4cf5bd0008955445c34cb9520e639dfdcbbcf549b633445b56f9 + checksum: aa11f01ef7765b96a8ad135f97eddd72e548e626bab4c9fcef7bf52633b1f49e5e4cd2f763356d4ae1ea8bb7f5f4d52110681e46913fc5800b0713a99c273316 languageName: node linkType: hard @@ -8928,20 +8915,6 @@ __metadata: languageName: node linkType: hard -"optionator@npm:^0.8.1": - version: 0.8.3 - resolution: "optionator@npm:0.8.3" - dependencies: - deep-is: ~0.1.3 - fast-levenshtein: ~2.0.6 - levn: ~0.3.0 - prelude-ls: ~1.1.2 - type-check: ~0.3.2 - word-wrap: ~1.2.3 - checksum: b8695ddf3d593203e25ab0900e265d860038486c943ff8b774f596a310f8ceebdb30c6832407a8198ba3ec9debe1abe1f51d4aad94843612db3b76d690c61d34 - languageName: node - linkType: hard - "optionator@npm:^0.9.1": version: 0.9.1 resolution: "optionator@npm:0.9.1" @@ -9024,31 +8997,30 @@ __metadata: languageName: node linkType: hard -"pac-proxy-agent@npm:^5.0.0": - version: 5.0.0 - resolution: "pac-proxy-agent@npm:5.0.0" +"pac-proxy-agent@npm:^7.0.0": + version: 7.0.1 + resolution: "pac-proxy-agent@npm:7.0.1" dependencies: - "@tootallnate/once": 1 - agent-base: 6 - debug: 4 - get-uri: 3 - http-proxy-agent: ^4.0.1 - https-proxy-agent: 5 - pac-resolver: ^5.0.0 - raw-body: ^2.2.0 - socks-proxy-agent: 5 - checksum: cfd26a0e2ebfea4ca6162465018ce093bf147d26cf6c8fb3e7155bc7c184370d80d4d09a1c097e3db7676d0e3f574ea1cb56a4aa7d1d2e5cca6238935fabf010 + "@tootallnate/quickjs-emscripten": ^0.23.0 + agent-base: ^7.0.2 + debug: ^4.3.4 + get-uri: ^6.0.1 + http-proxy-agent: ^7.0.0 + https-proxy-agent: ^7.0.2 + pac-resolver: ^7.0.0 + socks-proxy-agent: ^8.0.2 + checksum: 3d4aa48ec1c19db10158ecc1c4c9a9f77792294412d225ceb3dfa45d5a06950dca9755e2db0d9b69f12769119bea0adf2b24390d9c73c8d81df75e28245ae451 languageName: node linkType: hard -"pac-resolver@npm:^5.0.0": - version: 5.0.1 - resolution: "pac-resolver@npm:5.0.1" +"pac-resolver@npm:^7.0.0": + version: 7.0.0 + resolution: "pac-resolver@npm:7.0.0" dependencies: - degenerator: ^3.0.2 - ip: ^1.1.5 + degenerator: ^5.0.0 + ip: ^1.1.8 netmask: ^2.0.2 - checksum: e3bd8aada70d173cd4cec1ac810fb56161678b7a597060a740c4a31d9c5f8cd95687b2d0fd90b69c0cafe5ef787404074f38042ba08c8d378fed48973f58e493 + checksum: fa3a898c09848e93e35f5e23443fea36ddb393a851c76a23664a5bf3fcbe58ff77a0bcdae1e4f01b9ea87ea493c52e14d97a0fe39f92474d14cd45559c6e3cde languageName: node linkType: hard @@ -9229,13 +9201,6 @@ __metadata: languageName: node linkType: hard -"prelude-ls@npm:~1.1.2": - version: 1.1.2 - resolution: "prelude-ls@npm:1.1.2" - checksum: c4867c87488e4a0c233e158e4d0d5565b609b105d75e4c05dc760840475f06b731332eb93cc8c9cecb840aa8ec323ca3c9a56ad7820ad2e63f0261dadcb154e4 - languageName: node - linkType: hard - "prettier@npm:^2.5.1": version: 2.5.1 resolution: "prettier@npm:2.5.1" @@ -9412,7 +9377,7 @@ __metadata: languageName: node linkType: hard -"raw-body@npm:2.5.2, raw-body@npm:^2.2.0, raw-body@npm:^2.4.1": +"raw-body@npm:2.5.2, raw-body@npm:^2.4.1": version: 2.5.2 resolution: "raw-body@npm:2.5.2" dependencies: @@ -9461,18 +9426,6 @@ __metadata: languageName: node linkType: hard -"readable-stream@npm:1.1.x": - version: 1.1.14 - resolution: "readable-stream@npm:1.1.14" - dependencies: - core-util-is: ~1.0.0 - inherits: ~2.0.1 - isarray: 0.0.1 - string_decoder: ~0.10.x - checksum: 17dfeae3e909945a4a1abc5613ea92d03269ef54c49288599507fc98ff4615988a1c39a999dcf9aacba70233d9b7040bc11a5f2bfc947e262dedcc0a8b32b5a0 - languageName: node - linkType: hard - "readable-stream@npm:^2.0.0, readable-stream@npm:^2.3.3": version: 2.3.8 resolution: "readable-stream@npm:2.3.8" @@ -9858,7 +9811,7 @@ __metadata: languageName: node linkType: hard -"semver@npm:^5.7.0, semver@npm:^5.7.1": +"semver@npm:^5.7.0": version: 5.7.1 resolution: "semver@npm:5.7.1" bin: @@ -10045,17 +9998,6 @@ __metadata: languageName: node linkType: hard -"socks-proxy-agent@npm:5": - version: 5.0.1 - resolution: "socks-proxy-agent@npm:5.0.1" - dependencies: - agent-base: ^6.0.2 - debug: 4 - socks: ^2.3.3 - checksum: 1b60c4977b2fef783f0fc4dc619cd2758aafdb43f3cf679f1e3627cb6c6e752811cee5513ebb4157ad26786033d2f85029440f197d321e8293b38cc5aab01e06 - languageName: node - linkType: hard - "socks-proxy-agent@npm:^6.1.1": version: 6.1.1 resolution: "socks-proxy-agent@npm:6.1.1" @@ -10078,7 +10020,18 @@ __metadata: languageName: node linkType: hard -"socks@npm:^2.3.3, socks@npm:^2.6.1, socks@npm:^2.6.2": +"socks-proxy-agent@npm:^8.0.2": + version: 8.0.2 + resolution: "socks-proxy-agent@npm:8.0.2" + dependencies: + agent-base: ^7.0.2 + debug: ^4.3.4 + socks: ^2.7.1 + checksum: 4fb165df08f1f380881dcd887b3cdfdc1aba3797c76c1e9f51d29048be6e494c5b06d68e7aea2e23df4572428f27a3ec22b3d7c75c570c5346507433899a4b6d + languageName: node + linkType: hard + +"socks@npm:^2.6.1, socks@npm:^2.6.2, socks@npm:^2.7.1": version: 2.7.1 resolution: "socks@npm:2.7.1" dependencies: @@ -10277,13 +10230,6 @@ __metadata: languageName: node linkType: hard -"string_decoder@npm:~0.10.x": - version: 0.10.31 - resolution: "string_decoder@npm:0.10.31" - checksum: fe00f8e303647e5db919948ccb5ce0da7dea209ab54702894dd0c664edd98e5d4df4b80d6fabf7b9e92b237359d21136c95bf068b2f7760b772ca974ba970202 - languageName: node - linkType: hard - "string_decoder@npm:~1.1.1": version: 1.1.1 resolution: "string_decoder@npm:1.1.1" @@ -10750,15 +10696,6 @@ __metadata: languageName: node linkType: hard -"type-check@npm:~0.3.2": - version: 0.3.2 - resolution: "type-check@npm:0.3.2" - dependencies: - prelude-ls: ~1.1.2 - checksum: dd3b1495642731bc0e1fc40abe5e977e0263005551ac83342ecb6f4f89551d106b368ec32ad3fb2da19b3bd7b2d1f64330da2ea9176d8ddbfe389fb286eb5124 - languageName: node - linkType: hard - "type-detect@npm:4.0.8": version: 4.0.8 resolution: "type-detect@npm:4.0.8" @@ -10853,7 +10790,7 @@ __metadata: debug: ^4.3.3 deep-equal: ^2.0.5 expect: 25.1.0 - 29 - mockttp: ^3.7.5 + mockttp: ^3.9.2 rimraf: ^5.0.1 rollup: ^3.21.1 rollup-plugin-dts: ^5.3.0 @@ -11029,18 +10966,6 @@ __metadata: languageName: node linkType: hard -"vm2@npm:^3.9.17": - version: 3.9.19 - resolution: "vm2@npm:3.9.19" - dependencies: - acorn: ^8.7.0 - acorn-walk: ^8.2.0 - bin: - vm2: bin/vm2 - checksum: fc6cf553134145cd7bb5246985bf242b056e3fb5ea71e2eef6710b2a5d6c6119cc6bc960435ff62480ee82efb43369be8f4db07b6690916ae7d3b2e714f395d8 - languageName: node - linkType: hard - "walker@npm:^1.0.8": version: 1.0.8 resolution: "walker@npm:1.0.8" @@ -11216,7 +11141,7 @@ __metadata: languageName: node linkType: hard -"word-wrap@npm:^1.2.3, word-wrap@npm:~1.2.3": +"word-wrap@npm:^1.2.3": version: 1.2.3 resolution: "word-wrap@npm:1.2.3" checksum: 30b48f91fcf12106ed3186ae4fa86a6a1842416df425be7b60485de14bec665a54a68e4b5156647dec3a70f25e84d270ca8bc8cd23182ed095f5c7206a938c1f @@ -11306,13 +11231,6 @@ __metadata: languageName: node linkType: hard -"xregexp@npm:2.0.0": - version: 2.0.0 - resolution: "xregexp@npm:2.0.0" - checksum: de62d1f01c9f1a67c80cafe48a3dc081b324249a0e88e65dc9acae9cce6d8e63c9d91c0f97e2ad2d8c5351c856c139c04dc55ebd941e59b7d1d5c1169e164cff - languageName: node - linkType: hard - "xtend@npm:^4.0.0": version: 4.0.2 resolution: "xtend@npm:4.0.2" From 302ad84f1218979d1bb83bec912e3536da543e7d Mon Sep 17 00:00:00 2001 From: "David A. Ramos" Date: Sun, 10 Sep 2023 18:54:37 -0700 Subject: [PATCH 34/53] Bump mkdirp to 0.5.6 This bumps minimist to 0.2.8, which addresses CVE-2021-44906. --- yarn.lock | 20 +------------------- 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/yarn.lock b/yarn.lock index d4d21d6..3e166cd 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8394,13 +8394,6 @@ __metadata: languageName: node linkType: hard -"minimist@npm:0.0.8": - version: 0.0.8 - resolution: "minimist@npm:0.0.8" - checksum: 042f8b626b1fa44dffc23bac55771425ac4ee9d267b56f9064c07713e516e1799f3ba933bb628d2475a210caf7dcdb98161611baa1f0daf49309a944cb4bc48f - languageName: node - linkType: hard - "minimist@npm:^1.2.0, minimist@npm:^1.2.6, minimist@npm:^1.2.8": version: 1.2.8 resolution: "minimist@npm:1.2.8" @@ -8485,18 +8478,7 @@ __metadata: languageName: node linkType: hard -"mkdirp@npm:0.5.1": - version: 0.5.1 - resolution: "mkdirp@npm:0.5.1" - dependencies: - minimist: 0.0.8 - bin: - mkdirp: bin/cmd.js - checksum: ed1ab49bb1d06c88dba7cfe930a3186f2605b5465aab7c8f24119baaba6e38f9ab4ac1695c68f476c65a48df2a69a8495049cd6e26c360ea082151a0771343d2 - languageName: node - linkType: hard - -"mkdirp@npm:^0.5.5": +"mkdirp@npm:0.5.1, mkdirp@npm:^0.5.5": version: 0.5.6 resolution: "mkdirp@npm:0.5.6" dependencies: From 677aac1149f468b1908389b9f393f8bf4d73c9bf Mon Sep 17 00:00:00 2001 From: "David A. Ramos" Date: Sun, 10 Sep 2023 19:00:31 -0700 Subject: [PATCH 35/53] Bump yargs-unparser to 1.6.4 This bumps the transitive dependency `flat` to 5.0.2, which addresses CVE-2020-36632. --- yarn.lock | 63 ++++++++++++++++++++++++++++++++----------------------- 1 file changed, 37 insertions(+), 26 deletions(-) diff --git a/yarn.lock b/yarn.lock index 3e166cd..ecbf89a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6027,14 +6027,12 @@ __metadata: languageName: node linkType: hard -"flat@npm:^4.1.0": - version: 4.1.1 - resolution: "flat@npm:4.1.1" - dependencies: - is-buffer: ~2.0.3 +"flat@npm:^5.0.2": + version: 5.0.2 + resolution: "flat@npm:5.0.2" bin: flat: cli.js - checksum: 398be12185eb0f3c59797c3670a8c35d07020b673363175676afbaf53d6b213660e060488554cf82c25504986e1a6059bdbcc5d562e87ca3e972e8a33148e3ae + checksum: 12a1536ac746db74881316a181499a78ef953632ddd28050b7a3a43c62ef5462e3357c8c29d76072bb635f147f7a9a1f0c02efef6b4be28f8db62ceb3d5c7f5d languageName: node linkType: hard @@ -6945,13 +6943,6 @@ __metadata: languageName: node linkType: hard -"is-buffer@npm:~2.0.3": - version: 2.0.5 - resolution: "is-buffer@npm:2.0.5" - checksum: 764c9ad8b523a9f5a32af29bdf772b08eb48c04d2ad0a7240916ac2688c983bf5f8504bf25b35e66240edeb9d9085461f9b5dae1f3d2861c6b06a65fe983de42 - languageName: node - linkType: hard - "is-builtin-module@npm:^3.2.1": version: 3.2.1 resolution: "is-builtin-module@npm:3.2.1" @@ -7104,6 +7095,13 @@ __metadata: languageName: node linkType: hard +"is-plain-obj@npm:^1.1.0": + version: 1.1.0 + resolution: "is-plain-obj@npm:1.1.0" + checksum: 0ee04807797aad50859652a7467481816cbb57e5cc97d813a7dcd8915da8195dc68c436010bf39d195226cde6a2d352f4b815f16f26b7bf486a5754290629931 + languageName: node + linkType: hard + "is-reference@npm:1.2.1": version: 1.2.1 resolution: "is-reference@npm:1.2.1" @@ -8122,7 +8120,7 @@ __metadata: languageName: node linkType: hard -"lodash@npm:^4.16.4, lodash@npm:^4.17.14, lodash@npm:^4.17.15, lodash@npm:^4.17.21": +"lodash@npm:^4.16.4, lodash@npm:^4.17.14, lodash@npm:^4.17.21": version: 4.17.21 resolution: "lodash@npm:4.17.21" checksum: eb835a2e51d381e561e508ce932ea50a8e5a68f4ebdd771ea240d3048244a8d13658acbd502cd4829768c56f2e16bdd4340b9ea141297d472517b83868e677f7 @@ -11265,7 +11263,7 @@ __metadata: languageName: node linkType: hard -"yargs-parser@npm:^13.1.1, yargs-parser@npm:^13.1.2": +"yargs-parser@npm:^13.1.1": version: 13.1.2 resolution: "yargs-parser@npm:13.1.2" dependencies: @@ -11275,6 +11273,16 @@ __metadata: languageName: node linkType: hard +"yargs-parser@npm:^15.0.1": + version: 15.0.3 + resolution: "yargs-parser@npm:15.0.3" + dependencies: + camelcase: ^5.0.0 + decamelize: ^1.2.0 + checksum: 06611c1893fa9f1c25ae79df3c6e2edbac7c8d75257a4b55b8432cbc87ee03eda86bea0537f65b4b8a0d9684c83fa6e9ef61ef720a1e5cc8a9aa6893b54ee4c3 + languageName: node + linkType: hard + "yargs-parser@npm:^21.0.1, yargs-parser@npm:^21.1.1": version: 21.1.1 resolution: "yargs-parser@npm:21.1.1" @@ -11283,13 +11291,15 @@ __metadata: linkType: hard "yargs-unparser@npm:1.6.0": - version: 1.6.0 - resolution: "yargs-unparser@npm:1.6.0" + version: 1.6.4 + resolution: "yargs-unparser@npm:1.6.4" dependencies: - flat: ^4.1.0 - lodash: ^4.17.15 - yargs: ^13.3.0 - checksum: ca662bb94af53d816d47f2162f0a1d135783f09de9fd47645a5cb18dd25532b0b710432b680d2c065ff45de122ba4a96433c41595fa7bfcc08eb12e889db95c1 + camelcase: ^5.3.1 + decamelize: ^1.2.0 + flat: ^5.0.2 + is-plain-obj: ^1.1.0 + yargs: ^14.2.3 + checksum: 428695924f6dc3b660cab37e5f1bb46a7bc64bb81e583beaaf40155f2d33440e3776518528e98902d256ed68fe4cc74c54c0188481ce8dba6857bc1656b5ca06 languageName: node linkType: hard @@ -11311,11 +11321,12 @@ __metadata: languageName: node linkType: hard -"yargs@npm:^13.3.0": - version: 13.3.2 - resolution: "yargs@npm:13.3.2" +"yargs@npm:^14.2.3": + version: 14.2.3 + resolution: "yargs@npm:14.2.3" dependencies: cliui: ^5.0.0 + decamelize: ^1.2.0 find-up: ^3.0.0 get-caller-file: ^2.0.1 require-directory: ^2.1.1 @@ -11324,8 +11335,8 @@ __metadata: string-width: ^3.0.0 which-module: ^2.0.0 y18n: ^4.0.0 - yargs-parser: ^13.1.2 - checksum: 75c13e837eb2bb25717957ba58d277e864efc0cca7f945c98bdf6477e6ec2f9be6afa9ed8a876b251a21423500c148d7b91e88dee7adea6029bdec97af1ef3e8 + yargs-parser: ^15.0.1 + checksum: 684fcb1896e6c873c31c09c5c16445d6253dfe505aa879cff56d49425f5bca44f2ab8d7a1c949f3b932ae8654128425e89770e5e2f2c3d816e5816b9eb6efb6f languageName: node linkType: hard From 6effd5358cd893bd59699227851f71422a0f7812 Mon Sep 17 00:00:00 2001 From: "David A. Ramos" Date: Sun, 10 Sep 2023 19:07:20 -0700 Subject: [PATCH 36/53] Bump word-wrap to 1.2.5 This addresses CVE-2023-26115. --- yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/yarn.lock b/yarn.lock index ecbf89a..421962b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -11122,9 +11122,9 @@ __metadata: linkType: hard "word-wrap@npm:^1.2.3": - version: 1.2.3 - resolution: "word-wrap@npm:1.2.3" - checksum: 30b48f91fcf12106ed3186ae4fa86a6a1842416df425be7b60485de14bec665a54a68e4b5156647dec3a70f25e84d270ca8bc8cd23182ed095f5c7206a938c1f + version: 1.2.5 + resolution: "word-wrap@npm:1.2.5" + checksum: f93ba3586fc181f94afdaff3a6fef27920b4b6d9eaefed0f428f8e07adea2a7f54a5f2830ce59406c8416f033f86902b91eb824072354645eea687dff3691ccb languageName: node linkType: hard From 17064fc2c9c626ee00b371f4bdb3b54a4b45cc4b Mon Sep 17 00:00:00 2001 From: "David A. Ramos" Date: Sun, 10 Sep 2023 19:04:08 -0700 Subject: [PATCH 37/53] Bump minimatch to 3.0.8 This addresses CVE-2022-3517. --- yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/yarn.lock b/yarn.lock index 421962b..d9ca68b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8357,11 +8357,11 @@ __metadata: linkType: hard "minimatch@npm:3.0.4": - version: 3.0.4 - resolution: "minimatch@npm:3.0.4" + version: 3.0.8 + resolution: "minimatch@npm:3.0.8" dependencies: brace-expansion: ^1.1.7 - checksum: 66ac295f8a7b59788000ea3749938b0970344c841750abd96694f80269b926ebcafad3deeb3f1da2522978b119e6ae3a5869b63b13a7859a456b3408bd18a078 + checksum: 850cca179cad715133132693e6963b0db64ab0988c4d211415b087fc23a3e46321e2c5376a01bf5623d8782aba8bdf43c571e2e902e51fdce7175c7215c29f8b languageName: node linkType: hard From 3e444c17b860117ae6a259a61fd5c3539d7b5cfa Mon Sep 17 00:00:00 2001 From: "David A. Ramos" Date: Sun, 10 Sep 2023 19:10:05 -0700 Subject: [PATCH 38/53] Bump semver to 5.7.2 This addresses CVE-2022-25883. --- yarn.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/yarn.lock b/yarn.lock index d9ca68b..ce5c3d8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -9792,11 +9792,11 @@ __metadata: linkType: hard "semver@npm:^5.7.0": - version: 5.7.1 - resolution: "semver@npm:5.7.1" + version: 5.7.2 + resolution: "semver@npm:5.7.2" bin: - semver: ./bin/semver - checksum: 57fd0acfd0bac382ee87cd52cd0aaa5af086a7dc8d60379dfe65fea491fb2489b6016400813930ecd61fd0952dae75c115287a1b16c234b1550887117744dfaf + semver: bin/semver + checksum: fb4ab5e0dd1c22ce0c937ea390b4a822147a9c53dbd2a9a0132f12fe382902beef4fbf12cf51bb955248d8d15874ce8cd89532569756384f994309825f10b686 languageName: node linkType: hard From 16186702ab57252521c78f5340b937d4b9ec5ddb Mon Sep 17 00:00:00 2001 From: "David A. Ramos" Date: Sun, 10 Sep 2023 19:13:12 -0700 Subject: [PATCH 39/53] Bump @cypress/request to 2.88.12 This bumps the transitive dependency `tough-cookie` to 4.1.3, which addresses CVE-2023-26136. --- yarn.lock | 53 +++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 43 insertions(+), 10 deletions(-) diff --git a/yarn.lock b/yarn.lock index ce5c3d8..3d57717 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1408,8 +1408,8 @@ __metadata: linkType: hard "@cypress/request@npm:^2.88.10": - version: 2.88.11 - resolution: "@cypress/request@npm:2.88.11" + version: 2.88.12 + resolution: "@cypress/request@npm:2.88.12" dependencies: aws-sign2: ~0.7.0 aws4: ^1.8.0 @@ -1426,10 +1426,10 @@ __metadata: performance-now: ^2.1.0 qs: ~6.10.3 safe-buffer: ^5.1.2 - tough-cookie: ~2.5.0 + tough-cookie: ^4.1.3 tunnel-agent: ^0.6.0 uuid: ^8.3.2 - checksum: e4b3f62e0c41c4ccca6c942828461d8ea717e752fd918d685e9f74e2ebcfa8b7942427f7ce971e502635c3bf3d40011476db84dc753d3dc360c6d08350da6f93 + checksum: 2c6fbf7f3127d41bffca8374beaa8cf95450495a8a077b00309ea9d94dd2a4da450a77fe038e8ad26c97cdd7c39b65c53c850f8338ce9bc2dbe23ce2e2b48329 languageName: node linkType: hard @@ -9266,7 +9266,7 @@ __metadata: languageName: node linkType: hard -"psl@npm:^1.1.28": +"psl@npm:^1.1.33": version: 1.9.0 resolution: "psl@npm:1.9.0" checksum: 20c4277f640c93d393130673f392618e9a8044c6c7bf61c53917a0fddb4952790f5f362c6c730a9c32b124813e173733f9895add8d26f566ed0ea0654b2e711d @@ -9315,6 +9315,13 @@ __metadata: languageName: node linkType: hard +"querystringify@npm:^2.1.1": + version: 2.2.0 + resolution: "querystringify@npm:2.2.0" + checksum: 5641ea231bad7ef6d64d9998faca95611ed4b11c2591a8cae741e178a974f6a8e0ebde008475259abe1621cb15e692404e6b6626e927f7b849d5c09392604b15 + languageName: node + linkType: hard + "queue-microtask@npm:^1.2.2": version: 1.2.3 resolution: "queue-microtask@npm:1.2.3" @@ -9532,6 +9539,13 @@ __metadata: languageName: node linkType: hard +"requires-port@npm:^1.0.0": + version: 1.0.0 + resolution: "requires-port@npm:1.0.0" + checksum: eee0e303adffb69be55d1a214e415cf42b7441ae858c76dfc5353148644f6fd6e698926fc4643f510d5c126d12a705e7c8ed7e38061113bdf37547ab356797ff + languageName: node + linkType: hard + "resolve-alpn@npm:^1.2.0": version: 1.2.1 resolution: "resolve-alpn@npm:1.2.1" @@ -10502,13 +10516,15 @@ __metadata: languageName: node linkType: hard -"tough-cookie@npm:~2.5.0": - version: 2.5.0 - resolution: "tough-cookie@npm:2.5.0" +"tough-cookie@npm:^4.1.3": + version: 4.1.3 + resolution: "tough-cookie@npm:4.1.3" dependencies: - psl: ^1.1.28 + psl: ^1.1.33 punycode: ^2.1.1 - checksum: 16a8cd090224dd176eee23837cbe7573ca0fa297d7e468ab5e1c02d49a4e9a97bb05fef11320605eac516f91d54c57838a25864e8680e27b069a5231d8264977 + universalify: ^0.2.0 + url-parse: ^1.5.3 + checksum: c9226afff36492a52118432611af083d1d8493a53ff41ec4ea48e5b583aec744b989e4280bcf476c910ec1525a89a4a0f1cae81c08b18fb2ec3a9b3a72b91dcc languageName: node linkType: hard @@ -10836,6 +10852,13 @@ __metadata: languageName: node linkType: hard +"universalify@npm:^0.2.0": + version: 0.2.0 + resolution: "universalify@npm:0.2.0" + checksum: e86134cb12919d177c2353196a4cc09981524ee87abf621f7bc8d249dbbbebaec5e7d1314b96061497981350df786e4c5128dbf442eba104d6e765bc260678b5 + languageName: node + linkType: hard + "universalify@npm:^2.0.0": version: 2.0.0 resolution: "universalify@npm:2.0.0" @@ -10880,6 +10903,16 @@ __metadata: languageName: node linkType: hard +"url-parse@npm:^1.5.3": + version: 1.5.10 + resolution: "url-parse@npm:1.5.10" + dependencies: + querystringify: ^2.1.1 + requires-port: ^1.0.0 + checksum: fbdba6b1d83336aca2216bbdc38ba658d9cfb8fc7f665eb8b17852de638ff7d1a162c198a8e4ed66001ddbf6c9888d41e4798912c62b4fd777a31657989f7bdf + languageName: node + linkType: hard + "util-deprecate@npm:^1.0.1, util-deprecate@npm:~1.0.1": version: 1.0.2 resolution: "util-deprecate@npm:1.0.2" From 752980a556995bb0ee24b260896c2da11cc169d7 Mon Sep 17 00:00:00 2001 From: "David A. Ramos" Date: Sun, 10 Sep 2023 19:15:22 -0700 Subject: [PATCH 40/53] Bump yargs-parser to 13.1.2 This addresses CVE-2020-7608. --- yarn.lock | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/yarn.lock b/yarn.lock index 3d57717..7398f4c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -11286,17 +11286,7 @@ __metadata: languageName: node linkType: hard -"yargs-parser@npm:13.1.1": - version: 13.1.1 - resolution: "yargs-parser@npm:13.1.1" - dependencies: - camelcase: ^5.0.0 - decamelize: ^1.2.0 - checksum: fa5fd27736aa423dc9a114d160dae94625f7faf19c252b8c91ac0197be9715d1dbc9b98fda893f75f182111fb6c3c0ce60c631b73859dd1a06bec07cddfb98f4 - languageName: node - linkType: hard - -"yargs-parser@npm:^13.1.1": +"yargs-parser@npm:13.1.1, yargs-parser@npm:^13.1.1": version: 13.1.2 resolution: "yargs-parser@npm:13.1.2" dependencies: From 15c870a94fcacc8390af70de38951c37f8253b13 Mon Sep 17 00:00:00 2001 From: "David A. Ramos" Date: Tue, 12 Sep 2023 13:51:01 -0700 Subject: [PATCH 41/53] [cypress] Support Cypress 13.2 --- .github/workflows/ci.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 7b043e5..7a0c862 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -129,6 +129,7 @@ jobs: - "12.17" - "13.0" - "13.1" + - "13.2" steps: - uses: actions/checkout@v4 @@ -236,6 +237,7 @@ jobs: - "12.17" - "13.0" - "13.1" + - "13.2" steps: - uses: actions/checkout@v4 From 285b34e605681d8b18dbf891391cf39c872de0a3 Mon Sep 17 00:00:00 2001 From: "David A. Ramos" Date: Tue, 12 Sep 2023 13:54:24 -0700 Subject: [PATCH 42/53] [jest] Support Jest 29.7 --- .github/workflows/ci.yaml | 2 ++ scripts/set-jest-version.ts | 13 +++++++++---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 7a0c862..7ccacca 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -332,6 +332,7 @@ jobs: - 18 - 20 jest: + - "29.7" - "29.6" - "29.5" - "29.4" @@ -448,6 +449,7 @@ jobs: - 18 - 20 jest: + - "29.7" - "29.6" - "29.5" - "29.4" diff --git a/scripts/set-jest-version.ts b/scripts/set-jest-version.ts index b12f92b..805403d 100644 --- a/scripts/set-jest-version.ts +++ b/scripts/set-jest-version.ts @@ -77,6 +77,7 @@ const PACKAGE_VERSION_MAP = { "25 - 27": null, "29.0 - 29.3": "~29.0", "29.4 - 29.5": "~29.4", + "29.6 - 29.7": "~29.6", }, "@jest/source-map": { "25.2 - 25.4": "~25.2", @@ -86,6 +87,7 @@ const PACKAGE_VERSION_MAP = { "29.0 - 29.1": "~29.0", "29.2 - 29.3": "~29.2", "29.4 - 29.5": "~29.4", + "29.6 - 29.7": "~29.6", }, "@jest/test-result": { "26.3 - 26.4": "~26.3", @@ -96,6 +98,7 @@ const PACKAGE_VERSION_MAP = { "@jest/types": { "26.3 - 26.4": "~26.3", "27.2 - 27.3": "~27.2", + "29.6 - 29.7": "~29.6", }, "babel-jest": { "26.3 - 26.4": "~26.3", @@ -107,7 +110,7 @@ const PACKAGE_VERSION_MAP = { "27.2 - 27.3": "~27.2", "29.0 - 29.1": "~29.0", "29.2 - 29.3": "~29.2", - "29.5 - 29.6": "~29.5", + "29.6 - 29.7": "~29.6", }, "babel-preset-jest": { "26.3 - 26.4": "~26.3", @@ -115,7 +118,7 @@ const PACKAGE_VERSION_MAP = { "27.2 - 27.3": "~27.2", "29.0 - 29.1": "~29.0", "29.2 - 29.3": "~29.2", - "29.5 - 29.6": "~29.5", + "29.6 - 29.7": "~29.6", }, "jest-changed-files": { "26.3 - 26.4": "~26.3", @@ -145,7 +148,8 @@ const PACKAGE_VERSION_MAP = { "^28": "~28.0", "29.0 - 29.1": "~29.0", "29.2 - 29.3": "~29.2", - "29.4 - 29.6": "~29.4", + "29.4 - 29.5": "~29.4", + "29.6 - 29.7": "~29.6", }, "jest-haste-map": { "26.3 - 26.4": "~26.3", @@ -163,7 +167,8 @@ const PACKAGE_VERSION_MAP = { "^28": "~28.0", "29.0 - 29.1": "~29.0", "29.2 - 29.3": "~29.2", - "29.4 - 29.6": "~29.4", + "29.4 - 29.5": "~29.4", + "29.6 - 29.7": "~29.6", }, "jest-serializer": { "25.2 - 25.4": "~25.2", From 48613d1e777782079472913e7bab6faeec97f3f5 Mon Sep 17 00:00:00 2001 From: "David A. Ramos" Date: Thu, 21 Sep 2023 19:37:12 -0700 Subject: [PATCH 43/53] Limit CI to 56 concurrent jobs Currently, a single run of this repository's CI pipeline will exhaust all 60 available concurrent GitHub Actions runners for our GitHub Team subscription. This makes it difficult to run CI for other repositories. --- .github/workflows/ci.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 7ccacca..007a034 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -110,6 +110,7 @@ jobs: if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || needs.check.outputs.affects_cypress == 'true' strategy: fail-fast: false + max-parallel: 16 matrix: node: - 16 @@ -218,6 +219,7 @@ jobs: if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || needs.check.outputs.affects_cypress == 'true' strategy: fail-fast: false + max-parallel: 16 matrix: node: - 16 @@ -326,6 +328,7 @@ jobs: if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || needs.check.outputs.affects_jest == 'true' strategy: fail-fast: false + max-parallel: 12 matrix: node: - 16 @@ -443,6 +446,7 @@ jobs: if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || needs.check.outputs.affects_jest == 'true' strategy: fail-fast: false + max-parallel: 12 matrix: node: - 16 From ced36a8f54c20248b311ea3737c5ec062b2a7315 Mon Sep 17 00:00:00 2001 From: "David A. Ramos" Date: Thu, 21 Sep 2023 22:36:54 -0700 Subject: [PATCH 44/53] [cypress] Avoid clobbering conflicting event handlers This works around https://github.com/cypress-io/cypress/issues/22428. --- packages/cypress-plugin/cypress-on-fix.d.ts | 8 ++++++++ packages/cypress-plugin/package.json | 1 + packages/cypress-plugin/src/index.ts | 8 +++++++- packages/cypress-plugin/src/tsconfig.json | 2 +- .../test/integration-input-manual/cypress-config.mjs | 11 +++++++---- .../test/integration-input-manual/cypress.config.js | 11 +++++++---- .../test/integration-input-manual/package.json | 1 + packages/cypress-plugin/tsconfig.json | 2 +- yarn.lock | 9 +++++++++ 9 files changed, 42 insertions(+), 11 deletions(-) create mode 100644 packages/cypress-plugin/cypress-on-fix.d.ts diff --git a/packages/cypress-plugin/cypress-on-fix.d.ts b/packages/cypress-plugin/cypress-on-fix.d.ts new file mode 100644 index 0000000..abce4b3 --- /dev/null +++ b/packages/cypress-plugin/cypress-on-fix.d.ts @@ -0,0 +1,8 @@ +// Copyright (c) 2023 Developer Innovations, LLC + +declare module "cypress-on-fix" { + import * as Cypress from "cypress"; + export default function onProxy( + on: Cypress.PluginEvents + ): Cypress.PluginEvents; +} diff --git a/packages/cypress-plugin/package.json b/packages/cypress-plugin/package.json index 604db86..0cd0dc7 100644 --- a/packages/cypress-plugin/package.json +++ b/packages/cypress-plugin/package.json @@ -52,6 +52,7 @@ "cli-table3": "0.5.1", "cosmiconfig": "^7.0.1", "cypress-multi-reporters": "^1.6.3", + "cypress-on-fix": "^1.0.2", "dayjs": "^1.10.4", "debug": "^4.3.3", "deep-equal": "^2.0.5", diff --git a/packages/cypress-plugin/src/index.ts b/packages/cypress-plugin/src/index.ts index 90b54e5..fb65ae9 100644 --- a/packages/cypress-plugin/src/index.ts +++ b/packages/cypress-plugin/src/index.ts @@ -19,6 +19,7 @@ import { ENV_VAR_AUTO_SUPPORT, ENV_VAR_UNFLAKABLE_RESOLVED_CONFIG_JSON, } from "./config-env-vars"; +import cypressOnFix from "cypress-on-fix"; export { PluginOptions }; @@ -100,9 +101,14 @@ const wrapSetupNodeEvents = | undefined ) => async ( - on: Cypress.PluginEvents, + baseOn: Cypress.PluginEvents, config: Cypress.PluginConfigOptions ): Promise => { + // Due to https://github.com/cypress-io/cypress/issues/22428, only the last event handler + // registered for each event type will be called. This means we'll clobber any event handlers + // the user registers. To avoid this, we use cypress-on-fix. + const on = cypressOnFix(baseOn); + const userModifiedConfig = userSetupNodeEvents !== undefined ? await userSetupNodeEvents(on, config) diff --git a/packages/cypress-plugin/src/tsconfig.json b/packages/cypress-plugin/src/tsconfig.json index 44badea..80d8585 100644 --- a/packages/cypress-plugin/src/tsconfig.json +++ b/packages/cypress-plugin/src/tsconfig.json @@ -1,4 +1,4 @@ { "extends": "../tsconfig.json", - "include": ["../mocha.d.ts", ".", ".eslintrc.js"] + "include": ["../cypress-on-fix.d.ts", "../mocha.d.ts", ".", ".eslintrc.js"] } diff --git a/packages/cypress-plugin/test/integration-input-manual/cypress-config.mjs b/packages/cypress-plugin/test/integration-input-manual/cypress-config.mjs index 952022a..d5d5a61 100644 --- a/packages/cypress-plugin/test/integration-input-manual/cypress-config.mjs +++ b/packages/cypress-plugin/test/integration-input-manual/cypress-config.mjs @@ -8,6 +8,7 @@ import { registerCosmiconfigMock } from "unflakable-test-common/dist/config.js"; import { registerUnflakable } from "@unflakable/cypress-plugin"; import semverGte from "semver/functions/gte.js"; import path from "path"; +import cypressOnFix from "cypress-on-fix"; /** * @type {Cypress.ConfigOptions} @@ -15,11 +16,12 @@ import path from "path"; export default { component: { /** - * @param {Cypress.PluginEvents} on + * @param {Cypress.PluginEvents} baseOn * @param {Cypress.PluginConfigOptions} config * @returns {Promise | Cypress.PluginConfigOptions | void} */ - setupNodeEvents(on, config) { + setupNodeEvents(baseOn, config) { + const on = cypressOnFix(baseOn); registerCosmiconfigMock(); registerSimpleGitMock(); tasks.registerTasks(on); @@ -35,11 +37,12 @@ export default { }, e2e: { /** - * @param {Cypress.PluginEvents} on + * @param {Cypress.PluginEvents} baseOn * @param {Cypress.PluginConfigOptions} config * @returns {Promise | Cypress.PluginConfigOptions | void} */ - setupNodeEvents(on, config) { + setupNodeEvents(baseOn, config) { + const on = cypressOnFix(baseOn); registerCosmiconfigMock(); registerSimpleGitMock(); tasks.registerTasks(on); diff --git a/packages/cypress-plugin/test/integration-input-manual/cypress.config.js b/packages/cypress-plugin/test/integration-input-manual/cypress.config.js index 9e0a4e8..376bd1f 100644 --- a/packages/cypress-plugin/test/integration-input-manual/cypress.config.js +++ b/packages/cypress-plugin/test/integration-input-manual/cypress.config.js @@ -11,6 +11,7 @@ const semverGte = require("semver/functions/gte"); const { registerUnflakable } = require("@unflakable/cypress-plugin"); const path = require("path"); +const cypressOnFix = require("cypress-on-fix"); module.exports = { /** @@ -18,11 +19,12 @@ module.exports = { */ component: { /** - * @param {Cypress.PluginEvents} on + * @param {Cypress.PluginEvents} baseOn * @param {Cypress.PluginConfigOptions} config * @returns {Promise | Cypress.PluginConfigOptions | void} */ - setupNodeEvents(on, config) { + setupNodeEvents(baseOn, config) { + const on = cypressOnFix(baseOn); registerCosmiconfigMock(); registerSimpleGitMock(); registerTasks(on); @@ -38,11 +40,12 @@ module.exports = { }, e2e: { /** - * @param {Cypress.PluginEvents} on + * @param {Cypress.PluginEvents} baseOn * @param {Cypress.PluginConfigOptions} config * @returns {Promise | Cypress.PluginConfigOptions | void} */ - setupNodeEvents(on, config) { + setupNodeEvents(baseOn, config) { + const on = cypressOnFix(baseOn); registerCosmiconfigMock(); registerSimpleGitMock(); registerTasks(on); diff --git a/packages/cypress-plugin/test/integration-input-manual/package.json b/packages/cypress-plugin/test/integration-input-manual/package.json index 49515a8..c67d68d 100644 --- a/packages/cypress-plugin/test/integration-input-manual/package.json +++ b/packages/cypress-plugin/test/integration-input-manual/package.json @@ -11,6 +11,7 @@ "@unflakable/cypress-plugin": "workspace:^", "cypress": "11.2 - 13", "cypress-multi-reporters": "^1.6.3", + "cypress-on-fix": "^1.0.2", "mocha": "=7.0.1", "mocha-junit-reporter": "^2.2.0", "process": "^0.11.10", diff --git a/packages/cypress-plugin/tsconfig.json b/packages/cypress-plugin/tsconfig.json index 9e9a7c8..786cbe3 100644 --- a/packages/cypress-plugin/tsconfig.json +++ b/packages/cypress-plugin/tsconfig.json @@ -15,5 +15,5 @@ // Avoids conflicting global definitions from, e.g., jest. "types": ["node"] }, - "include": ["mocha.d.ts", "rollup.config.mjs"] + "include": ["cypress-on-fix.d.ts", "mocha.d.ts", "rollup.config.mjs"] } diff --git a/yarn.lock b/yarn.lock index 7398f4c..5eb8cd1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3218,6 +3218,7 @@ __metadata: cross-env: ^7.0.3 cypress: 11.2 - 13 cypress-multi-reporters: ^1.6.3 + cypress-on-fix: ^1.0.2 dayjs: ^1.10.4 debug: ^4.3.3 deep-equal: ^2.0.5 @@ -4789,6 +4790,7 @@ __metadata: "@unflakable/cypress-plugin": "workspace:^" cypress: 11.2 - 13 cypress-multi-reporters: ^1.6.3 + cypress-on-fix: ^1.0.2 mocha: =7.0.1 mocha-junit-reporter: ^2.2.0 process: ^0.11.10 @@ -4856,6 +4858,13 @@ __metadata: languageName: node linkType: hard +"cypress-on-fix@npm:^1.0.2": + version: 1.0.2 + resolution: "cypress-on-fix@npm:1.0.2" + checksum: b35e0d49e4270237e7cbe95c21d458772d3df6bbb4423346c70f9417e61fdf061ad1d83aca76a854a378d001a68f50c17b8dd312fbe9c50b5d12e61fc317a785 + languageName: node + linkType: hard + "cypress@npm:11.2 - 13": version: 12.14.0 resolution: "cypress@npm:12.14.0" From 56acafcf56b934e4c159ee5a31938f9bac10c084 Mon Sep 17 00:00:00 2001 From: "David A. Ramos" Date: Thu, 21 Sep 2023 19:00:01 -0700 Subject: [PATCH 45/53] [cypress] Work around SIGSEGV in Chrome 117 w/ headless mode The latest Chrome 117 release (now rolled out to GitHub Actions runners) broke interoperability with Cypress <= 12.14 in headless mode. See https://github.com/cypress-io/cypress/issues/27804. This change implements the workaround in https://github.com/cypress-io/cypress/issues/27804#issuecomment-1719882404, which replaces `--headless` with `--headless=new`. See also the underlying Chrome bug here: https://bugs.chromium.org/p/chromium/issues/detail?id=1483163 --- .../config-js/headless.js | 31 +++++++++++++++++ .../integration-input-esm/config/headless.ts | 24 ++++++++++++++ .../integration-input-esm/cypress-config.cjs | 6 ++++ .../integration-input-esm/cypress-config.js | 3 ++ .../integration-input-esm/cypress.config.ts | 3 ++ .../config/headless.js | 33 +++++++++++++++++++ .../cypress-config.mjs | 3 ++ .../cypress.config.js | 3 ++ .../integration-input/config-js/headless.js | 33 +++++++++++++++++++ .../test/integration-input/config/headless.ts | 24 ++++++++++++++ .../test/integration-input/cypress-config.js | 3 ++ .../test/integration-input/cypress-config.mjs | 3 ++ .../test/integration-input/cypress.config.ts | 14 +++++++- .../test/integration-input/tsconfig.json | 1 + 14 files changed, 183 insertions(+), 1 deletion(-) create mode 100644 packages/cypress-plugin/test/integration-input-esm/config-js/headless.js create mode 100644 packages/cypress-plugin/test/integration-input-esm/config/headless.ts create mode 100644 packages/cypress-plugin/test/integration-input-manual/config/headless.js create mode 100644 packages/cypress-plugin/test/integration-input/config-js/headless.js create mode 100644 packages/cypress-plugin/test/integration-input/config/headless.ts diff --git a/packages/cypress-plugin/test/integration-input-esm/config-js/headless.js b/packages/cypress-plugin/test/integration-input-esm/config-js/headless.js new file mode 100644 index 0000000..093bf8c --- /dev/null +++ b/packages/cypress-plugin/test/integration-input-esm/config-js/headless.js @@ -0,0 +1,31 @@ +// Copyright (c) 2023 Developer Innovations, LLC + +/** + * Workaround for https://github.com/cypress-io/cypress/issues/27804. + * + * @param {Cypress.PluginEvents} on + * @returns void + */ +export const fixHeadlessChrome = (on) => { + on( + "before:browser:launch", + /** + * @param {Cypress.Browser} browser, + * @param {Cypress.BrowserLaunchOptions} launchOptions + * @returns {void | Cypress.BrowserLaunchOptions} + */ + (browser, launchOptions) => { + if ( + browser.family === "chromium" && + browser.name !== "electron" && + browser.isHeadless + ) { + launchOptions.args = launchOptions.args.map((arg) => + arg === "--headless" ? "--headless=new" : arg + ); + } + + return launchOptions; + } + ); +}; diff --git a/packages/cypress-plugin/test/integration-input-esm/config/headless.ts b/packages/cypress-plugin/test/integration-input-esm/config/headless.ts new file mode 100644 index 0000000..05519ff --- /dev/null +++ b/packages/cypress-plugin/test/integration-input-esm/config/headless.ts @@ -0,0 +1,24 @@ +// Copyright (c) 2023 Developer Innovations, LLC + +// Workaround for https://github.com/cypress-io/cypress/issues/27804. +export const fixHeadlessChrome = (on: Cypress.PluginEvents): void => { + on( + "before:browser:launch", + ( + browser: Cypress.Browser, + launchOptions: Cypress.BrowserLaunchOptions + ): void | Cypress.BrowserLaunchOptions => { + if ( + browser.family === "chromium" && + browser.name !== "electron" && + browser.isHeadless + ) { + launchOptions.args = launchOptions.args.map((arg) => + arg === "--headless" ? "--headless=new" : arg + ); + } + + return launchOptions; + } + ); +}; diff --git a/packages/cypress-plugin/test/integration-input-esm/cypress-config.cjs b/packages/cypress-plugin/test/integration-input-esm/cypress-config.cjs index cb9736e..ecde34b 100644 --- a/packages/cypress-plugin/test/integration-input-esm/cypress-config.cjs +++ b/packages/cypress-plugin/test/integration-input-esm/cypress-config.cjs @@ -17,6 +17,9 @@ module.exports = { * @returns {Promise | Cypress.PluginConfigOptions | void} */ async setupNodeEvents(on, _config) { + const { fixHeadlessChrome } = await import("./config-js/headless.js"); + + fixHeadlessChrome(on); registerCosmiconfigMock(); registerSimpleGitMock(); @@ -38,6 +41,9 @@ module.exports = { * @returns {Promise | Cypress.PluginConfigOptions | void} */ async setupNodeEvents(on, _config) { + const { fixHeadlessChrome } = await import("./config-js/headless.js"); + + fixHeadlessChrome(on); registerCosmiconfigMock(); registerSimpleGitMock(); diff --git a/packages/cypress-plugin/test/integration-input-esm/cypress-config.js b/packages/cypress-plugin/test/integration-input-esm/cypress-config.js index ee5d049..c203809 100644 --- a/packages/cypress-plugin/test/integration-input-esm/cypress-config.js +++ b/packages/cypress-plugin/test/integration-input-esm/cypress-config.js @@ -5,6 +5,7 @@ import { registerTasks } from "./config-js/tasks.js"; import webpackConfig from "./config-js/webpack.js"; import { registerSimpleGitMock } from "unflakable-test-common/dist/git.js"; import { registerCosmiconfigMock } from "unflakable-test-common/dist/config.js"; +import { fixHeadlessChrome } from "./config-js/headless.js"; /** * @type {Cypress.ConfigOptions} @@ -17,6 +18,7 @@ export default { * @returns {Promise | Cypress.PluginConfigOptions | void} */ setupNodeEvents(on, _config) { + fixHeadlessChrome(on); registerCosmiconfigMock(); registerSimpleGitMock(); registerTasks(on); @@ -35,6 +37,7 @@ export default { * @returns {Promise | Cypress.PluginConfigOptions | void} */ setupNodeEvents(on, _config) { + fixHeadlessChrome(on); registerCosmiconfigMock(); registerSimpleGitMock(); registerTasks(on); diff --git a/packages/cypress-plugin/test/integration-input-esm/cypress.config.ts b/packages/cypress-plugin/test/integration-input-esm/cypress.config.ts index c7f9649..8a8ca57 100644 --- a/packages/cypress-plugin/test/integration-input-esm/cypress.config.ts +++ b/packages/cypress-plugin/test/integration-input-esm/cypress.config.ts @@ -8,10 +8,12 @@ import { registerTasks } from "./config/tasks.js"; import webpackConfig from "./config/webpack.js"; import { registerSimpleGitMock } from "unflakable-test-common/dist/git.js"; import { registerCosmiconfigMock } from "unflakable-test-common/dist/config.js"; +import { fixHeadlessChrome } from "./config/headless.js"; export default defineConfig({ component: { setupNodeEvents(on: Cypress.PluginEvents, _config) { + fixHeadlessChrome(on); registerCosmiconfigMock(); registerSimpleGitMock(); registerTasks(on); @@ -31,6 +33,7 @@ export default defineConfig({ | Promise | Cypress.PluginConfigOptions | void { + fixHeadlessChrome(on); registerCosmiconfigMock(); registerSimpleGitMock(); registerTasks(on); diff --git a/packages/cypress-plugin/test/integration-input-manual/config/headless.js b/packages/cypress-plugin/test/integration-input-manual/config/headless.js new file mode 100644 index 0000000..a116630 --- /dev/null +++ b/packages/cypress-plugin/test/integration-input-manual/config/headless.js @@ -0,0 +1,33 @@ +// Copyright (c) 2023 Developer Innovations, LLC + +module.exports = { + /** + * Workaround for https://github.com/cypress-io/cypress/issues/27804. + * + * @param {Cypress.PluginEvents} on + * @returns void + */ + fixHeadlessChrome: (on) => { + on( + "before:browser:launch", + /** + * @param {Cypress.Browser} browser, + * @param {Cypress.BrowserLaunchOptions} launchOptions + * @returns {void | Cypress.BrowserLaunchOptions} + */ + (browser, launchOptions) => { + if ( + browser.family === "chromium" && + browser.name !== "electron" && + browser.isHeadless + ) { + launchOptions.args = launchOptions.args.map((arg) => + arg === "--headless" ? "--headless=new" : arg + ); + } + + return launchOptions; + } + ); + }, +}; diff --git a/packages/cypress-plugin/test/integration-input-manual/cypress-config.mjs b/packages/cypress-plugin/test/integration-input-manual/cypress-config.mjs index d5d5a61..1f976f6 100644 --- a/packages/cypress-plugin/test/integration-input-manual/cypress-config.mjs +++ b/packages/cypress-plugin/test/integration-input-manual/cypress-config.mjs @@ -8,6 +8,7 @@ import { registerCosmiconfigMock } from "unflakable-test-common/dist/config.js"; import { registerUnflakable } from "@unflakable/cypress-plugin"; import semverGte from "semver/functions/gte.js"; import path from "path"; +import headless from "./config/headless.js"; import cypressOnFix from "cypress-on-fix"; /** @@ -22,6 +23,7 @@ export default { */ setupNodeEvents(baseOn, config) { const on = cypressOnFix(baseOn); + headless.fixHeadlessChrome(on); registerCosmiconfigMock(); registerSimpleGitMock(); tasks.registerTasks(on); @@ -43,6 +45,7 @@ export default { */ setupNodeEvents(baseOn, config) { const on = cypressOnFix(baseOn); + headless.fixHeadlessChrome(on); registerCosmiconfigMock(); registerSimpleGitMock(); tasks.registerTasks(on); diff --git a/packages/cypress-plugin/test/integration-input-manual/cypress.config.js b/packages/cypress-plugin/test/integration-input-manual/cypress.config.js index 376bd1f..4f34373 100644 --- a/packages/cypress-plugin/test/integration-input-manual/cypress.config.js +++ b/packages/cypress-plugin/test/integration-input-manual/cypress.config.js @@ -11,6 +11,7 @@ const semverGte = require("semver/functions/gte"); const { registerUnflakable } = require("@unflakable/cypress-plugin"); const path = require("path"); +const { fixHeadlessChrome } = require("./config/headless"); const cypressOnFix = require("cypress-on-fix"); module.exports = { @@ -25,6 +26,7 @@ module.exports = { */ setupNodeEvents(baseOn, config) { const on = cypressOnFix(baseOn); + fixHeadlessChrome(on); registerCosmiconfigMock(); registerSimpleGitMock(); registerTasks(on); @@ -46,6 +48,7 @@ module.exports = { */ setupNodeEvents(baseOn, config) { const on = cypressOnFix(baseOn); + fixHeadlessChrome(on); registerCosmiconfigMock(); registerSimpleGitMock(); registerTasks(on); diff --git a/packages/cypress-plugin/test/integration-input/config-js/headless.js b/packages/cypress-plugin/test/integration-input/config-js/headless.js new file mode 100644 index 0000000..a116630 --- /dev/null +++ b/packages/cypress-plugin/test/integration-input/config-js/headless.js @@ -0,0 +1,33 @@ +// Copyright (c) 2023 Developer Innovations, LLC + +module.exports = { + /** + * Workaround for https://github.com/cypress-io/cypress/issues/27804. + * + * @param {Cypress.PluginEvents} on + * @returns void + */ + fixHeadlessChrome: (on) => { + on( + "before:browser:launch", + /** + * @param {Cypress.Browser} browser, + * @param {Cypress.BrowserLaunchOptions} launchOptions + * @returns {void | Cypress.BrowserLaunchOptions} + */ + (browser, launchOptions) => { + if ( + browser.family === "chromium" && + browser.name !== "electron" && + browser.isHeadless + ) { + launchOptions.args = launchOptions.args.map((arg) => + arg === "--headless" ? "--headless=new" : arg + ); + } + + return launchOptions; + } + ); + }, +}; diff --git a/packages/cypress-plugin/test/integration-input/config/headless.ts b/packages/cypress-plugin/test/integration-input/config/headless.ts new file mode 100644 index 0000000..05519ff --- /dev/null +++ b/packages/cypress-plugin/test/integration-input/config/headless.ts @@ -0,0 +1,24 @@ +// Copyright (c) 2023 Developer Innovations, LLC + +// Workaround for https://github.com/cypress-io/cypress/issues/27804. +export const fixHeadlessChrome = (on: Cypress.PluginEvents): void => { + on( + "before:browser:launch", + ( + browser: Cypress.Browser, + launchOptions: Cypress.BrowserLaunchOptions + ): void | Cypress.BrowserLaunchOptions => { + if ( + browser.family === "chromium" && + browser.name !== "electron" && + browser.isHeadless + ) { + launchOptions.args = launchOptions.args.map((arg) => + arg === "--headless" ? "--headless=new" : arg + ); + } + + return launchOptions; + } + ); +}; diff --git a/packages/cypress-plugin/test/integration-input/cypress-config.js b/packages/cypress-plugin/test/integration-input/cypress-config.js index 50319f0..2ffbfe6 100644 --- a/packages/cypress-plugin/test/integration-input/cypress-config.js +++ b/packages/cypress-plugin/test/integration-input/cypress-config.js @@ -7,6 +7,7 @@ const { registerSimpleGitMock } = require("unflakable-test-common/dist/git"); const { registerCosmiconfigMock, } = require("unflakable-test-common/dist/config"); +const { fixHeadlessChrome } = require("config-js/headless"); module.exports = { /** @@ -20,6 +21,7 @@ module.exports = { * @returns {Promise | Cypress.PluginConfigOptions | void} */ setupNodeEvents(on, _config) { + fixHeadlessChrome(on); registerCosmiconfigMock(); registerSimpleGitMock(); registerTasks(on); @@ -38,6 +40,7 @@ module.exports = { * @returns {Promise | Cypress.PluginConfigOptions | void} */ setupNodeEvents(on, _config) { + fixHeadlessChrome(on); registerCosmiconfigMock(); registerSimpleGitMock(); registerTasks(on); diff --git a/packages/cypress-plugin/test/integration-input/cypress-config.mjs b/packages/cypress-plugin/test/integration-input/cypress-config.mjs index b6e6c0f..c5d6b38 100644 --- a/packages/cypress-plugin/test/integration-input/cypress-config.mjs +++ b/packages/cypress-plugin/test/integration-input/cypress-config.mjs @@ -5,6 +5,7 @@ import tasks from "./config-js/tasks.js"; import webpackConfig from "./config-js/webpack.js"; import { registerSimpleGitMock } from "unflakable-test-common/dist/git.js"; import { registerCosmiconfigMock } from "unflakable-test-common/dist/config.js"; +import headless from "./config-js/headless.js"; /** * @type {Cypress.ConfigOptions} @@ -17,6 +18,7 @@ export default { * @returns {Promise | Cypress.PluginConfigOptions | void} */ setupNodeEvents(on, _config) { + headless.fixHeadlessChrome(on); registerCosmiconfigMock(); registerSimpleGitMock(); tasks.registerTasks(on); @@ -35,6 +37,7 @@ export default { * @returns {Promise | Cypress.PluginConfigOptions | void} */ setupNodeEvents(on, _config) { + headless.fixHeadlessChrome(on); registerCosmiconfigMock(); registerSimpleGitMock(); tasks.registerTasks(on); diff --git a/packages/cypress-plugin/test/integration-input/cypress.config.ts b/packages/cypress-plugin/test/integration-input/cypress.config.ts index 1a6359b..5375c6a 100644 --- a/packages/cypress-plugin/test/integration-input/cypress.config.ts +++ b/packages/cypress-plugin/test/integration-input/cypress.config.ts @@ -12,10 +12,14 @@ import { openDevToolsOnLaunch } from "config/devtools"; import webpackConfig from "config/webpack"; // eslint-disable-next-line import/no-unresolved import { registerTasks } from "config/tasks"; +// eslint-disable-next-line import/no-unresolved +import { fixHeadlessChrome } from "config/headless"; +import cypressOnFix from "cypress-on-fix"; export default defineConfig({ component: { setupNodeEvents(on: Cypress.PluginEvents, _config) { + fixHeadlessChrome(on); registerCosmiconfigMock(); registerSimpleGitMock(); registerTasks(on); @@ -30,12 +34,20 @@ export default defineConfig({ }, e2e: { setupNodeEvents( - on: Cypress.PluginEvents, + baseOn: Cypress.PluginEvents, _config ): | Promise | Cypress.PluginConfigOptions | void { + // Due to https://github.com/cypress-io/cypress/issues/22428, only the last event handler + // registered for each event type will be called. This means we'll clobber any event handlers + // the user registers. To avoid this, we use cypress-on-fix. + // NB: Our plugin ordinarily does this for us, but we use this package to test what happens + // when the plugin is disabled. + const on = cypressOnFix(baseOn); + + fixHeadlessChrome(on); registerCosmiconfigMock(); registerSimpleGitMock(); registerTasks(on); diff --git a/packages/cypress-plugin/test/integration-input/tsconfig.json b/packages/cypress-plugin/test/integration-input/tsconfig.json index e3ff93e..e7f8717 100644 --- a/packages/cypress-plugin/test/integration-input/tsconfig.json +++ b/packages/cypress-plugin/test/integration-input/tsconfig.json @@ -11,6 +11,7 @@ "types": ["cypress", "node"] }, "include": [ + "../../cypress-on-fix.d.ts", ".eslintrc.js", "config", "config-js", From 7aa9fd19cafa12730fc2b622e544c615d6e5eee4 Mon Sep 17 00:00:00 2001 From: "David A. Ramos" Date: Mon, 25 Sep 2023 17:40:51 -0700 Subject: [PATCH 46/53] Run CI at midnight Pacific Many Cypress Windows tests are timing out even after increasing the timeout to 2 hours, likely due to https://github.com/actions/runner-images/issues/7320. Hopefully running the tests at night during reduced load will reduce the prevalence of noisy neighbors since the GitHub Windows runners seem unable to properly isolate compute-heavy workloads. --- .github/workflows/ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 007a034..bf8e32c 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -8,8 +8,8 @@ on: # Run the pipeline daily so that we get continuous dogfooding. schedule: - # Run at 6pm UTC/10am Pacific - - cron: 0 18 * * * + # Run at 8am UTC/midnight Pacific + - cron: 0 8 * * * # Allows you to run this workflow manually from the Actions tab workflow_dispatch: {} From df0ced3d73a86a2f0bfd75d57acdf2bc59c0e3fa Mon Sep 17 00:00:00 2001 From: "David A. Ramos" Date: Wed, 27 Sep 2023 13:44:01 -0700 Subject: [PATCH 47/53] [cypress] Support Cypress 13.3 --- .github/workflows/ci.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index bf8e32c..0ab8e81 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -131,6 +131,7 @@ jobs: - "13.0" - "13.1" - "13.2" + - "13.3" steps: - uses: actions/checkout@v4 @@ -240,6 +241,7 @@ jobs: - "13.0" - "13.1" - "13.2" + - "13.3" steps: - uses: actions/checkout@v4 From ed1c924156eefffb3f5903ed7e80293cefa2ab2a Mon Sep 17 00:00:00 2001 From: "David A. Ramos" Date: Wed, 27 Sep 2023 16:31:51 -0700 Subject: [PATCH 48/53] [cypress] Increase Windows CI timeout to 3 hours Github Actions Windows runners have become abysmally slow, with individual trivial Cypress test cases often taking two minutes just to launch Edge, do nothing, and shut down. Without paying for self-hosted runners, there's not much we can do except increase the timeout further. --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 0ab8e81..4bd9a3f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -213,7 +213,7 @@ jobs: name: "Cypress ${{ matrix.cypress }} Windows Node ${{ matrix.node }} Integration Tests" runs-on: windows-2019 # Cypress on Windows is slowwww... - timeout-minutes: 120 + timeout-minutes: 180 needs: # Don't incur the cost of the test matrix if the basic build fails. - check From 8cb9af1bb23c1d8981aef94e99c4410fdd0dddc8 Mon Sep 17 00:00:00 2001 From: "David A. Ramos" Date: Mon, 30 Oct 2023 15:36:24 -0700 Subject: [PATCH 49/53] [cypress] Support Cypress 13.4 --- .github/workflows/ci.yaml | 2 + .../integration/src/hook-failures.test.ts | 53 +++++++++++-------- .../test/integration/src/verify-output.ts | 19 ++++--- 3 files changed, 47 insertions(+), 27 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4bd9a3f..c30d6b8 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -132,6 +132,7 @@ jobs: - "13.1" - "13.2" - "13.3" + - "13.4" steps: - uses: actions/checkout@v4 @@ -242,6 +243,7 @@ jobs: - "13.1" - "13.2" - "13.3" + - "13.4" steps: - uses: actions/checkout@v4 diff --git a/packages/cypress-plugin/test/integration/src/hook-failures.test.ts b/packages/cypress-plugin/test/integration/src/hook-failures.test.ts index 06df22f..1294218 100644 --- a/packages/cypress-plugin/test/integration/src/hook-failures.test.ts +++ b/packages/cypress-plugin/test/integration/src/hook-failures.test.ts @@ -1,6 +1,14 @@ // Copyright (c) 2023 Developer Innovations, LLC import { integrationTest, integrationTestSuite } from "./test-wrappers"; +import semverLt from "semver/functions/lt"; +import cypressPackage from "cypress/package.json"; + +// Cypress 13.4 broke the handling of multiple hook failures. Prior to that version, Cypress/Mocha +// reported both errors as failures of the first test in the suite, and then skipped all remaining +// tests. Beginning in 13.4, Cypress skips all tests in the suite and never reports either error. +// This was most likely introduced in https://github.com/cypress-io/cypress/pull/27930. +const supportMultipleHookErrors = semverLt(cypressPackage.version, "13.4.0"); integrationTestSuite((mockBackend) => { it("run should succeed when before() fails and both tests are quarantined", (done) => @@ -303,28 +311,31 @@ integrationTestSuite((mockBackend) => { done )); - it("multiple before() hook errors", (done) => - integrationTest( - { - params: { - multipleHookErrors: true, - specNameStubs: ["hook-fail"], - }, - expectedExitCode: 1, - summaryTotals: { - icon: "fail", - numFailing: 1, - numFlaky: 0, - numPassing: 0, - numPending: 0, - numQuarantined: 0, - numSkipped: 1, - numTests: 2, + (supportMultipleHookErrors ? it : it.skip)( + "multiple before() hook errors", + (done) => + integrationTest( + { + params: { + multipleHookErrors: true, + specNameStubs: ["hook-fail"], + }, + expectedExitCode: 1, + summaryTotals: { + icon: "fail", + numFailing: 1, + numFlaky: 0, + numPassing: 0, + numPending: 0, + numQuarantined: 0, + numSkipped: 1, + numTests: 2, + }, }, - }, - mockBackend, - done - )); + mockBackend, + done + ) + ); it("test and afterEach() hook errors", (done) => integrationTest( diff --git a/packages/cypress-plugin/test/integration/src/verify-output.ts b/packages/cypress-plugin/test/integration/src/verify-output.ts index 92c79c7..f8d5f0d 100644 --- a/packages/cypress-plugin/test/integration/src/verify-output.ts +++ b/packages/cypress-plugin/test/integration/src/verify-output.ts @@ -22,11 +22,18 @@ import { import { expect as expectExt } from "@jest/globals"; import escapeStringRegexp from "escape-string-regexp"; import { TestAttemptResult } from "@unflakable/js-api"; +import semverGte from "semver/functions/gte"; +import cypressPackage from "cypress/package.json"; const THROWN_ERROR = "\x1B[0m\x1B[31m Error\x1B[0m\x1B[90m"; const FAIL_SYMBOL = process.platform === "win32" ? "×" : "✖"; const PASS_SYMBOL = process.platform === "win32" ? "√" : "✓"; +// Cypress 13.4 changed the number of retries returned after a retry passes (i.e., when a test is +// flaky). See: +// https://github.com/cypress-io/cypress/blob/5a95541c3c4e48bfc67a54642abc949576fa6f05/packages/driver/patches/mocha%2B7.0.1.dev.patch +const adjustRetriesOnPass = semverGte(cypressPackage.version, "13.4.0"); + const verifySpecOutput = ( params: TestCaseParams, specOutputs: SpecOutput[], @@ -141,7 +148,7 @@ const verifySpecOutputs = ( new RegExp( // eslint-disable-next-line no-control-regex `^ {2}\x1B\\[35m {2}${PASS_SYMBOL}\x1B\\[39m\x1B\\[90m mixed: flake should be quarantined\x1B\\[0m\x1B\\[35m \\[flaky, quarantined]\x1B\\[39m\x1B\\[33m \\(attempt 2 of ${ - expectedRetries + 1 + adjustRetriesOnPass ? 2 : expectedRetries + 1 }\\)\x1B\\[0m\x1B\\[(?:33|90)m \\([0-9]+.+?\\)\x1B\\[0m$` ) ), @@ -169,7 +176,7 @@ const verifySpecOutputs = ( new RegExp( // eslint-disable-next-line no-control-regex `^ {2}\x1B\\[33m {2}${PASS_SYMBOL}\x1B\\[0m\x1B\\[90m mixed: flake should be quarantined\x1B\\[0m\x1B\\[33m \\[flaky]\x1B\\[0m\x1B\\[33m \\(attempt 2 of ${ - expectedRetries + 1 + adjustRetriesOnPass ? 2 : expectedRetries + 1 }\\)\x1B\\[0m\x1B\\[(?:33|90)m \\([0-9]+.+?\\)\x1B\\[0m$` ) ), @@ -208,7 +215,7 @@ const verifySpecOutputs = ( new RegExp( // eslint-disable-next-line no-control-regex `^ {2}\x1B\\[35m {2}${PASS_SYMBOL}\x1B\\[39m\x1B\\[90m mixed: should be flaky\x1B\\[0m\x1B\\[35m \\[flaky, quarantined]\x1B\\[39m\x1B\\[33m \\(attempt 2 of ${ - expectedRetries + 1 + adjustRetriesOnPass ? 2 : expectedRetries + 1 }\\)\x1B\\[0m\x1B\\[(?:33|90)m \\([0-9]+.+?\\)\x1B\\[0m$` ) ) @@ -216,7 +223,7 @@ const verifySpecOutputs = ( new RegExp( // eslint-disable-next-line no-control-regex `^ {2}\x1B\\[33m {2}${PASS_SYMBOL}\x1B\\[0m\x1B\\[90m mixed: should be flaky\x1B\\[0m\x1B\\[33m \\[flaky]\x1B\\[0m\x1B\\[33m \\(attempt 2 of ${ - expectedRetries + 1 + adjustRetriesOnPass ? 2 : expectedRetries + 1 }\\)\x1B\\[0m\x1B\\[(?:33|90)m \\([0-9]+.+?\\)\x1B\\[0m$` ) ), @@ -708,7 +715,7 @@ const verifySpecOutputs = ( `^\x1B\\[35m {2}${PASS_SYMBOL}\x1B\\[39m\x1B\\[90m should be flaky${escapeStringRegexp( expectedFlakeTestNameSuffix )}\x1B\\[0m\x1B\\[35m \\[flaky, quarantined]\x1B\\[39m\x1B\\[33m \\(attempt 2 of ${ - expectedRetries + 1 + adjustRetriesOnPass ? 2 : expectedRetries + 1 }\\)\x1B\\[0m\x1B\\[(?:33|90)m \\([0-9]+.+?\\)\x1B\\[0m$` ) ) @@ -718,7 +725,7 @@ const verifySpecOutputs = ( `^\x1B\\[33m {2}${PASS_SYMBOL}\x1B\\[0m\x1B\\[90m should be flaky${escapeStringRegexp( expectedFlakeTestNameSuffix )}\x1B\\[0m\x1B\\[33m \\[flaky]\x1B\\[0m\x1B\\[33m \\(attempt 2 of ${ - expectedRetries + 1 + adjustRetriesOnPass ? 2 : expectedRetries + 1 }\\)\x1B\\[0m\x1B\\[(?:33|90)m \\([0-9]+.+?\\)\x1B\\[0m$` ) ), From d9de4571085121f920c815968a97490a847554d7 Mon Sep 17 00:00:00 2001 From: "David A. Ramos" Date: Tue, 31 Oct 2023 12:55:04 -0700 Subject: [PATCH 50/53] [cypress] Disable Cypress 13.3 in CI Due to https://github.com/cypress-io/cypress/issues/28141 and https://github.com/cypress-io/cypress/issues/28148, Cypress 13.3 is too flaky to test in CI. This regression was fixed in Cypress 13.4. --- .github/workflows/ci.yaml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c30d6b8..3bf1171 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -131,7 +131,11 @@ jobs: - "13.0" - "13.1" - "13.2" - - "13.3" + # 13.3.2 introduced a regression that made that version (and 13.3.3) too flaky to test. + # See: + # - https://github.com/cypress-io/cypress/issues/28141 + # - https://github.com/cypress-io/cypress/issues/28148 + #- "13.3" - "13.4" steps: - uses: actions/checkout@v4 @@ -242,7 +246,7 @@ jobs: - "13.0" - "13.1" - "13.2" - - "13.3" + #- "13.3" - "13.4" steps: - uses: actions/checkout@v4 From b30237dd280135c5eca4adf64a6fb7de5cfcbd6b Mon Sep 17 00:00:00 2001 From: "David A. Ramos" Date: Tue, 31 Oct 2023 21:43:08 -0700 Subject: [PATCH 51/53] [cypress] Treat Cypress verification timeout as test-independent --- packages/cypress-plugin/test/integration/unflakable.js | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/cypress-plugin/test/integration/unflakable.js b/packages/cypress-plugin/test/integration/unflakable.js index 96914fb..7e9c473 100644 --- a/packages/cypress-plugin/test/integration/unflakable.js +++ b/packages/cypress-plugin/test/integration/unflakable.js @@ -16,5 +16,6 @@ module.exports = { // bug. Instead, we treat it as a test independent failure iff this error message is in the // output. Otherwise, we'll still treat it as a true failure. /Timed out waiting for the browser to connect. Retrying\.\.\./, + /Cypress verification timed out\./, ], }; From dff9b5903199039366eb257709abb36649224846 Mon Sep 17 00:00:00 2001 From: "David A. Ramos" Date: Wed, 8 Nov 2023 16:23:34 -0800 Subject: [PATCH 52/53] [cypress] Support Cypress 13.5 --- .github/workflows/ci.yaml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 3bf1171..840be35 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -137,6 +137,7 @@ jobs: # - https://github.com/cypress-io/cypress/issues/28148 #- "13.3" - "13.4" + - "13.5" steps: - uses: actions/checkout@v4 @@ -175,6 +176,9 @@ jobs: mv package-new.json package.json yarn install --no-immutable + - name: Build test dependencies + run: yarn build:plugins-common && yarn build:test-common && yarn build:cypress-tests + - name: Set Cypress version env: CYPRESS_INSTALL_BINARY: "0" @@ -185,9 +189,6 @@ jobs: - name: Install Cypress binary run: yarn workspace cypress-integration exec cypress install - - name: Build test dependencies - run: yarn build:plugins-common && yarn build:test-common && yarn build:cypress-tests - - name: Test env: # Enable debug logs within the Jest tests that run Cypress. WARNING: these are very @@ -248,6 +249,7 @@ jobs: - "13.2" #- "13.3" - "13.4" + - "13.5" steps: - uses: actions/checkout@v4 @@ -290,6 +292,9 @@ jobs: mv package-new.json package.json yarn install --no-immutable + - name: Build test dependencies + run: yarn build:plugins-common && yarn build:test-common && yarn build:cypress-tests + - name: Set Cypress version env: CYPRESS_INSTALL_BINARY: "0" @@ -300,9 +305,6 @@ jobs: - name: Install Cypress binary run: yarn workspace cypress-integration exec cypress install - - name: Build test dependencies - run: yarn build:plugins-common && yarn build:test-common && yarn build:cypress-tests - - name: Test env: # Enable debug logs within the Jest tests that run Cypress. WARNING: these are very From b91440d765b090b7661ed19410481f1bd2764a4f Mon Sep 17 00:00:00 2001 From: "David A. Ramos" Date: Mon, 1 Jan 2024 22:58:32 -0800 Subject: [PATCH 53/53] Update copyright years --- .eslintrc-ts.js | 2 +- .eslintrc.js | 2 +- LICENSE | 2 +- packages/cypress-plugin/cypress-on-fix.d.ts | 2 +- packages/cypress-plugin/jest.config.js | 2 +- packages/cypress-plugin/mocha.d.ts | 2 +- packages/cypress-plugin/rollup.config.mjs | 2 +- packages/cypress-plugin/src/.eslintrc.js | 2 +- packages/cypress-plugin/src/config-env-vars.ts | 2 +- packages/cypress-plugin/src/config-wrapper-sync.ts | 2 +- packages/cypress-plugin/src/config-wrapper.ts | 2 +- packages/cypress-plugin/src/cypress-env-vars.ts | 2 +- packages/cypress-plugin/src/index.ts | 2 +- packages/cypress-plugin/src/load-user-config.ts | 2 +- packages/cypress-plugin/src/main.ts | 2 +- packages/cypress-plugin/src/plugin.ts | 2 +- packages/cypress-plugin/src/reporter-common.ts | 2 +- packages/cypress-plugin/src/reporter-config.test.ts | 2 +- packages/cypress-plugin/src/reporter-config.ts | 2 +- packages/cypress-plugin/src/reporter.ts | 2 +- packages/cypress-plugin/src/skip-tests.ts | 2 +- packages/cypress-plugin/src/utils.ts | 2 +- packages/cypress-plugin/src/vendored/cli-table3.d.ts | 2 +- packages/cypress-plugin/test/.eslintrc.cjs | 2 +- .../cypress-plugin/test/integration-input-esm/.eslintrc.cjs | 2 +- .../test/integration-input-esm/config-js/devtools.js | 2 +- .../test/integration-input-esm/config-js/headless.js | 2 +- .../test/integration-input-esm/config-js/tasks.js | 2 +- .../test/integration-input-esm/config-js/webpack.js | 2 +- .../test/integration-input-esm/config/devtools.ts | 2 +- .../test/integration-input-esm/config/headless.ts | 2 +- .../cypress-plugin/test/integration-input-esm/config/tasks.ts | 2 +- .../cypress-plugin/test/integration-input-esm/config/webpack.ts | 2 +- .../test/integration-input-esm/cypress-config.cjs | 2 +- .../cypress-plugin/test/integration-input-esm/cypress-config.js | 2 +- .../cypress-plugin/test/integration-input-esm/cypress.config.ts | 2 +- .../test/integration-input-esm/cypress/support-js/commands.js | 2 +- .../test/integration-input-esm/cypress/support-js/component.cjs | 2 +- .../test/integration-input-esm/cypress/support-js/component.js | 2 +- .../test/integration-input-esm/cypress/support-js/e2e.cjs | 2 +- .../test/integration-input-esm/cypress/support-js/e2e.js | 2 +- .../test/integration-input-esm/cypress/support/commands.ts | 2 +- .../integration-input-esm/cypress/support/component-index.html | 2 ++ .../test/integration-input-esm/cypress/support/component.ts | 2 +- .../test/integration-input-esm/cypress/support/e2e.ts | 2 +- .../test/integration-input-manual/config/devtools.js | 2 +- .../test/integration-input-manual/config/headless.js | 2 +- .../test/integration-input-manual/config/tasks.js | 2 +- .../test/integration-input-manual/config/webpack.js | 2 +- .../test/integration-input-manual/cypress-config.mjs | 2 +- .../test/integration-input-manual/cypress.config.js | 2 +- .../test/integration-input-manual/cypress/e2e/fail.cy.js | 2 +- .../test/integration-input-manual/cypress/e2e/flake.cy.js | 2 +- .../test/integration-input-manual/cypress/e2e/hook-fail.cy.js | 2 +- .../test/integration-input-manual/cypress/e2e/invalid.cy.js | 2 +- .../test/integration-input-manual/cypress/e2e/mixed/mixed.cy.js | 2 +- .../test/integration-input-manual/cypress/e2e/pass.cy.js | 2 +- .../test/integration-input-manual/cypress/e2e/pending.cy.js | 2 +- .../test/integration-input-manual/cypress/e2e/quarantined.cy.js | 2 +- .../test/integration-input-manual/cypress/support/commands.js | 2 +- .../cypress/support/component-index.html | 2 ++ .../test/integration-input-manual/cypress/support/component.js | 2 +- .../integration-input-manual/cypress/support/e2e-webpack5.js | 2 +- .../test/integration-input-manual/cypress/support/e2e.js | 2 +- packages/cypress-plugin/test/integration-input/.eslintrc.js | 2 +- .../cypress-plugin/test/integration-input/config-js/devtools.js | 2 +- .../cypress-plugin/test/integration-input/config-js/headless.js | 2 +- .../cypress-plugin/test/integration-input/config-js/tasks.js | 2 +- .../cypress-plugin/test/integration-input/config-js/webpack.js | 2 +- .../cypress-plugin/test/integration-input/config/devtools.ts | 2 +- .../cypress-plugin/test/integration-input/config/headless.ts | 2 +- packages/cypress-plugin/test/integration-input/config/tasks.ts | 2 +- .../cypress-plugin/test/integration-input/config/webpack.ts | 2 +- .../cypress-plugin/test/integration-input/cypress-config.js | 2 +- .../cypress-plugin/test/integration-input/cypress-config.mjs | 2 +- .../cypress-plugin/test/integration-input/cypress.config.ts | 2 +- .../test/integration-input/cypress/component/fail.cy.ts | 2 +- .../test/integration-input/cypress/component/flake.cy.ts | 2 +- .../test/integration-input/cypress/component/hook-fail.cy.ts | 2 +- .../test/integration-input/cypress/component/invalid.cy.ts | 2 +- .../test/integration-input/cypress/component/mixed/mixed.cy.ts | 2 +- .../test/integration-input/cypress/component/pass.cy.ts | 2 +- .../test/integration-input/cypress/component/pending.cy.ts | 2 +- .../test/integration-input/cypress/component/quarantined.cy.ts | 2 +- .../test/integration-input/cypress/support-js/commands.js | 2 +- .../test/integration-input/cypress/support-js/component.js | 2 +- .../test/integration-input/cypress/support-js/component.mjs | 2 +- .../test/integration-input/cypress/support-js/e2e.js | 2 +- .../test/integration-input/cypress/support-js/e2e.mjs | 2 +- .../test/integration-input/cypress/support/commands.ts | 2 +- .../test/integration-input/cypress/support/component-index.html | 2 ++ .../test/integration-input/cypress/support/component.ts | 2 +- .../test/integration-input/cypress/support/e2e.ts | 2 +- packages/cypress-plugin/test/integration/.eslintrc.js | 2 +- packages/cypress-plugin/test/integration/jest.config.js | 2 +- .../cypress-plugin/test/integration/src/basic-matrix.test.ts | 2 +- packages/cypress-plugin/test/integration/src/basic.test.ts | 2 +- packages/cypress-plugin/test/integration/src/config.test.ts | 2 +- .../cypress-plugin/test/integration/src/disable-plugin.test.ts | 2 +- .../cypress-plugin/test/integration/src/disable-upload.test.ts | 2 +- packages/cypress-plugin/test/integration/src/git.test.ts | 2 +- .../cypress-plugin/test/integration/src/hook-failures.test.ts | 2 +- packages/cypress-plugin/test/integration/src/long-names.test.ts | 2 +- packages/cypress-plugin/test/integration/src/matchers.ts | 2 +- .../cypress-plugin/test/integration/src/no-quarantine.test.ts | 2 +- packages/cypress-plugin/test/integration/src/parse-output.ts | 2 +- .../cypress-plugin/test/integration/src/plugin-failures.test.ts | 2 +- packages/cypress-plugin/test/integration/src/retries.test.ts | 2 +- packages/cypress-plugin/test/integration/src/run-test-case.ts | 2 +- packages/cypress-plugin/test/integration/src/test-wrappers.ts | 2 +- packages/cypress-plugin/test/integration/src/unicode.test.ts | 2 +- packages/cypress-plugin/test/integration/src/verify-output.ts | 2 +- packages/cypress-plugin/test/integration/unflakable.js | 2 +- packages/jest-plugin/.eslintrc.js | 2 +- packages/jest-plugin/LICENSE | 2 +- packages/jest-plugin/jest-circus.d.ts | 2 +- packages/jest-plugin/jest.config.js | 2 +- packages/jest-plugin/rollup.config.mjs | 2 +- packages/jest-plugin/src/config.test.ts | 2 +- packages/jest-plugin/src/config.ts | 2 +- packages/jest-plugin/src/index.ts | 2 +- packages/jest-plugin/src/reporter.ts | 2 +- packages/jest-plugin/src/runner.ts | 2 +- packages/jest-plugin/src/test-runner.ts | 2 +- packages/jest-plugin/src/types.ts | 2 +- packages/jest-plugin/src/utils.ts | 2 +- packages/jest-plugin/src/vendored/SummaryReporter.ts | 2 +- packages/jest-plugin/src/vendored/getResultHeader.ts | 2 +- packages/jest-plugin/src/vendored/getSummary.ts | 2 +- packages/jest-plugin/test/.eslintrc.js | 2 +- packages/jest-plugin/test/babel.config.js | 2 +- packages/jest-plugin/test/integration-input/jest.config.js | 2 +- packages/jest-plugin/test/integration-input/src/fail.test.ts | 2 +- packages/jest-plugin/test/integration-input/src/flake.test.ts | 2 +- packages/jest-plugin/test/integration-input/src/invalid.test.ts | 2 +- packages/jest-plugin/test/integration-input/src/mixed.test.ts | 2 +- packages/jest-plugin/test/integration-input/src/pass.test.ts | 2 +- .../jest-plugin/test/integration-input/src/quarantined.test.ts | 2 +- packages/jest-plugin/test/integration/jest.config.js | 2 +- packages/jest-plugin/test/integration/src/basic.test.ts | 2 +- packages/jest-plugin/test/integration/src/config.test.ts | 2 +- .../jest-plugin/test/integration/src/disable-plugin.test.ts | 2 +- .../jest-plugin/test/integration/src/disable-upload.test.ts | 2 +- packages/jest-plugin/test/integration/src/force-color.js | 2 +- packages/jest-plugin/test/integration/src/git.test.ts | 2 +- .../jest-plugin/test/integration/src/ignore-failures.test.ts | 2 +- packages/jest-plugin/test/integration/src/long-names.test.ts | 2 +- packages/jest-plugin/test/integration/src/matchers.ts | 2 +- packages/jest-plugin/test/integration/src/no-quarantine.test.ts | 2 +- .../jest-plugin/test/integration/src/plugin-failures.test.ts | 2 +- packages/jest-plugin/test/integration/src/retries.test.ts | 2 +- packages/jest-plugin/test/integration/src/run-test-case.ts | 2 +- packages/jest-plugin/test/integration/src/skip-tests.test.ts | 2 +- packages/jest-plugin/test/integration/src/snapshots.test.ts | 2 +- .../jest-plugin/test/integration/src/test-independence.test.ts | 2 +- .../jest-plugin/test/integration/src/test-name-pattern.test.ts | 2 +- packages/jest-plugin/test/integration/src/test-wrappers.ts | 2 +- packages/jest-plugin/test/integration/src/verify-output.ts | 2 +- packages/jest-plugin/window.d.ts | 2 +- packages/js-api/.eslintrc.js | 2 +- packages/js-api/LICENSE | 2 +- packages/js-api/src/consts.ts | 2 +- packages/js-api/src/index.ts | 2 +- packages/plugins-common/.eslintrc.js | 2 +- packages/plugins-common/src/config.ts | 2 +- packages/plugins-common/src/env.ts | 2 +- packages/plugins-common/src/git.ts | 2 +- packages/plugins-common/src/index.ts | 2 +- packages/plugins-common/src/manifest.ts | 2 +- packages/plugins-common/src/quarantine.ts | 2 +- packages/test-common/.eslintrc.js | 2 +- packages/test-common/rollup.config.mjs | 2 +- packages/test-common/src/config.ts | 2 +- packages/test-common/src/git.ts | 2 +- packages/test-common/src/mock-backend.ts | 2 +- packages/test-common/src/mock-cosmiconfig.ts | 2 +- packages/test-common/src/mock-git.ts | 2 +- packages/test-common/src/spawn.ts | 2 +- scripts/.eslintrc.js | 2 +- scripts/set-jest-version.ts | 2 +- 180 files changed, 183 insertions(+), 177 deletions(-) diff --git a/.eslintrc-ts.js b/.eslintrc-ts.js index a8798c8..1f41f9c 100644 --- a/.eslintrc-ts.js +++ b/.eslintrc-ts.js @@ -1,4 +1,4 @@ -// Copyright (c) 2022-2023 Developer Innovations, LLC +// Copyright (c) 2022-2024 Developer Innovations, LLC module.exports = { extends: [ diff --git a/.eslintrc.js b/.eslintrc.js index 689ec41..fde84cc 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -1,4 +1,4 @@ -// Copyright (c) 2022-2023 Developer Innovations, LLC +// Copyright (c) 2022-2024 Developer Innovations, LLC module.exports = { env: { diff --git a/LICENSE b/LICENSE index 53bf01b..fc24ed8 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2022-2023 Developer Innovations, LLC +Copyright (c) 2022-2024 Developer Innovations, LLC Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/packages/cypress-plugin/cypress-on-fix.d.ts b/packages/cypress-plugin/cypress-on-fix.d.ts index abce4b3..7d24e1b 100644 --- a/packages/cypress-plugin/cypress-on-fix.d.ts +++ b/packages/cypress-plugin/cypress-on-fix.d.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC declare module "cypress-on-fix" { import * as Cypress from "cypress"; diff --git a/packages/cypress-plugin/jest.config.js b/packages/cypress-plugin/jest.config.js index d2d9b3c..5ed0700 100644 --- a/packages/cypress-plugin/jest.config.js +++ b/packages/cypress-plugin/jest.config.js @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC /** @type {import('ts-jest').JestConfigWithTsJest} */ module.exports = { diff --git a/packages/cypress-plugin/mocha.d.ts b/packages/cypress-plugin/mocha.d.ts index 428fca1..937533b 100644 --- a/packages/cypress-plugin/mocha.d.ts +++ b/packages/cypress-plugin/mocha.d.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC type GlobalError = Error; diff --git a/packages/cypress-plugin/rollup.config.mjs b/packages/cypress-plugin/rollup.config.mjs index f7b899a..3b6678e 100644 --- a/packages/cypress-plugin/rollup.config.mjs +++ b/packages/cypress-plugin/rollup.config.mjs @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC import pluginCommonJs from "@rollup/plugin-commonjs"; import pluginDts from "rollup-plugin-dts"; diff --git a/packages/cypress-plugin/src/.eslintrc.js b/packages/cypress-plugin/src/.eslintrc.js index 62e8179..4070ee2 100644 --- a/packages/cypress-plugin/src/.eslintrc.js +++ b/packages/cypress-plugin/src/.eslintrc.js @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC module.exports = { extends: ["../../../.eslintrc-ts.js"], diff --git a/packages/cypress-plugin/src/config-env-vars.ts b/packages/cypress-plugin/src/config-env-vars.ts index d9623b1..721480f 100644 --- a/packages/cypress-plugin/src/config-env-vars.ts +++ b/packages/cypress-plugin/src/config-env-vars.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC import { EnvVar } from "@unflakable/plugins-common"; diff --git a/packages/cypress-plugin/src/config-wrapper-sync.ts b/packages/cypress-plugin/src/config-wrapper-sync.ts index 3b4e983..96b672f 100644 --- a/packages/cypress-plugin/src/config-wrapper-sync.ts +++ b/packages/cypress-plugin/src/config-wrapper-sync.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC import { wrapCypressConfig } from "./index"; import _debug from "debug"; diff --git a/packages/cypress-plugin/src/config-wrapper.ts b/packages/cypress-plugin/src/config-wrapper.ts index 507c638..8b01fca 100644 --- a/packages/cypress-plugin/src/config-wrapper.ts +++ b/packages/cypress-plugin/src/config-wrapper.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC import { wrapCypressConfig } from "./index"; import _debug from "debug"; diff --git a/packages/cypress-plugin/src/cypress-env-vars.ts b/packages/cypress-plugin/src/cypress-env-vars.ts index 14c03b0..52f34fd 100644 --- a/packages/cypress-plugin/src/cypress-env-vars.ts +++ b/packages/cypress-plugin/src/cypress-env-vars.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC // NB: This file is separate from config-env-vars.ts so that it can be included by skip-tests.ts // without adding any Node.JS dependencies (since that file needs to run in the browser). diff --git a/packages/cypress-plugin/src/index.ts b/packages/cypress-plugin/src/index.ts index fb65ae9..9a7cbdc 100644 --- a/packages/cypress-plugin/src/index.ts +++ b/packages/cypress-plugin/src/index.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC import _debug from "debug"; import { diff --git a/packages/cypress-plugin/src/load-user-config.ts b/packages/cypress-plugin/src/load-user-config.ts index 48a26d1..d56ecc5 100644 --- a/packages/cypress-plugin/src/load-user-config.ts +++ b/packages/cypress-plugin/src/load-user-config.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC import _debug from "debug"; import { require } from "./utils"; diff --git a/packages/cypress-plugin/src/main.ts b/packages/cypress-plugin/src/main.ts index ccf5ad2..314509e 100755 --- a/packages/cypress-plugin/src/main.ts +++ b/packages/cypress-plugin/src/main.ts @@ -1,6 +1,6 @@ //#!/usr/bin/env node -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC import * as cypress from "cypress"; import _debug from "debug"; diff --git a/packages/cypress-plugin/src/plugin.ts b/packages/cypress-plugin/src/plugin.ts index 158e548..9cf075f 100644 --- a/packages/cypress-plugin/src/plugin.ts +++ b/packages/cypress-plugin/src/plugin.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC import { createTestSuiteRun, diff --git a/packages/cypress-plugin/src/reporter-common.ts b/packages/cypress-plugin/src/reporter-common.ts index 11d6a62..3b8bbbc 100644 --- a/packages/cypress-plugin/src/reporter-common.ts +++ b/packages/cypress-plugin/src/reporter-common.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC // Reporter stats from running a single spec. export type ReporterStats = Mocha.Stats & { diff --git a/packages/cypress-plugin/src/reporter-config.test.ts b/packages/cypress-plugin/src/reporter-config.test.ts index 55d40b6..9fe454d 100644 --- a/packages/cypress-plugin/src/reporter-config.test.ts +++ b/packages/cypress-plugin/src/reporter-config.test.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC import { configureMochaReporter } from "./reporter-config"; import { QuarantineMode } from "@unflakable/plugins-common"; diff --git a/packages/cypress-plugin/src/reporter-config.ts b/packages/cypress-plugin/src/reporter-config.ts index d8e4951..dfbfbe8 100644 --- a/packages/cypress-plugin/src/reporter-config.ts +++ b/packages/cypress-plugin/src/reporter-config.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC import { ReporterConfig } from "./reporter"; import { printWarning, require } from "./utils"; diff --git a/packages/cypress-plugin/src/reporter.ts b/packages/cypress-plugin/src/reporter.ts index 3475e69..c1fdddd 100644 --- a/packages/cypress-plugin/src/reporter.ts +++ b/packages/cypress-plugin/src/reporter.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC import _debug from "debug"; import { diff --git a/packages/cypress-plugin/src/skip-tests.ts b/packages/cypress-plugin/src/skip-tests.ts index 18531b2..deabbc8 100644 --- a/packages/cypress-plugin/src/skip-tests.ts +++ b/packages/cypress-plugin/src/skip-tests.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC // This file runs inside the browser as part of the test code. It gets loaded by a Cypress support // file injected by the plugin when quarantine mode is set to `skip_tests`. diff --git a/packages/cypress-plugin/src/utils.ts b/packages/cypress-plugin/src/utils.ts index 95d5fa7..f2f0bad 100644 --- a/packages/cypress-plugin/src/utils.ts +++ b/packages/cypress-plugin/src/utils.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC import chalk from "chalk"; import { readFileSync } from "fs"; diff --git a/packages/cypress-plugin/src/vendored/cli-table3.d.ts b/packages/cypress-plugin/src/vendored/cli-table3.d.ts index 14c7749..f7b1b70 100644 --- a/packages/cypress-plugin/src/vendored/cli-table3.d.ts +++ b/packages/cypress-plugin/src/vendored/cli-table3.d.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC // Cypress imports this even though it's not part of the cli-table3 exports, so we add some types // here. diff --git a/packages/cypress-plugin/test/.eslintrc.cjs b/packages/cypress-plugin/test/.eslintrc.cjs index 7b41633..ea0aeee 100644 --- a/packages/cypress-plugin/test/.eslintrc.cjs +++ b/packages/cypress-plugin/test/.eslintrc.cjs @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC module.exports = { env: { diff --git a/packages/cypress-plugin/test/integration-input-esm/.eslintrc.cjs b/packages/cypress-plugin/test/integration-input-esm/.eslintrc.cjs index b6febf5..0015a28 100644 --- a/packages/cypress-plugin/test/integration-input-esm/.eslintrc.cjs +++ b/packages/cypress-plugin/test/integration-input-esm/.eslintrc.cjs @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC module.exports = { extends: ["../../../../.eslintrc-ts.js"], diff --git a/packages/cypress-plugin/test/integration-input-esm/config-js/devtools.js b/packages/cypress-plugin/test/integration-input-esm/config-js/devtools.js index afbffdc..b21da85 100644 --- a/packages/cypress-plugin/test/integration-input-esm/config-js/devtools.js +++ b/packages/cypress-plugin/test/integration-input-esm/config-js/devtools.js @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC export const openDevToolsOnLaunch = /** diff --git a/packages/cypress-plugin/test/integration-input-esm/config-js/headless.js b/packages/cypress-plugin/test/integration-input-esm/config-js/headless.js index 093bf8c..82602cc 100644 --- a/packages/cypress-plugin/test/integration-input-esm/config-js/headless.js +++ b/packages/cypress-plugin/test/integration-input-esm/config-js/headless.js @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC /** * Workaround for https://github.com/cypress-io/cypress/issues/27804. diff --git a/packages/cypress-plugin/test/integration-input-esm/config-js/tasks.js b/packages/cypress-plugin/test/integration-input-esm/config-js/tasks.js index 4f0f5a3..6be9d7f 100644 --- a/packages/cypress-plugin/test/integration-input-esm/config-js/tasks.js +++ b/packages/cypress-plugin/test/integration-input-esm/config-js/tasks.js @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC /** * @param {Cypress.PluginEvents} on diff --git a/packages/cypress-plugin/test/integration-input-esm/config-js/webpack.js b/packages/cypress-plugin/test/integration-input-esm/config-js/webpack.js index d159eac..60d0dad 100644 --- a/packages/cypress-plugin/test/integration-input-esm/config-js/webpack.js +++ b/packages/cypress-plugin/test/integration-input-esm/config-js/webpack.js @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC import { default as webpack } from "webpack"; diff --git a/packages/cypress-plugin/test/integration-input-esm/config/devtools.ts b/packages/cypress-plugin/test/integration-input-esm/config/devtools.ts index 58d80d5..f117e1a 100644 --- a/packages/cypress-plugin/test/integration-input-esm/config/devtools.ts +++ b/packages/cypress-plugin/test/integration-input-esm/config/devtools.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC export const openDevToolsOnLaunch = (on: Cypress.PluginEvents): void => { // Open DevTools automatically. Only works for headed modes (i.e., not in screenshots or diff --git a/packages/cypress-plugin/test/integration-input-esm/config/headless.ts b/packages/cypress-plugin/test/integration-input-esm/config/headless.ts index 05519ff..81a2718 100644 --- a/packages/cypress-plugin/test/integration-input-esm/config/headless.ts +++ b/packages/cypress-plugin/test/integration-input-esm/config/headless.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC // Workaround for https://github.com/cypress-io/cypress/issues/27804. export const fixHeadlessChrome = (on: Cypress.PluginEvents): void => { diff --git a/packages/cypress-plugin/test/integration-input-esm/config/tasks.ts b/packages/cypress-plugin/test/integration-input-esm/config/tasks.ts index 5d492df..0a5cb4f 100644 --- a/packages/cypress-plugin/test/integration-input-esm/config/tasks.ts +++ b/packages/cypress-plugin/test/integration-input-esm/config/tasks.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC export const registerTasks = (on: Cypress.PluginEvents): void => { // Used for both testing that the support file gets loaded and testing that the project's diff --git a/packages/cypress-plugin/test/integration-input-esm/config/webpack.ts b/packages/cypress-plugin/test/integration-input-esm/config/webpack.ts index ff784d0..ca31892 100644 --- a/packages/cypress-plugin/test/integration-input-esm/config/webpack.ts +++ b/packages/cypress-plugin/test/integration-input-esm/config/webpack.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC import { default as webpack } from "webpack"; diff --git a/packages/cypress-plugin/test/integration-input-esm/cypress-config.cjs b/packages/cypress-plugin/test/integration-input-esm/cypress-config.cjs index ecde34b..d85ac80 100644 --- a/packages/cypress-plugin/test/integration-input-esm/cypress-config.cjs +++ b/packages/cypress-plugin/test/integration-input-esm/cypress-config.cjs @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC const { registerSimpleGitMock } = require("unflakable-test-common/dist/git"); const { diff --git a/packages/cypress-plugin/test/integration-input-esm/cypress-config.js b/packages/cypress-plugin/test/integration-input-esm/cypress-config.js index c203809..4cd3f6e 100644 --- a/packages/cypress-plugin/test/integration-input-esm/cypress-config.js +++ b/packages/cypress-plugin/test/integration-input-esm/cypress-config.js @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC import { openDevToolsOnLaunch } from "./config-js/devtools.js"; import { registerTasks } from "./config-js/tasks.js"; diff --git a/packages/cypress-plugin/test/integration-input-esm/cypress.config.ts b/packages/cypress-plugin/test/integration-input-esm/cypress.config.ts index 8a8ca57..9b90f72 100644 --- a/packages/cypress-plugin/test/integration-input-esm/cypress.config.ts +++ b/packages/cypress-plugin/test/integration-input-esm/cypress.config.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC import { defineConfig } from "cypress"; // Relative import paths require file extensions in ESM. diff --git a/packages/cypress-plugin/test/integration-input-esm/cypress/support-js/commands.js b/packages/cypress-plugin/test/integration-input-esm/cypress/support-js/commands.js index 1a7516c..5eb705e 100644 --- a/packages/cypress-plugin/test/integration-input-esm/cypress/support-js/commands.js +++ b/packages/cypress-plugin/test/integration-input-esm/cypress/support-js/commands.js @@ -1,3 +1,3 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC Cypress.Commands.add("consoleLog", (msg) => cy.task("log", msg)); diff --git a/packages/cypress-plugin/test/integration-input-esm/cypress/support-js/component.cjs b/packages/cypress-plugin/test/integration-input-esm/cypress/support-js/component.cjs index 0b677aa..b257777 100644 --- a/packages/cypress-plugin/test/integration-input-esm/cypress/support-js/component.cjs +++ b/packages/cypress-plugin/test/integration-input-esm/cypress/support-js/component.cjs @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC require("./commands.js"); diff --git a/packages/cypress-plugin/test/integration-input-esm/cypress/support-js/component.js b/packages/cypress-plugin/test/integration-input-esm/cypress/support-js/component.js index 3c38662..e784a05 100644 --- a/packages/cypress-plugin/test/integration-input-esm/cypress/support-js/component.js +++ b/packages/cypress-plugin/test/integration-input-esm/cypress/support-js/component.js @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC // Import commands.js using ES2015 syntax: import "./commands.js"; diff --git a/packages/cypress-plugin/test/integration-input-esm/cypress/support-js/e2e.cjs b/packages/cypress-plugin/test/integration-input-esm/cypress/support-js/e2e.cjs index c254fe4..cd10e9e 100644 --- a/packages/cypress-plugin/test/integration-input-esm/cypress/support-js/e2e.cjs +++ b/packages/cypress-plugin/test/integration-input-esm/cypress/support-js/e2e.cjs @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC // Import commands.js using CJS syntax: require("./commands.js"); diff --git a/packages/cypress-plugin/test/integration-input-esm/cypress/support-js/e2e.js b/packages/cypress-plugin/test/integration-input-esm/cypress/support-js/e2e.js index 0447f29..1e2cc15 100644 --- a/packages/cypress-plugin/test/integration-input-esm/cypress/support-js/e2e.js +++ b/packages/cypress-plugin/test/integration-input-esm/cypress/support-js/e2e.js @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC // Import commands.js using ES2015 syntax: import "./commands.js"; diff --git a/packages/cypress-plugin/test/integration-input-esm/cypress/support/commands.ts b/packages/cypress-plugin/test/integration-input-esm/cypress/support/commands.ts index a6cb68a..876f349 100644 --- a/packages/cypress-plugin/test/integration-input-esm/cypress/support/commands.ts +++ b/packages/cypress-plugin/test/integration-input-esm/cypress/support/commands.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC // Webpack non-deterministically reports a TypeScript error here when run through Cypress. It seems // to happen when `cypress-support-file` includes diff --git a/packages/cypress-plugin/test/integration-input-esm/cypress/support/component-index.html b/packages/cypress-plugin/test/integration-input-esm/cypress/support/component-index.html index e39ba42..92d3f43 100644 --- a/packages/cypress-plugin/test/integration-input-esm/cypress/support/component-index.html +++ b/packages/cypress-plugin/test/integration-input-esm/cypress/support/component-index.html @@ -1,3 +1,5 @@ + + diff --git a/packages/cypress-plugin/test/integration-input-esm/cypress/support/component.ts b/packages/cypress-plugin/test/integration-input-esm/cypress/support/component.ts index 87db971..8225d55 100644 --- a/packages/cypress-plugin/test/integration-input-esm/cypress/support/component.ts +++ b/packages/cypress-plugin/test/integration-input-esm/cypress/support/component.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC import "./commands.ts"; diff --git a/packages/cypress-plugin/test/integration-input-esm/cypress/support/e2e.ts b/packages/cypress-plugin/test/integration-input-esm/cypress/support/e2e.ts index 8fd06b4..9127c7b 100644 --- a/packages/cypress-plugin/test/integration-input-esm/cypress/support/e2e.ts +++ b/packages/cypress-plugin/test/integration-input-esm/cypress/support/e2e.ts @@ -1,3 +1,3 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC import "./commands.ts"; diff --git a/packages/cypress-plugin/test/integration-input-manual/config/devtools.js b/packages/cypress-plugin/test/integration-input-manual/config/devtools.js index c117765..e96f27e 100644 --- a/packages/cypress-plugin/test/integration-input-manual/config/devtools.js +++ b/packages/cypress-plugin/test/integration-input-manual/config/devtools.js @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC module.exports = { /** diff --git a/packages/cypress-plugin/test/integration-input-manual/config/headless.js b/packages/cypress-plugin/test/integration-input-manual/config/headless.js index a116630..f6540a0 100644 --- a/packages/cypress-plugin/test/integration-input-manual/config/headless.js +++ b/packages/cypress-plugin/test/integration-input-manual/config/headless.js @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC module.exports = { /** diff --git a/packages/cypress-plugin/test/integration-input-manual/config/tasks.js b/packages/cypress-plugin/test/integration-input-manual/config/tasks.js index 95cba3e..45c7d8a 100644 --- a/packages/cypress-plugin/test/integration-input-manual/config/tasks.js +++ b/packages/cypress-plugin/test/integration-input-manual/config/tasks.js @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC module.exports = { /** diff --git a/packages/cypress-plugin/test/integration-input-manual/config/webpack.js b/packages/cypress-plugin/test/integration-input-manual/config/webpack.js index c6501b8..6e44c41 100644 --- a/packages/cypress-plugin/test/integration-input-manual/config/webpack.js +++ b/packages/cypress-plugin/test/integration-input-manual/config/webpack.js @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC const webpack = require("webpack"); diff --git a/packages/cypress-plugin/test/integration-input-manual/cypress-config.mjs b/packages/cypress-plugin/test/integration-input-manual/cypress-config.mjs index 1f976f6..e372c25 100644 --- a/packages/cypress-plugin/test/integration-input-manual/cypress-config.mjs +++ b/packages/cypress-plugin/test/integration-input-manual/cypress-config.mjs @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC import devtools from "./config/devtools.js"; import tasks from "./config/tasks.js"; diff --git a/packages/cypress-plugin/test/integration-input-manual/cypress.config.js b/packages/cypress-plugin/test/integration-input-manual/cypress.config.js index 4f34373..b45cbd1 100644 --- a/packages/cypress-plugin/test/integration-input-manual/cypress.config.js +++ b/packages/cypress-plugin/test/integration-input-manual/cypress.config.js @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC const { openDevToolsOnLaunch } = require("./config/devtools"); const { registerTasks } = require("./config/tasks"); diff --git a/packages/cypress-plugin/test/integration-input-manual/cypress/e2e/fail.cy.js b/packages/cypress-plugin/test/integration-input-manual/cypress/e2e/fail.cy.js index c1bec18..cc6a872 100644 --- a/packages/cypress-plugin/test/integration-input-manual/cypress/e2e/fail.cy.js +++ b/packages/cypress-plugin/test/integration-input-manual/cypress/e2e/fail.cy.js @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC const testFn = Cypress.env("SKIP_FAILURES") !== undefined ? it.skip : it; diff --git a/packages/cypress-plugin/test/integration-input-manual/cypress/e2e/flake.cy.js b/packages/cypress-plugin/test/integration-input-manual/cypress/e2e/flake.cy.js index 1bab7bc..fef71a2 100644 --- a/packages/cypress-plugin/test/integration-input-manual/cypress/e2e/flake.cy.js +++ b/packages/cypress-plugin/test/integration-input-manual/cypress/e2e/flake.cy.js @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC /* let calls = 0; diff --git a/packages/cypress-plugin/test/integration-input-manual/cypress/e2e/hook-fail.cy.js b/packages/cypress-plugin/test/integration-input-manual/cypress/e2e/hook-fail.cy.js index 297f0bb..3478d79 100644 --- a/packages/cypress-plugin/test/integration-input-manual/cypress/e2e/hook-fail.cy.js +++ b/packages/cypress-plugin/test/integration-input-manual/cypress/e2e/hook-fail.cy.js @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC describe("describe block", () => { if (Cypress.env("SKIP_BEFORE_HOOK") === undefined) { diff --git a/packages/cypress-plugin/test/integration-input-manual/cypress/e2e/invalid.cy.js b/packages/cypress-plugin/test/integration-input-manual/cypress/e2e/invalid.cy.js index aa580f4..a5af12b 100644 --- a/packages/cypress-plugin/test/integration-input-manual/cypress/e2e/invalid.cy.js +++ b/packages/cypress-plugin/test/integration-input-manual/cypress/e2e/invalid.cy.js @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC if (Cypress.env("SKIP_FAILURES") === undefined) { throw new Error("invalid test file"); diff --git a/packages/cypress-plugin/test/integration-input-manual/cypress/e2e/mixed/mixed.cy.js b/packages/cypress-plugin/test/integration-input-manual/cypress/e2e/mixed/mixed.cy.js index e731026..3094edd 100644 --- a/packages/cypress-plugin/test/integration-input-manual/cypress/e2e/mixed/mixed.cy.js +++ b/packages/cypress-plugin/test/integration-input-manual/cypress/e2e/mixed/mixed.cy.js @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC describe("spec with mixed test results", () => { const quarantinedTestFn = diff --git a/packages/cypress-plugin/test/integration-input-manual/cypress/e2e/pass.cy.js b/packages/cypress-plugin/test/integration-input-manual/cypress/e2e/pass.cy.js index 05d345a..5b22ca4 100644 --- a/packages/cypress-plugin/test/integration-input-manual/cypress/e2e/pass.cy.js +++ b/packages/cypress-plugin/test/integration-input-manual/cypress/e2e/pass.cy.js @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC it("should pass", () => { // Make sure the project's support file works even when skip_tests generates a temporary one on diff --git a/packages/cypress-plugin/test/integration-input-manual/cypress/e2e/pending.cy.js b/packages/cypress-plugin/test/integration-input-manual/cypress/e2e/pending.cy.js index 2704e82..8f7488a 100644 --- a/packages/cypress-plugin/test/integration-input-manual/cypress/e2e/pending.cy.js +++ b/packages/cypress-plugin/test/integration-input-manual/cypress/e2e/pending.cy.js @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC it("stub should be pending"); diff --git a/packages/cypress-plugin/test/integration-input-manual/cypress/e2e/quarantined.cy.js b/packages/cypress-plugin/test/integration-input-manual/cypress/e2e/quarantined.cy.js index f43ebc0..6682e4c 100644 --- a/packages/cypress-plugin/test/integration-input-manual/cypress/e2e/quarantined.cy.js +++ b/packages/cypress-plugin/test/integration-input-manual/cypress/e2e/quarantined.cy.js @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC describe("describe block", () => { (Cypress.env("SKIP_QUARANTINED") !== undefined ? it.skip : it)( diff --git a/packages/cypress-plugin/test/integration-input-manual/cypress/support/commands.js b/packages/cypress-plugin/test/integration-input-manual/cypress/support/commands.js index 1a7516c..5eb705e 100644 --- a/packages/cypress-plugin/test/integration-input-manual/cypress/support/commands.js +++ b/packages/cypress-plugin/test/integration-input-manual/cypress/support/commands.js @@ -1,3 +1,3 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC Cypress.Commands.add("consoleLog", (msg) => cy.task("log", msg)); diff --git a/packages/cypress-plugin/test/integration-input-manual/cypress/support/component-index.html b/packages/cypress-plugin/test/integration-input-manual/cypress/support/component-index.html index e39ba42..92d3f43 100644 --- a/packages/cypress-plugin/test/integration-input-manual/cypress/support/component-index.html +++ b/packages/cypress-plugin/test/integration-input-manual/cypress/support/component-index.html @@ -1,3 +1,5 @@ + + diff --git a/packages/cypress-plugin/test/integration-input-manual/cypress/support/component.js b/packages/cypress-plugin/test/integration-input-manual/cypress/support/component.js index 5d0823f..d60f49e 100644 --- a/packages/cypress-plugin/test/integration-input-manual/cypress/support/component.js +++ b/packages/cypress-plugin/test/integration-input-manual/cypress/support/component.js @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC require("./commands"); diff --git a/packages/cypress-plugin/test/integration-input-manual/cypress/support/e2e-webpack5.js b/packages/cypress-plugin/test/integration-input-manual/cypress/support/e2e-webpack5.js index 088357d..8defad6 100644 --- a/packages/cypress-plugin/test/integration-input-manual/cypress/support/e2e-webpack5.js +++ b/packages/cypress-plugin/test/integration-input-manual/cypress/support/e2e-webpack5.js @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC // Import commands.js using CJS syntax: require("./commands.js"); diff --git a/packages/cypress-plugin/test/integration-input-manual/cypress/support/e2e.js b/packages/cypress-plugin/test/integration-input-manual/cypress/support/e2e.js index 9033340..5ae257d 100644 --- a/packages/cypress-plugin/test/integration-input-manual/cypress/support/e2e.js +++ b/packages/cypress-plugin/test/integration-input-manual/cypress/support/e2e.js @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC // Import commands.js using CJS syntax: require("./commands.js"); diff --git a/packages/cypress-plugin/test/integration-input/.eslintrc.js b/packages/cypress-plugin/test/integration-input/.eslintrc.js index b6febf5..0015a28 100644 --- a/packages/cypress-plugin/test/integration-input/.eslintrc.js +++ b/packages/cypress-plugin/test/integration-input/.eslintrc.js @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC module.exports = { extends: ["../../../../.eslintrc-ts.js"], diff --git a/packages/cypress-plugin/test/integration-input/config-js/devtools.js b/packages/cypress-plugin/test/integration-input/config-js/devtools.js index c117765..e96f27e 100644 --- a/packages/cypress-plugin/test/integration-input/config-js/devtools.js +++ b/packages/cypress-plugin/test/integration-input/config-js/devtools.js @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC module.exports = { /** diff --git a/packages/cypress-plugin/test/integration-input/config-js/headless.js b/packages/cypress-plugin/test/integration-input/config-js/headless.js index a116630..f6540a0 100644 --- a/packages/cypress-plugin/test/integration-input/config-js/headless.js +++ b/packages/cypress-plugin/test/integration-input/config-js/headless.js @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC module.exports = { /** diff --git a/packages/cypress-plugin/test/integration-input/config-js/tasks.js b/packages/cypress-plugin/test/integration-input/config-js/tasks.js index 95cba3e..45c7d8a 100644 --- a/packages/cypress-plugin/test/integration-input/config-js/tasks.js +++ b/packages/cypress-plugin/test/integration-input/config-js/tasks.js @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC module.exports = { /** diff --git a/packages/cypress-plugin/test/integration-input/config-js/webpack.js b/packages/cypress-plugin/test/integration-input/config-js/webpack.js index c6501b8..6e44c41 100644 --- a/packages/cypress-plugin/test/integration-input/config-js/webpack.js +++ b/packages/cypress-plugin/test/integration-input/config-js/webpack.js @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC const webpack = require("webpack"); diff --git a/packages/cypress-plugin/test/integration-input/config/devtools.ts b/packages/cypress-plugin/test/integration-input/config/devtools.ts index 58d80d5..f117e1a 100644 --- a/packages/cypress-plugin/test/integration-input/config/devtools.ts +++ b/packages/cypress-plugin/test/integration-input/config/devtools.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC export const openDevToolsOnLaunch = (on: Cypress.PluginEvents): void => { // Open DevTools automatically. Only works for headed modes (i.e., not in screenshots or diff --git a/packages/cypress-plugin/test/integration-input/config/headless.ts b/packages/cypress-plugin/test/integration-input/config/headless.ts index 05519ff..81a2718 100644 --- a/packages/cypress-plugin/test/integration-input/config/headless.ts +++ b/packages/cypress-plugin/test/integration-input/config/headless.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC // Workaround for https://github.com/cypress-io/cypress/issues/27804. export const fixHeadlessChrome = (on: Cypress.PluginEvents): void => { diff --git a/packages/cypress-plugin/test/integration-input/config/tasks.ts b/packages/cypress-plugin/test/integration-input/config/tasks.ts index 5d492df..0a5cb4f 100644 --- a/packages/cypress-plugin/test/integration-input/config/tasks.ts +++ b/packages/cypress-plugin/test/integration-input/config/tasks.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC export const registerTasks = (on: Cypress.PluginEvents): void => { // Used for both testing that the support file gets loaded and testing that the project's diff --git a/packages/cypress-plugin/test/integration-input/config/webpack.ts b/packages/cypress-plugin/test/integration-input/config/webpack.ts index d159eac..60d0dad 100644 --- a/packages/cypress-plugin/test/integration-input/config/webpack.ts +++ b/packages/cypress-plugin/test/integration-input/config/webpack.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC import { default as webpack } from "webpack"; diff --git a/packages/cypress-plugin/test/integration-input/cypress-config.js b/packages/cypress-plugin/test/integration-input/cypress-config.js index 2ffbfe6..2131165 100644 --- a/packages/cypress-plugin/test/integration-input/cypress-config.js +++ b/packages/cypress-plugin/test/integration-input/cypress-config.js @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC const { openDevToolsOnLaunch } = require("config-js/devtools"); const webpackConfig = require("config-js/webpack"); diff --git a/packages/cypress-plugin/test/integration-input/cypress-config.mjs b/packages/cypress-plugin/test/integration-input/cypress-config.mjs index c5d6b38..caa4a1c 100644 --- a/packages/cypress-plugin/test/integration-input/cypress-config.mjs +++ b/packages/cypress-plugin/test/integration-input/cypress-config.mjs @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC import devtools from "./config-js/devtools.js"; import tasks from "./config-js/tasks.js"; diff --git a/packages/cypress-plugin/test/integration-input/cypress.config.ts b/packages/cypress-plugin/test/integration-input/cypress.config.ts index 5375c6a..2cc784d 100644 --- a/packages/cypress-plugin/test/integration-input/cypress.config.ts +++ b/packages/cypress-plugin/test/integration-input/cypress.config.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC import { registerSimpleGitMock } from "unflakable-test-common/dist/git"; import { registerCosmiconfigMock } from "unflakable-test-common/dist/config"; diff --git a/packages/cypress-plugin/test/integration-input/cypress/component/fail.cy.ts b/packages/cypress-plugin/test/integration-input/cypress/component/fail.cy.ts index 94b40a9..167c20f 100644 --- a/packages/cypress-plugin/test/integration-input/cypress/component/fail.cy.ts +++ b/packages/cypress-plugin/test/integration-input/cypress/component/fail.cy.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC const testFn = Cypress.env("SKIP_FAILURES") !== undefined ? it.skip : it; diff --git a/packages/cypress-plugin/test/integration-input/cypress/component/flake.cy.ts b/packages/cypress-plugin/test/integration-input/cypress/component/flake.cy.ts index 6cda5a0..6011bf6 100644 --- a/packages/cypress-plugin/test/integration-input/cypress/component/flake.cy.ts +++ b/packages/cypress-plugin/test/integration-input/cypress/component/flake.cy.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC /* let calls = 0; diff --git a/packages/cypress-plugin/test/integration-input/cypress/component/hook-fail.cy.ts b/packages/cypress-plugin/test/integration-input/cypress/component/hook-fail.cy.ts index 08c00f3..ba0a1c1 100644 --- a/packages/cypress-plugin/test/integration-input/cypress/component/hook-fail.cy.ts +++ b/packages/cypress-plugin/test/integration-input/cypress/component/hook-fail.cy.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC describe("describe block", () => { if (Cypress.env("SKIP_BEFORE_HOOK") === undefined) { diff --git a/packages/cypress-plugin/test/integration-input/cypress/component/invalid.cy.ts b/packages/cypress-plugin/test/integration-input/cypress/component/invalid.cy.ts index aa580f4..a5af12b 100644 --- a/packages/cypress-plugin/test/integration-input/cypress/component/invalid.cy.ts +++ b/packages/cypress-plugin/test/integration-input/cypress/component/invalid.cy.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC if (Cypress.env("SKIP_FAILURES") === undefined) { throw new Error("invalid test file"); diff --git a/packages/cypress-plugin/test/integration-input/cypress/component/mixed/mixed.cy.ts b/packages/cypress-plugin/test/integration-input/cypress/component/mixed/mixed.cy.ts index b0ee438..5fd29be 100644 --- a/packages/cypress-plugin/test/integration-input/cypress/component/mixed/mixed.cy.ts +++ b/packages/cypress-plugin/test/integration-input/cypress/component/mixed/mixed.cy.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC describe("spec with mixed test results", () => { const quarantinedTestFn = diff --git a/packages/cypress-plugin/test/integration-input/cypress/component/pass.cy.ts b/packages/cypress-plugin/test/integration-input/cypress/component/pass.cy.ts index 05d345a..5b22ca4 100644 --- a/packages/cypress-plugin/test/integration-input/cypress/component/pass.cy.ts +++ b/packages/cypress-plugin/test/integration-input/cypress/component/pass.cy.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC it("should pass", () => { // Make sure the project's support file works even when skip_tests generates a temporary one on diff --git a/packages/cypress-plugin/test/integration-input/cypress/component/pending.cy.ts b/packages/cypress-plugin/test/integration-input/cypress/component/pending.cy.ts index 2704e82..8f7488a 100644 --- a/packages/cypress-plugin/test/integration-input/cypress/component/pending.cy.ts +++ b/packages/cypress-plugin/test/integration-input/cypress/component/pending.cy.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC it("stub should be pending"); diff --git a/packages/cypress-plugin/test/integration-input/cypress/component/quarantined.cy.ts b/packages/cypress-plugin/test/integration-input/cypress/component/quarantined.cy.ts index f43ebc0..6682e4c 100644 --- a/packages/cypress-plugin/test/integration-input/cypress/component/quarantined.cy.ts +++ b/packages/cypress-plugin/test/integration-input/cypress/component/quarantined.cy.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC describe("describe block", () => { (Cypress.env("SKIP_QUARANTINED") !== undefined ? it.skip : it)( diff --git a/packages/cypress-plugin/test/integration-input/cypress/support-js/commands.js b/packages/cypress-plugin/test/integration-input/cypress/support-js/commands.js index 1a7516c..5eb705e 100644 --- a/packages/cypress-plugin/test/integration-input/cypress/support-js/commands.js +++ b/packages/cypress-plugin/test/integration-input/cypress/support-js/commands.js @@ -1,3 +1,3 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC Cypress.Commands.add("consoleLog", (msg) => cy.task("log", msg)); diff --git a/packages/cypress-plugin/test/integration-input/cypress/support-js/component.js b/packages/cypress-plugin/test/integration-input/cypress/support-js/component.js index fea2a9b..04783d5 100644 --- a/packages/cypress-plugin/test/integration-input/cypress/support-js/component.js +++ b/packages/cypress-plugin/test/integration-input/cypress/support-js/component.js @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC require("./commands"); diff --git a/packages/cypress-plugin/test/integration-input/cypress/support-js/component.mjs b/packages/cypress-plugin/test/integration-input/cypress/support-js/component.mjs index 3c38662..e784a05 100644 --- a/packages/cypress-plugin/test/integration-input/cypress/support-js/component.mjs +++ b/packages/cypress-plugin/test/integration-input/cypress/support-js/component.mjs @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC // Import commands.js using ES2015 syntax: import "./commands.js"; diff --git a/packages/cypress-plugin/test/integration-input/cypress/support-js/e2e.js b/packages/cypress-plugin/test/integration-input/cypress/support-js/e2e.js index c254fe4..cd10e9e 100644 --- a/packages/cypress-plugin/test/integration-input/cypress/support-js/e2e.js +++ b/packages/cypress-plugin/test/integration-input/cypress/support-js/e2e.js @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC // Import commands.js using CJS syntax: require("./commands.js"); diff --git a/packages/cypress-plugin/test/integration-input/cypress/support-js/e2e.mjs b/packages/cypress-plugin/test/integration-input/cypress/support-js/e2e.mjs index 0447f29..1e2cc15 100644 --- a/packages/cypress-plugin/test/integration-input/cypress/support-js/e2e.mjs +++ b/packages/cypress-plugin/test/integration-input/cypress/support-js/e2e.mjs @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC // Import commands.js using ES2015 syntax: import "./commands.js"; diff --git a/packages/cypress-plugin/test/integration-input/cypress/support/commands.ts b/packages/cypress-plugin/test/integration-input/cypress/support/commands.ts index 966021a..ad471e1 100644 --- a/packages/cypress-plugin/test/integration-input/cypress/support/commands.ts +++ b/packages/cypress-plugin/test/integration-input/cypress/support/commands.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC /// diff --git a/packages/cypress-plugin/test/integration-input/cypress/support/component-index.html b/packages/cypress-plugin/test/integration-input/cypress/support/component-index.html index e39ba42..b2019a7 100644 --- a/packages/cypress-plugin/test/integration-input/cypress/support/component-index.html +++ b/packages/cypress-plugin/test/integration-input/cypress/support/component-index.html @@ -1,3 +1,5 @@ + + diff --git a/packages/cypress-plugin/test/integration-input/cypress/support/component.ts b/packages/cypress-plugin/test/integration-input/cypress/support/component.ts index 5fbee34..08f39a2 100644 --- a/packages/cypress-plugin/test/integration-input/cypress/support/component.ts +++ b/packages/cypress-plugin/test/integration-input/cypress/support/component.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC // Import commands.js using ES2015 syntax: import "./commands"; diff --git a/packages/cypress-plugin/test/integration-input/cypress/support/e2e.ts b/packages/cypress-plugin/test/integration-input/cypress/support/e2e.ts index 5aa0b7c..4284eff 100644 --- a/packages/cypress-plugin/test/integration-input/cypress/support/e2e.ts +++ b/packages/cypress-plugin/test/integration-input/cypress/support/e2e.ts @@ -1,3 +1,3 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC import "./commands"; diff --git a/packages/cypress-plugin/test/integration/.eslintrc.js b/packages/cypress-plugin/test/integration/.eslintrc.js index b6febf5..0015a28 100644 --- a/packages/cypress-plugin/test/integration/.eslintrc.js +++ b/packages/cypress-plugin/test/integration/.eslintrc.js @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC module.exports = { extends: ["../../../../.eslintrc-ts.js"], diff --git a/packages/cypress-plugin/test/integration/jest.config.js b/packages/cypress-plugin/test/integration/jest.config.js index 5184100..db3dac5 100644 --- a/packages/cypress-plugin/test/integration/jest.config.js +++ b/packages/cypress-plugin/test/integration/jest.config.js @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC // NB: We *MUST* run this test suite with --runInBand because running multiple instances of Cypress // concurrently on the same machine is not supported and runs into a bunch of race conditions that diff --git a/packages/cypress-plugin/test/integration/src/basic-matrix.test.ts b/packages/cypress-plugin/test/integration/src/basic-matrix.test.ts index 60afb14..29cd967 100644 --- a/packages/cypress-plugin/test/integration/src/basic-matrix.test.ts +++ b/packages/cypress-plugin/test/integration/src/basic-matrix.test.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC import { defaultSummaryTotals, diff --git a/packages/cypress-plugin/test/integration/src/basic.test.ts b/packages/cypress-plugin/test/integration/src/basic.test.ts index 9435b3c..d785029 100644 --- a/packages/cypress-plugin/test/integration/src/basic.test.ts +++ b/packages/cypress-plugin/test/integration/src/basic.test.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC import { integrationTestSuite, diff --git a/packages/cypress-plugin/test/integration/src/config.test.ts b/packages/cypress-plugin/test/integration/src/config.test.ts index fb40b62..ffc364b 100644 --- a/packages/cypress-plugin/test/integration/src/config.test.ts +++ b/packages/cypress-plugin/test/integration/src/config.test.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC import { integrationTestSuite, integrationTest } from "./test-wrappers"; diff --git a/packages/cypress-plugin/test/integration/src/disable-plugin.test.ts b/packages/cypress-plugin/test/integration/src/disable-plugin.test.ts index 59eb9e1..2539b47 100644 --- a/packages/cypress-plugin/test/integration/src/disable-plugin.test.ts +++ b/packages/cypress-plugin/test/integration/src/disable-plugin.test.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC import { defaultSummaryTotals, diff --git a/packages/cypress-plugin/test/integration/src/disable-upload.test.ts b/packages/cypress-plugin/test/integration/src/disable-upload.test.ts index 14b1b3a..fe6f5b4 100644 --- a/packages/cypress-plugin/test/integration/src/disable-upload.test.ts +++ b/packages/cypress-plugin/test/integration/src/disable-upload.test.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC import { integrationTest, integrationTestSuite } from "./test-wrappers"; diff --git a/packages/cypress-plugin/test/integration/src/git.test.ts b/packages/cypress-plugin/test/integration/src/git.test.ts index 5546179..1594a79 100644 --- a/packages/cypress-plugin/test/integration/src/git.test.ts +++ b/packages/cypress-plugin/test/integration/src/git.test.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC import { integrationTest, integrationTestSuite } from "./test-wrappers"; import path from "path"; diff --git a/packages/cypress-plugin/test/integration/src/hook-failures.test.ts b/packages/cypress-plugin/test/integration/src/hook-failures.test.ts index 1294218..68fedf1 100644 --- a/packages/cypress-plugin/test/integration/src/hook-failures.test.ts +++ b/packages/cypress-plugin/test/integration/src/hook-failures.test.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC import { integrationTest, integrationTestSuite } from "./test-wrappers"; import semverLt from "semver/functions/lt"; diff --git a/packages/cypress-plugin/test/integration/src/long-names.test.ts b/packages/cypress-plugin/test/integration/src/long-names.test.ts index 9d18150..9b242f7 100644 --- a/packages/cypress-plugin/test/integration/src/long-names.test.ts +++ b/packages/cypress-plugin/test/integration/src/long-names.test.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC import { defaultSummaryTotals, diff --git a/packages/cypress-plugin/test/integration/src/matchers.ts b/packages/cypress-plugin/test/integration/src/matchers.ts index 065a17f..b092d2c 100644 --- a/packages/cypress-plugin/test/integration/src/matchers.ts +++ b/packages/cypress-plugin/test/integration/src/matchers.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC import { expect } from "@jest/globals"; import type { MatcherFunction } from "expect"; diff --git a/packages/cypress-plugin/test/integration/src/no-quarantine.test.ts b/packages/cypress-plugin/test/integration/src/no-quarantine.test.ts index 3710a9c..c31e12b 100644 --- a/packages/cypress-plugin/test/integration/src/no-quarantine.test.ts +++ b/packages/cypress-plugin/test/integration/src/no-quarantine.test.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC import { defaultSummaryTotals, diff --git a/packages/cypress-plugin/test/integration/src/parse-output.ts b/packages/cypress-plugin/test/integration/src/parse-output.ts index 17ad736..b82d778 100644 --- a/packages/cypress-plugin/test/integration/src/parse-output.ts +++ b/packages/cypress-plugin/test/integration/src/parse-output.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC /* eslint-disable no-control-regex */ diff --git a/packages/cypress-plugin/test/integration/src/plugin-failures.test.ts b/packages/cypress-plugin/test/integration/src/plugin-failures.test.ts index b6434d7..09dd6ba 100644 --- a/packages/cypress-plugin/test/integration/src/plugin-failures.test.ts +++ b/packages/cypress-plugin/test/integration/src/plugin-failures.test.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC import { defaultSummaryTotals, diff --git a/packages/cypress-plugin/test/integration/src/retries.test.ts b/packages/cypress-plugin/test/integration/src/retries.test.ts index bc99335..ab1acf6 100644 --- a/packages/cypress-plugin/test/integration/src/retries.test.ts +++ b/packages/cypress-plugin/test/integration/src/retries.test.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC import { defaultSummaryTotals, diff --git a/packages/cypress-plugin/test/integration/src/run-test-case.ts b/packages/cypress-plugin/test/integration/src/run-test-case.ts index 414f076..a463246 100644 --- a/packages/cypress-plugin/test/integration/src/run-test-case.ts +++ b/packages/cypress-plugin/test/integration/src/run-test-case.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC import { CreateTestSuiteRunInlineRequest, diff --git a/packages/cypress-plugin/test/integration/src/test-wrappers.ts b/packages/cypress-plugin/test/integration/src/test-wrappers.ts index 090ecaa..278ccb8 100644 --- a/packages/cypress-plugin/test/integration/src/test-wrappers.ts +++ b/packages/cypress-plugin/test/integration/src/test-wrappers.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC import { runTestCase, TestCaseParams } from "./run-test-case"; import path from "path"; diff --git a/packages/cypress-plugin/test/integration/src/unicode.test.ts b/packages/cypress-plugin/test/integration/src/unicode.test.ts index a400392..d2bdd3e 100644 --- a/packages/cypress-plugin/test/integration/src/unicode.test.ts +++ b/packages/cypress-plugin/test/integration/src/unicode.test.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC import { defaultSummaryTotals, diff --git a/packages/cypress-plugin/test/integration/src/verify-output.ts b/packages/cypress-plugin/test/integration/src/verify-output.ts index f8d5f0d..33ebb92 100644 --- a/packages/cypress-plugin/test/integration/src/verify-output.ts +++ b/packages/cypress-plugin/test/integration/src/verify-output.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC // Make sure expected output is present and chalk-formatted correctly. import { diff --git a/packages/cypress-plugin/test/integration/unflakable.js b/packages/cypress-plugin/test/integration/unflakable.js index 7e9c473..f29e4fc 100644 --- a/packages/cypress-plugin/test/integration/unflakable.js +++ b/packages/cypress-plugin/test/integration/unflakable.js @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC module.exports = { __unstableIsFailureTestIndependent: [ diff --git a/packages/jest-plugin/.eslintrc.js b/packages/jest-plugin/.eslintrc.js index dcbec19..f943e67 100644 --- a/packages/jest-plugin/.eslintrc.js +++ b/packages/jest-plugin/.eslintrc.js @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC module.exports = { extends: ["../../.eslintrc-ts.js"], diff --git a/packages/jest-plugin/LICENSE b/packages/jest-plugin/LICENSE index 53bf01b..fc24ed8 100644 --- a/packages/jest-plugin/LICENSE +++ b/packages/jest-plugin/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2022-2023 Developer Innovations, LLC +Copyright (c) 2022-2024 Developer Innovations, LLC Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/packages/jest-plugin/jest-circus.d.ts b/packages/jest-plugin/jest-circus.d.ts index c17e897..833746d 100644 --- a/packages/jest-plugin/jest-circus.d.ts +++ b/packages/jest-plugin/jest-circus.d.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC // jest-circus doesn't export the types for runner. See: // https://github.com/jestjs/jest/blob/6d2632adae0f0fa1fe116d3b475fd9783d0de1b5/packages/jest-circus/runner.js#L10-L9 diff --git a/packages/jest-plugin/jest.config.js b/packages/jest-plugin/jest.config.js index 6318fbd..6cd3940 100644 --- a/packages/jest-plugin/jest.config.js +++ b/packages/jest-plugin/jest.config.js @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC /** @type {import('ts-jest').JestConfigWithTsJest} */ module.exports = { diff --git a/packages/jest-plugin/rollup.config.mjs b/packages/jest-plugin/rollup.config.mjs index 797cee5..c57bf72 100644 --- a/packages/jest-plugin/rollup.config.mjs +++ b/packages/jest-plugin/rollup.config.mjs @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC import path from "path"; import pluginCommonJs from "@rollup/plugin-commonjs"; diff --git a/packages/jest-plugin/src/config.test.ts b/packages/jest-plugin/src/config.test.ts index ece7c1d..b105138 100644 --- a/packages/jest-plugin/src/config.test.ts +++ b/packages/jest-plugin/src/config.test.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC import { loadConfig } from "./config"; import { cosmiconfigSync, Options } from "cosmiconfig"; diff --git a/packages/jest-plugin/src/config.ts b/packages/jest-plugin/src/config.ts index b3a87d0..97159dc 100644 --- a/packages/jest-plugin/src/config.ts +++ b/packages/jest-plugin/src/config.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC import { IsFailureTestIndependentFn, diff --git a/packages/jest-plugin/src/index.ts b/packages/jest-plugin/src/index.ts index 0c72377..be0a8c0 100644 --- a/packages/jest-plugin/src/index.ts +++ b/packages/jest-plugin/src/index.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2022-2023 Developer Innovations, LLC +// Copyright (c) 2022-2024 Developer Innovations, LLC import reporter from "./reporter"; import runner from "./runner"; diff --git a/packages/jest-plugin/src/reporter.ts b/packages/jest-plugin/src/reporter.ts index 89e0ae9..6c1fe51 100644 --- a/packages/jest-plugin/src/reporter.ts +++ b/packages/jest-plugin/src/reporter.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2022-2023 Developer Innovations, LLC +// Copyright (c) 2022-2024 Developer Innovations, LLC import * as path from "path"; import type { diff --git a/packages/jest-plugin/src/runner.ts b/packages/jest-plugin/src/runner.ts index 7097f01..17dbcda 100644 --- a/packages/jest-plugin/src/runner.ts +++ b/packages/jest-plugin/src/runner.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2022-2023 Developer Innovations, LLC +// Copyright (c) 2022-2024 Developer Innovations, LLC import * as path from "path"; import type { SerializableError, TestResult } from "@jest/test-result"; diff --git a/packages/jest-plugin/src/test-runner.ts b/packages/jest-plugin/src/test-runner.ts index 1884aaf..4f50009 100644 --- a/packages/jest-plugin/src/test-runner.ts +++ b/packages/jest-plugin/src/test-runner.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC import { TestEvents, TestFileEvent, TestResult } from "@jest/test-result"; import { JestEnvironment } from "@jest/environment"; diff --git a/packages/jest-plugin/src/types.ts b/packages/jest-plugin/src/types.ts index 849a5da..3f72bbf 100644 --- a/packages/jest-plugin/src/types.ts +++ b/packages/jest-plugin/src/types.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2022-2023 Developer Innovations, LLC +// Copyright (c) 2022-2024 Developer Innovations, LLC import type { AggregatedResult, diff --git a/packages/jest-plugin/src/utils.ts b/packages/jest-plugin/src/utils.ts index 9a4e0b9..2138d0b 100644 --- a/packages/jest-plugin/src/utils.ts +++ b/packages/jest-plugin/src/utils.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2022-2023 Developer Innovations, LLC +// Copyright (c) 2022-2024 Developer Innovations, LLC import type { AssertionResult, Status } from "@jest/test-result"; import jestPackage from "jest/package.json"; diff --git a/packages/jest-plugin/src/vendored/SummaryReporter.ts b/packages/jest-plugin/src/vendored/SummaryReporter.ts index 91c9b13..5f54f95 100644 --- a/packages/jest-plugin/src/vendored/SummaryReporter.ts +++ b/packages/jest-plugin/src/vendored/SummaryReporter.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2022-2023 Developer Innovations, LLC +// Copyright (c) 2022-2024 Developer Innovations, LLC /* This file includes portions of a Jest source code file originally downloaded from: diff --git a/packages/jest-plugin/src/vendored/getResultHeader.ts b/packages/jest-plugin/src/vendored/getResultHeader.ts index 37f80cf..86cc030 100644 --- a/packages/jest-plugin/src/vendored/getResultHeader.ts +++ b/packages/jest-plugin/src/vendored/getResultHeader.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2022-2023 Developer Innovations, LLC +// Copyright (c) 2022-2024 Developer Innovations, LLC import { utils } from "@jest/reporters"; import { TestResult } from "@jest/test-result"; diff --git a/packages/jest-plugin/src/vendored/getSummary.ts b/packages/jest-plugin/src/vendored/getSummary.ts index 8422414..5b88c23 100644 --- a/packages/jest-plugin/src/vendored/getSummary.ts +++ b/packages/jest-plugin/src/vendored/getSummary.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2022-2023 Developer Innovations, LLC +// Copyright (c) 2022-2024 Developer Innovations, LLC /* This file includes portions of a Jest source code file originally downloaded from: diff --git a/packages/jest-plugin/test/.eslintrc.js b/packages/jest-plugin/test/.eslintrc.js index 4d0da9e..b7a3a23 100644 --- a/packages/jest-plugin/test/.eslintrc.js +++ b/packages/jest-plugin/test/.eslintrc.js @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC module.exports = { env: { diff --git a/packages/jest-plugin/test/babel.config.js b/packages/jest-plugin/test/babel.config.js index 67d5947..d3cffba 100644 --- a/packages/jest-plugin/test/babel.config.js +++ b/packages/jest-plugin/test/babel.config.js @@ -1,4 +1,4 @@ -// Copyright (c) 2022-2023 Developer Innovations, LLC +// Copyright (c) 2022-2024 Developer Innovations, LLC /* eslint-env node */ diff --git a/packages/jest-plugin/test/integration-input/jest.config.js b/packages/jest-plugin/test/integration-input/jest.config.js index e23590c..75d15b8 100644 --- a/packages/jest-plugin/test/integration-input/jest.config.js +++ b/packages/jest-plugin/test/integration-input/jest.config.js @@ -1,4 +1,4 @@ -// Copyright (c) 2022-2023 Developer Innovations, LLC +// Copyright (c) 2022-2024 Developer Innovations, LLC module.exports = { clearMocks: true, diff --git a/packages/jest-plugin/test/integration-input/src/fail.test.ts b/packages/jest-plugin/test/integration-input/src/fail.test.ts index cf69151..7448062 100644 --- a/packages/jest-plugin/test/integration-input/src/fail.test.ts +++ b/packages/jest-plugin/test/integration-input/src/fail.test.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2022-2023 Developer Innovations, LLC +// Copyright (c) 2022-2024 Developer Innovations, LLC describe("describe block", () => { (process.env.SKIP_FAILURES !== undefined ? it.skip : it)( diff --git a/packages/jest-plugin/test/integration-input/src/flake.test.ts b/packages/jest-plugin/test/integration-input/src/flake.test.ts index 6acbbbb..9aaf8f0 100644 --- a/packages/jest-plugin/test/integration-input/src/flake.test.ts +++ b/packages/jest-plugin/test/integration-input/src/flake.test.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2022-2023 Developer Innovations, LLC +// Copyright (c) 2022-2024 Developer Innovations, LLC import fs from "fs/promises"; diff --git a/packages/jest-plugin/test/integration-input/src/invalid.test.ts b/packages/jest-plugin/test/integration-input/src/invalid.test.ts index 626b065..7760c16 100644 --- a/packages/jest-plugin/test/integration-input/src/invalid.test.ts +++ b/packages/jest-plugin/test/integration-input/src/invalid.test.ts @@ -1,3 +1,3 @@ -// Copyright (c) 2022-2023 Developer Innovations, LLC +// Copyright (c) 2022-2024 Developer Innovations, LLC throw new Error("invalid test file"); diff --git a/packages/jest-plugin/test/integration-input/src/mixed.test.ts b/packages/jest-plugin/test/integration-input/src/mixed.test.ts index 129635d..05b0bf7 100644 --- a/packages/jest-plugin/test/integration-input/src/mixed.test.ts +++ b/packages/jest-plugin/test/integration-input/src/mixed.test.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2022-2023 Developer Innovations, LLC +// Copyright (c) 2022-2024 Developer Innovations, LLC // This test contains both a failed test and a quarantined one, which the reporter should treat as // a failed test file. However, the quarantined test should still be reported as having been diff --git a/packages/jest-plugin/test/integration-input/src/pass.test.ts b/packages/jest-plugin/test/integration-input/src/pass.test.ts index 8e79dbf..dbd1e58 100644 --- a/packages/jest-plugin/test/integration-input/src/pass.test.ts +++ b/packages/jest-plugin/test/integration-input/src/pass.test.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2022-2023 Developer Innovations, LLC +// Copyright (c) 2022-2024 Developer Innovations, LLC it("should pass", () => { if (process.env.TEST_SNAPSHOTS !== undefined) { diff --git a/packages/jest-plugin/test/integration-input/src/quarantined.test.ts b/packages/jest-plugin/test/integration-input/src/quarantined.test.ts index 397bae1..1b12471 100644 --- a/packages/jest-plugin/test/integration-input/src/quarantined.test.ts +++ b/packages/jest-plugin/test/integration-input/src/quarantined.test.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2022-2023 Developer Innovations, LLC +// Copyright (c) 2022-2024 Developer Innovations, LLC describe("describe block", () => { (process.env.SKIP_QUARANTINED !== undefined ? it.skip : it)( diff --git a/packages/jest-plugin/test/integration/jest.config.js b/packages/jest-plugin/test/integration/jest.config.js index ac2e971..ceb5c34 100644 --- a/packages/jest-plugin/test/integration/jest.config.js +++ b/packages/jest-plugin/test/integration/jest.config.js @@ -1,4 +1,4 @@ -// Copyright (c) 2022-2023 Developer Innovations, LLC +// Copyright (c) 2022-2024 Developer Innovations, LLC module.exports = { setupFilesAfterEnv: ["./src/matchers.ts"], diff --git a/packages/jest-plugin/test/integration/src/basic.test.ts b/packages/jest-plugin/test/integration/src/basic.test.ts index 2f81bec..c9427b3 100644 --- a/packages/jest-plugin/test/integration/src/basic.test.ts +++ b/packages/jest-plugin/test/integration/src/basic.test.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2022-2023 Developer Innovations, LLC +// Copyright (c) 2022-2024 Developer Innovations, LLC import { defaultExpectedResults, diff --git a/packages/jest-plugin/test/integration/src/config.test.ts b/packages/jest-plugin/test/integration/src/config.test.ts index 0f56c1e..59976d0 100644 --- a/packages/jest-plugin/test/integration/src/config.test.ts +++ b/packages/jest-plugin/test/integration/src/config.test.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2022-2023 Developer Innovations, LLC +// Copyright (c) 2022-2024 Developer Innovations, LLC import { defaultExpectedResults, diff --git a/packages/jest-plugin/test/integration/src/disable-plugin.test.ts b/packages/jest-plugin/test/integration/src/disable-plugin.test.ts index 5673309..d70dd1a 100644 --- a/packages/jest-plugin/test/integration/src/disable-plugin.test.ts +++ b/packages/jest-plugin/test/integration/src/disable-plugin.test.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2022-2023 Developer Innovations, LLC +// Copyright (c) 2022-2024 Developer Innovations, LLC import { defaultExpectedResults, diff --git a/packages/jest-plugin/test/integration/src/disable-upload.test.ts b/packages/jest-plugin/test/integration/src/disable-upload.test.ts index 38706c2..59f680f 100644 --- a/packages/jest-plugin/test/integration/src/disable-upload.test.ts +++ b/packages/jest-plugin/test/integration/src/disable-upload.test.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2022-2023 Developer Innovations, LLC +// Copyright (c) 2022-2024 Developer Innovations, LLC import { defaultExpectedResults, diff --git a/packages/jest-plugin/test/integration/src/force-color.js b/packages/jest-plugin/test/integration/src/force-color.js index 9807cd1..0978afe 100644 --- a/packages/jest-plugin/test/integration/src/force-color.js +++ b/packages/jest-plugin/test/integration/src/force-color.js @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC // Jest changes FORCE_COLOR to 1 when it forks child processes. We execute this script with // `node --require` so that each subprocess has the correct FORCE_COLOR value to produce the color diff --git a/packages/jest-plugin/test/integration/src/git.test.ts b/packages/jest-plugin/test/integration/src/git.test.ts index 0f8487d..62c3d4f 100644 --- a/packages/jest-plugin/test/integration/src/git.test.ts +++ b/packages/jest-plugin/test/integration/src/git.test.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2022-2023 Developer Innovations, LLC +// Copyright (c) 2022-2024 Developer Innovations, LLC import path from "path"; import { diff --git a/packages/jest-plugin/test/integration/src/ignore-failures.test.ts b/packages/jest-plugin/test/integration/src/ignore-failures.test.ts index e2e68c4..e8263b7 100644 --- a/packages/jest-plugin/test/integration/src/ignore-failures.test.ts +++ b/packages/jest-plugin/test/integration/src/ignore-failures.test.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2022-2023 Developer Innovations, LLC +// Copyright (c) 2022-2024 Developer Innovations, LLC import { defaultExpectedResults, diff --git a/packages/jest-plugin/test/integration/src/long-names.test.ts b/packages/jest-plugin/test/integration/src/long-names.test.ts index 27aa9d8..8284f15 100644 --- a/packages/jest-plugin/test/integration/src/long-names.test.ts +++ b/packages/jest-plugin/test/integration/src/long-names.test.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2022-2023 Developer Innovations, LLC +// Copyright (c) 2022-2024 Developer Innovations, LLC import { defaultExpectedResults, diff --git a/packages/jest-plugin/test/integration/src/matchers.ts b/packages/jest-plugin/test/integration/src/matchers.ts index 6919967..be025b9 100644 --- a/packages/jest-plugin/test/integration/src/matchers.ts +++ b/packages/jest-plugin/test/integration/src/matchers.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC import { MatcherHintOptions, diff --git a/packages/jest-plugin/test/integration/src/no-quarantine.test.ts b/packages/jest-plugin/test/integration/src/no-quarantine.test.ts index 97e735f..4d909b6 100644 --- a/packages/jest-plugin/test/integration/src/no-quarantine.test.ts +++ b/packages/jest-plugin/test/integration/src/no-quarantine.test.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2022-2023 Developer Innovations, LLC +// Copyright (c) 2022-2024 Developer Innovations, LLC import { integrationTest, integrationTestSuite } from "./test-wrappers"; diff --git a/packages/jest-plugin/test/integration/src/plugin-failures.test.ts b/packages/jest-plugin/test/integration/src/plugin-failures.test.ts index af57524..07ec795 100644 --- a/packages/jest-plugin/test/integration/src/plugin-failures.test.ts +++ b/packages/jest-plugin/test/integration/src/plugin-failures.test.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2022-2023 Developer Innovations, LLC +// Copyright (c) 2022-2024 Developer Innovations, LLC import { defaultExpectedResults, diff --git a/packages/jest-plugin/test/integration/src/retries.test.ts b/packages/jest-plugin/test/integration/src/retries.test.ts index 95f57e9..972ca92 100644 --- a/packages/jest-plugin/test/integration/src/retries.test.ts +++ b/packages/jest-plugin/test/integration/src/retries.test.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2022-2023 Developer Innovations, LLC +// Copyright (c) 2022-2024 Developer Innovations, LLC import { defaultExpectedResults, diff --git a/packages/jest-plugin/test/integration/src/run-test-case.ts b/packages/jest-plugin/test/integration/src/run-test-case.ts index fa869f3..d27fd17 100644 --- a/packages/jest-plugin/test/integration/src/run-test-case.ts +++ b/packages/jest-plugin/test/integration/src/run-test-case.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2022-2023 Developer Innovations, LLC +// Copyright (c) 2022-2024 Developer Innovations, LLC import { tmpName, TmpNameOptions } from "tmp"; import { diff --git a/packages/jest-plugin/test/integration/src/skip-tests.test.ts b/packages/jest-plugin/test/integration/src/skip-tests.test.ts index b122fbf..6586044 100644 --- a/packages/jest-plugin/test/integration/src/skip-tests.test.ts +++ b/packages/jest-plugin/test/integration/src/skip-tests.test.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2022-2023 Developer Innovations, LLC +// Copyright (c) 2022-2024 Developer Innovations, LLC import { integrationTest, integrationTestSuite } from "./test-wrappers"; diff --git a/packages/jest-plugin/test/integration/src/snapshots.test.ts b/packages/jest-plugin/test/integration/src/snapshots.test.ts index 7c4948c..afb9383 100644 --- a/packages/jest-plugin/test/integration/src/snapshots.test.ts +++ b/packages/jest-plugin/test/integration/src/snapshots.test.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2022-2023 Developer Innovations, LLC +// Copyright (c) 2022-2024 Developer Innovations, LLC import { defaultExpectedResults, diff --git a/packages/jest-plugin/test/integration/src/test-independence.test.ts b/packages/jest-plugin/test/integration/src/test-independence.test.ts index 9905c71..e60ef6f 100644 --- a/packages/jest-plugin/test/integration/src/test-independence.test.ts +++ b/packages/jest-plugin/test/integration/src/test-independence.test.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC import jestPackage from "jest/package.json"; import { diff --git a/packages/jest-plugin/test/integration/src/test-name-pattern.test.ts b/packages/jest-plugin/test/integration/src/test-name-pattern.test.ts index a1d6a7a..a7786ad 100644 --- a/packages/jest-plugin/test/integration/src/test-name-pattern.test.ts +++ b/packages/jest-plugin/test/integration/src/test-name-pattern.test.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2022-2023 Developer Innovations, LLC +// Copyright (c) 2022-2024 Developer Innovations, LLC import { defaultExpectedResults, diff --git a/packages/jest-plugin/test/integration/src/test-wrappers.ts b/packages/jest-plugin/test/integration/src/test-wrappers.ts index 70c53f7..7cae8b6 100644 --- a/packages/jest-plugin/test/integration/src/test-wrappers.ts +++ b/packages/jest-plugin/test/integration/src/test-wrappers.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC import jestPackage from "jest/package.json"; import path from "path"; diff --git a/packages/jest-plugin/test/integration/src/verify-output.ts b/packages/jest-plugin/test/integration/src/verify-output.ts index d0029de..7fee318 100644 --- a/packages/jest-plugin/test/integration/src/verify-output.ts +++ b/packages/jest-plugin/test/integration/src/verify-output.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC // These are the chalk-formatted strings that include console color codes. import escapeStringRegexp from "escape-string-regexp"; diff --git a/packages/jest-plugin/window.d.ts b/packages/jest-plugin/window.d.ts index acaa11a..e0f548f 100644 --- a/packages/jest-plugin/window.d.ts +++ b/packages/jest-plugin/window.d.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2022-2023 Developer Innovations, LLC +// Copyright (c) 2022-2024 Developer Innovations, LLC export {}; diff --git a/packages/js-api/.eslintrc.js b/packages/js-api/.eslintrc.js index 02409c8..ff96017 100644 --- a/packages/js-api/.eslintrc.js +++ b/packages/js-api/.eslintrc.js @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC module.exports = { extends: ["../../.eslintrc-ts.js"], diff --git a/packages/js-api/LICENSE b/packages/js-api/LICENSE index 53bf01b..fc24ed8 100644 --- a/packages/js-api/LICENSE +++ b/packages/js-api/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2022-2023 Developer Innovations, LLC +Copyright (c) 2022-2024 Developer Innovations, LLC Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/packages/js-api/src/consts.ts b/packages/js-api/src/consts.ts index cf19eab..8823e7f 100644 --- a/packages/js-api/src/consts.ts +++ b/packages/js-api/src/consts.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC export const JS_API_VERSION: string = // eslint-disable-next-line @typescript-eslint/no-var-requires diff --git a/packages/js-api/src/index.ts b/packages/js-api/src/index.ts index b2cc254..5ff206e 100644 --- a/packages/js-api/src/index.ts +++ b/packages/js-api/src/index.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2022-2023 Developer Innovations, LLC +// Copyright (c) 2022-2024 Developer Innovations, LLC import nodeFetch, { RequestInit, Response } from "node-fetch"; import _debug = require("debug"); diff --git a/packages/plugins-common/.eslintrc.js b/packages/plugins-common/.eslintrc.js index 02409c8..ff96017 100644 --- a/packages/plugins-common/.eslintrc.js +++ b/packages/plugins-common/.eslintrc.js @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC module.exports = { extends: ["../../.eslintrc-ts.js"], diff --git a/packages/plugins-common/src/config.ts b/packages/plugins-common/src/config.ts index d277fdd..23c171f 100644 --- a/packages/plugins-common/src/config.ts +++ b/packages/plugins-common/src/config.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC import { CosmiconfigResult } from "cosmiconfig/dist/types"; import { cosmiconfig, cosmiconfigSync } from "cosmiconfig"; diff --git a/packages/plugins-common/src/env.ts b/packages/plugins-common/src/env.ts index 8ac5d40..a89b628 100644 --- a/packages/plugins-common/src/env.ts +++ b/packages/plugins-common/src/env.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC import process from "process"; diff --git a/packages/plugins-common/src/git.ts b/packages/plugins-common/src/git.ts index d5d94cf..e2907f7 100644 --- a/packages/plugins-common/src/git.ts +++ b/packages/plugins-common/src/git.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2022-2023 Developer Innovations, LLC +// Copyright (c) 2022-2024 Developer Innovations, LLC import _debug from "debug"; import { simpleGit, SimpleGit, SimpleGitFactory } from "simple-git"; diff --git a/packages/plugins-common/src/index.ts b/packages/plugins-common/src/index.ts index ee6fa9e..5e9e953 100644 --- a/packages/plugins-common/src/index.ts +++ b/packages/plugins-common/src/index.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC import path from "path"; diff --git a/packages/plugins-common/src/manifest.ts b/packages/plugins-common/src/manifest.ts index e4799f1..c552b0c 100644 --- a/packages/plugins-common/src/manifest.ts +++ b/packages/plugins-common/src/manifest.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC import _debug from "debug"; import { diff --git a/packages/plugins-common/src/quarantine.ts b/packages/plugins-common/src/quarantine.ts index 70cd1b6..e8c83fb 100644 --- a/packages/plugins-common/src/quarantine.ts +++ b/packages/plugins-common/src/quarantine.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC // Avoid depending on the core of the js-api, which includes a bunch of Node dependencies. We need // this module to work in the browser for the Cypress plugin's skip-tests module. diff --git a/packages/test-common/.eslintrc.js b/packages/test-common/.eslintrc.js index 02409c8..ff96017 100644 --- a/packages/test-common/.eslintrc.js +++ b/packages/test-common/.eslintrc.js @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC module.exports = { extends: ["../../.eslintrc-ts.js"], diff --git a/packages/test-common/rollup.config.mjs b/packages/test-common/rollup.config.mjs index 0e63ae9..9ad0ca1 100644 --- a/packages/test-common/rollup.config.mjs +++ b/packages/test-common/rollup.config.mjs @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC import path from "path"; import pluginCommonJs from "@rollup/plugin-commonjs"; diff --git a/packages/test-common/src/config.ts b/packages/test-common/src/config.ts index c39d9a6..c9492da 100644 --- a/packages/test-common/src/config.ts +++ b/packages/test-common/src/config.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC import _debug from "debug"; import { diff --git a/packages/test-common/src/git.ts b/packages/test-common/src/git.ts index bbbadd7..4ae081d 100644 --- a/packages/test-common/src/git.ts +++ b/packages/test-common/src/git.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC import _debug from "debug"; import { setSimpleGitFactory } from "@unflakable/plugins-common"; diff --git a/packages/test-common/src/mock-backend.ts b/packages/test-common/src/mock-backend.ts index 2479a87..70dcb0a 100644 --- a/packages/test-common/src/mock-backend.ts +++ b/packages/test-common/src/mock-backend.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC import { CompletedRequest, diff --git a/packages/test-common/src/mock-cosmiconfig.ts b/packages/test-common/src/mock-cosmiconfig.ts index 1f6936e..90049d3 100644 --- a/packages/test-common/src/mock-cosmiconfig.ts +++ b/packages/test-common/src/mock-cosmiconfig.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC // Script loaded by Node.JS via --require that mocks cosmiconfig for testing. diff --git a/packages/test-common/src/mock-git.ts b/packages/test-common/src/mock-git.ts index fc84fea..477ac4e 100644 --- a/packages/test-common/src/mock-git.ts +++ b/packages/test-common/src/mock-git.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC // Script loaded by Node.JS via --require that mocks simple-git for testing. diff --git a/packages/test-common/src/spawn.ts b/packages/test-common/src/spawn.ts index 099042c..9143bcf 100644 --- a/packages/test-common/src/spawn.ts +++ b/packages/test-common/src/spawn.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC import { TextDecoder } from "util"; import treeKill from "tree-kill"; diff --git a/scripts/.eslintrc.js b/scripts/.eslintrc.js index 653facc..34454d6 100644 --- a/scripts/.eslintrc.js +++ b/scripts/.eslintrc.js @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Developer Innovations, LLC +// Copyright (c) 2023-2024 Developer Innovations, LLC module.exports = { extends: ["../.eslintrc-ts.js"], diff --git a/scripts/set-jest-version.ts b/scripts/set-jest-version.ts index 805403d..66037ca 100644 --- a/scripts/set-jest-version.ts +++ b/scripts/set-jest-version.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2022-2023 Developer Innovations, LLC +// Copyright (c) 2022-2024 Developer Innovations, LLC import { execSync, spawnSync } from "child_process"; import * as fs from "fs"; pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy