diff --git a/docs/configuration.md b/docs/configuration.md index 51d8ff097..360fff1cb 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -44,6 +44,7 @@ export const config = { - `timeout` — default per-test timeout in seconds; a test is killed if it stops responding. - `mocha` — [Mocha options](https://mochajs.org/#configuring-mocha-nodejs), including extra reporters. See [Reporters](/reports). +- `serial` — pattern for tests that must not run in parallel, e.g. `serial: '@serial'`. With `run-workers` they run one by one after all parallel workers finish. See [Serial tests](/parallel#serial-tests). - `workerInitializationDelay` — delay in milliseconds between spinning up parallel workers to prevent CPU spikes and stagger browser startup. Defaults to `200`. Set to `0` to disable. - `workerInitializationMaxDelay` — maximum total delay (in milliseconds) for worker initialization staggering. Defaults to `10000` (10 s). Set to `0` to disable capping. diff --git a/docs/parallel.md b/docs/parallel.md index ebefbe27c..2362c625f 100644 --- a/docs/parallel.md +++ b/docs/parallel.md @@ -40,6 +40,34 @@ npx codeceptjs run-workers 4 --by pool `--suites` is shorthand for `--by suite`. +### Serial tests + +Some tests can't share the stage: they change global settings, reset data, or toggle a feature flag that other tests read. Mark them with a tag and set the `serial` pattern in config: + +```js +// codecept.conf.js +export const config = { + serial: '@serial', +} +``` + +```js +Scenario('change account timezone @serial', ({ I }) => { + // ... +}) +``` + +`run-workers` keeps matching tests out of the parallel workers. When those workers finish, one more worker starts and runs the serial tests one by one. It is still a single run: `bootstrapAll`, `teardownAll` and `event.workers.result` fire once, and the exit code covers both parts. + +``` +Test distribution: + Worker 1: 14 tests + Worker 2: 13 tests + Worker 3: 4 tests (serial) +``` + +`serial` is matched against the full test title the same way `--grep` is, so a tag on a `Feature` makes all its scenarios serial. It works with every `--by` strategy and combines with `--grep`: only serial tests that also match grep are run. With [multiple browsers](#multiple-browsers) each profile gets its own serial worker, started one after another. + ### Multiple browsers Define browser profiles under `multiple` in `codecept.conf.js`: diff --git a/lib/command/run-workers.js b/lib/command/run-workers.js index 00c343532..76ecea0a0 100644 --- a/lib/command/run-workers.js +++ b/lib/command/run-workers.js @@ -55,7 +55,7 @@ export default async function (workerCount, selectedRuns, options) { output.print('Test distribution:') workerObjects.forEach((worker, index) => { const testCount = worker.tests.length - output.print(` Worker ${index + 1}: ${testCount} test${testCount !== 1 ? 's' : ''}`) + output.print(` Worker ${index + 1}: ${testCount} test${testCount !== 1 ? 's' : ''}${worker.serial ? ' (serial)' : ''}`) }) output.print() diff --git a/lib/command/workers/runTests.js b/lib/command/workers/runTests.js index 9d6519117..a0e12ca90 100644 --- a/lib/command/workers/runTests.js +++ b/lib/command/workers/runTests.js @@ -17,7 +17,7 @@ let stdout = '' const stderr = '' -const { options, tests, testRoot, workerIndex, poolMode } = workerData +const { options, tests, skipTests = [], testRoot, workerIndex, poolMode } = workerData // Global error handlers to catch critical errors but not test failures process.on('uncaughtException', (err) => { @@ -287,6 +287,7 @@ async function runPoolTests() { // Load only the assigned test file mocha.files = [testIdentifier] await loadTests(mocha) + filterSuites(mocha.suite.suites, test => !skipTests.includes(test.uid)) if (mocha.suite.total() > 0) { // Run only the tests in the current mocha suite @@ -423,16 +424,14 @@ async function filterTests() { mocha.files = files await loadTests(mocha) - // Recursively filter tests in all suites (including nested ones) - const filterSuiteTests = (suite) => { - suite.tests = suite.tests.filter(test => tests.indexOf(test.uid) >= 0) - for (const childSuite of suite.suites) { - filterSuiteTests(childSuite) - } - } + filterSuites(mocha.suite.suites, test => tests.indexOf(test.uid) >= 0) +} - for (const suite of mocha.suite.suites) { - filterSuiteTests(suite) +// Recursively filter tests in all suites (including nested ones) +function filterSuites(suites, keep) { + for (const suite of suites) { + suite.tests = suite.tests.filter(keep) + filterSuites(suite.suites, keep) } } diff --git a/lib/workers.js b/lib/workers.js index 1957bb8a9..ed3816aed 100644 --- a/lib/workers.js +++ b/lib/workers.js @@ -68,11 +68,12 @@ const withoutEmptyGroups = groups => { return nonEmpty.length ? nonEmpty : groups.slice(0, 1) } -const createWorker = (workerObject, isPoolMode = false) => { +const createWorker = (workerObject, isPoolMode = false, skipTests = []) => { const worker = new Worker(pathToWorker, { workerData: { options: simplifyObject(workerObject.options), tests: workerObject.tests, + skipTests, testRoot: workerObject.testRoot, workerIndex: workerObject.workerIndex + 1, poolMode: isPoolMode, @@ -224,6 +225,7 @@ class WorkerObject { */ constructor(workerIndex) { this.workerIndex = workerIndex + this.serial = false this.options = {} this.tests = [] this.testRoot = getTestRoot() @@ -299,6 +301,7 @@ class Workers extends EventEmitter { this.testGroups = [] this.testPool = [] this.testPoolInitialized = false + this.serialTests = [] this.isPoolMode = config.by === 'pool' this.activeWorkers = new Map() this.maxWorkers = numberOfWorkers // Track original worker count for pool mode @@ -323,9 +326,21 @@ class Workers extends EventEmitter { async _initWorkers(numberOfWorkers, config) { await this.splitTestsByGroups(numberOfWorkers, config) + this.serialTests = await this.createGroupOfSerialTests() + if (this.serialTests.length && !this.isPoolMode) { + this.testGroups = this.testGroups.map(group => group.filter(uid => !this.serialTests.includes(uid))).filter(group => group.length) + } // For function-based grouping, use the actual number of test groups created const actualNumberOfWorkers = isFunction(config.by) ? this.testGroups.length : numberOfWorkers this.workers = createWorkerObjects(this.testGroups, this.codecept.config, getTestRoot(config.testConfig), config.options, config.selectedRuns) + if (this.serialTests.length) { + const serialWorkers = createWorkerObjects([this.serialTests], this.codecept.config, getTestRoot(config.testConfig), config.options, config.selectedRuns) + for (const worker of serialWorkers) { + worker.workerIndex = this.workers.length + worker.serial = true + this.workers.push(worker) + } + } this.numberOfWorkers = this.workers.length } @@ -408,6 +423,32 @@ class Workers extends EventEmitter { return groups } + /** + * Collects tests matching `serial` config pattern. + * These tests are excluded from parallel workers and run one by one after them. + * + * @returns {Promise} + */ + async createGroupOfSerialTests() { + const pattern = this.codecept?.config.serial + if (!pattern) return [] + const serial = new RegExp(pattern) + + Container.createMocha(this.codecept.config.mocha || {}, this.options) + const mocha = Container.mocha() + mocha.files = this.codecept.testFiles + await loadTests(mocha, { reload: true }) + + const group = [] + mocha.suite.eachTest(test => { + if (test && serial.test(test.fullTitle()) && matchesGrep(mocha, test)) group.push(test.uid) + }) + + mocha.unloadFiles() + + return group + } + /** * @param {Number} numberOfWorkers */ @@ -534,7 +575,8 @@ class Workers extends EventEmitter { : 0 for (const worker of this.workers) { - const workerThread = createWorker(worker, this.isPoolMode) + if (worker.serial) continue + const workerThread = createWorker(worker, this.isPoolMode, this.serialTests) this._listenWorkerEvents(workerThread) workerThreads.push(workerThread) @@ -545,6 +587,8 @@ class Workers extends EventEmitter { } } + if (!workerThreads.length) this._runSerialWorker() + recorder.add('workers started', () => { // Workers are already running, this is just a placeholder step }) @@ -556,6 +600,14 @@ class Workers extends EventEmitter { }) } + _runSerialWorker() { + const parallelWorkers = this.workers.filter(worker => !worker.serial) + if (this.closedWorkers < parallelWorkers.length) return + const worker = this.workers[this.closedWorkers] + if (!worker) return + this._listenWorkerEvents(createWorker(worker)) + } + /** * @returns {Array} */ @@ -787,14 +839,10 @@ class Workers extends EventEmitter { process.exitCode = 1 } - if (this.isPoolMode) { - // Pool mode: finish when all workers have exited and no more tests - if (this.closedWorkers === this.numberOfWorkers) { - this._finishRun() - } - } else if (this.closedWorkers === this.numberOfWorkers) { - // Regular mode: finish when all original workers have exited + if (this.closedWorkers === this.numberOfWorkers) { this._finishRun() + } else { + this._runSerialWorker() } }) } diff --git a/test/data/sandbox/codecept.workers-serial.conf.js b/test/data/sandbox/codecept.workers-serial.conf.js new file mode 100644 index 000000000..d827a7c90 --- /dev/null +++ b/test/data/sandbox/codecept.workers-serial.conf.js @@ -0,0 +1,12 @@ +export const config = { + tests: './workers-serial/*_test.js', + timeout: 10000, + output: './output', + helpers: { + FileSystem: {}, + }, + include: {}, + serial: '@serial', + mocha: {}, + name: 'sandbox', +}; diff --git a/test/data/sandbox/workers-serial/a_serial_test.js b/test/data/sandbox/workers-serial/a_serial_test.js new file mode 100644 index 000000000..b383661ef --- /dev/null +++ b/test/data/sandbox/workers-serial/a_serial_test.js @@ -0,0 +1,11 @@ +import { track } from './timeline.js' + +Feature('Serial lane @serial') + +Scenario('serial one', async () => { + await track('serial one') +}) + +Scenario('serial two', async () => { + await track('serial two') +}) diff --git a/test/data/sandbox/workers-serial/b_parallel_test.js b/test/data/sandbox/workers-serial/b_parallel_test.js new file mode 100644 index 000000000..f23d39de7 --- /dev/null +++ b/test/data/sandbox/workers-serial/b_parallel_test.js @@ -0,0 +1,19 @@ +import { track } from './timeline.js' + +Feature('Parallel lane') + +Scenario('parallel one', async () => { + await track('parallel one') +}) + +Scenario('parallel two', async () => { + await track('parallel two') +}) + +Scenario('serial three @serial', async () => { + await track('serial three') +}) + +Scenario('parallel three', async () => { + await track('parallel three') +}) diff --git a/test/data/sandbox/workers-serial/timeline.js b/test/data/sandbox/workers-serial/timeline.js new file mode 100644 index 000000000..d407a2b78 --- /dev/null +++ b/test/data/sandbox/workers-serial/timeline.js @@ -0,0 +1,10 @@ +import fs from 'fs' +import { fileURLToPath } from 'url' + +export const timelineFile = fileURLToPath(new URL('../output/serial_timeline.log', import.meta.url)) + +export async function track(name, duration = 300) { + fs.appendFileSync(timelineFile, `start ${name}\n`) + await new Promise(resolve => setTimeout(resolve, duration)) + fs.appendFileSync(timelineFile, `end ${name}\n`) +} diff --git a/test/runner/run_workers_test.js b/test/runner/run_workers_test.js index 9781e9446..75d28f050 100644 --- a/test/runner/run_workers_test.js +++ b/test/runner/run_workers_test.js @@ -591,4 +591,67 @@ describe('CodeceptJS Workers Runner', function () { done() }) }) + + describe('serial tests', () => { + const timelineFile = path.join(codecept_dir, 'output', 'serial_timeline.log') + const serial_run = `${codecept_run_glob('codecept.workers-serial.conf.js')} 2` + + const timeline = () => fs.readFileSync(timelineFile, 'utf8').trim().split('\n') + + const expectSerialAfterParallel = (events, serialTests) => { + const serialEvents = serialTests.flatMap(name => [`start ${name}`, `end ${name}`]) + expect(events.slice(-serialEvents.length)).toEqual(serialEvents) + expect(events.slice(0, -serialEvents.length).filter(e => e.includes('serial'))).toEqual([]) + } + + beforeEach(() => { + fs.mkdirSync(path.dirname(timelineFile), { recursive: true }) + fs.rmSync(timelineFile, { force: true }) + }) + + after(() => fs.rmSync(timelineFile, { force: true })) + + for (const by of ['test', 'suite', 'pool']) { + it(`should run serial tests one by one after parallel ones with --by ${by}`, function (done) { + exec(`${serial_run} --by ${by}`, (err, stdout) => { + expect(stdout).toContain('3 tests (serial)') + expect(stdout).toContain('OK | 6 passed') + const events = timeline() + expect(events).toHaveLength(12) + expectSerialAfterParallel(events, ['serial three', 'serial one', 'serial two']) + expect(err).toEqual(null) + done() + }) + }) + } + + it('should apply grep to serial tests', function (done) { + exec(`${serial_run} --grep "serial one|parallel"`, (err, stdout) => { + expect(stdout).toContain('1 test (serial)') + expect(stdout).toContain('OK | 4 passed') + expectSerialAfterParallel(timeline(), ['serial one']) + expect(err).toEqual(null) + done() + }) + }) + + it('should run only serial tests when no parallel tests match grep', function (done) { + exec(`${serial_run} --grep "@serial"`, (err, stdout) => { + expect(stdout).toContain('Worker 1: 3 tests (serial)') + expect(stdout).not.toContain('Worker 2:') + expect(stdout).toContain('OK | 3 passed') + expect(err).toEqual(null) + done() + }) + }) + + it('should not start a serial worker when no serial tests match grep', function (done) { + exec(`${serial_run} --grep "parallel one"`, (err, stdout) => { + expect(stdout).not.toContain('(serial)') + expect(stdout).toContain('OK | 1 passed') + expect(err).toEqual(null) + done() + }) + }) + }) }) diff --git a/typings/index.d.ts b/typings/index.d.ts index 3d57f98ba..8686a5c3b 100644 --- a/typings/index.d.ts +++ b/typings/index.d.ts @@ -132,6 +132,15 @@ declare namespace CodeceptJS { * ``` */ grep?: string + /** + * Pattern for tests that must not run in parallel. + * In `run-workers` matching tests run one by one after all parallel workers finish. + * + * ```js + * serial: '@serial' + * ``` + */ + serial?: string | RegExp /** * Enable and configure helpers: *