feat(workers): run serial tests after parallel workers

docs/configuration.md

@@ -44,6 +44,7 @@ export const config = {
 
 - `timeout` — default per-test timeout in seconds; a test is killed if it stops responding.
 - `mocha` — [Mocha options](https://mochajs.org/#configuring-mocha-nodejs), including extra reporters. See [Reporters](/reports).
+- `serial` — pattern for tests that must not run in parallel, e.g. `serial: '@serial'`. With `run-workers` they run one by one after all parallel workers finish. See [Serial tests](/parallel#serial-tests).
 - `workerInitializationDelay` — delay in milliseconds between spinning up parallel workers to prevent CPU spikes and stagger browser startup. Defaults to `200`. Set to `0` to disable.
 - `workerInitializationMaxDelay` — maximum total delay (in milliseconds) for worker initialization staggering. Defaults to `10000` (10 s). Set to `0` to disable capping.
 

docs/parallel.md

@@ -40,6 +40,34 @@ npx codeceptjs run-workers 4 --by pool
 
 `--suites` is shorthand for `--by suite`.
 
+### Serial tests
+
+Some tests can't share the stage: they change global settings, reset data, or toggle a feature flag that other tests read. Mark them with a tag and set the `serial` pattern in config:
+
+```js
+// codecept.conf.js
+export const config = {
+  serial: '@serial',
+}
+```
+
+```js
+Scenario('change account timezone @serial', ({ I }) => {
+  // ...
+})
+```
+
+`run-workers` keeps matching tests out of the parallel workers. When those workers finish, one more worker starts and runs the serial tests one by one. It is still a single run: `bootstrapAll`, `teardownAll` and `event.workers.result` fire once, and the exit code covers both parts.
+
+```
+Test distribution:
+  Worker 1: 14 tests
+  Worker 2: 13 tests
+  Worker 3: 4 tests (serial)
+```
+
+`serial` is matched against the full test title the same way `--grep` is, so a tag on a `Feature` makes all its scenarios serial. It works with every `--by` strategy and combines with `--grep`: only serial tests that also match grep are run. With [multiple browsers](#multiple-browsers) each profile gets its own serial worker, started one after another.
+
 ### Multiple browsers
 
 Define browser profiles under `multiple` in `codecept.conf.js`:

lib/command/run-workers.js

@@ -55,7 +55,7 @@ export default async function (workerCount, selectedRuns, options) {
   output.print('Test distribution:')
   workerObjects.forEach((worker, index) => {
     const testCount = worker.tests.length
-    output.print(`  Worker ${index + 1}: ${testCount} test${testCount !== 1 ? 's' : ''}`)
+    output.print(`  Worker ${index + 1}: ${testCount} test${testCount !== 1 ? 's' : ''}${worker.serial ? ' (serial)' : ''}`)
   })
   output.print()
 

lib/command/workers/runTests.js

@@ -17,7 +17,7 @@ let stdout = ''
 
 const stderr = ''
 
-const { options, tests, testRoot, workerIndex, poolMode } = workerData
+const { options, tests, skipTests = [], testRoot, workerIndex, poolMode } = workerData
 
 // Global error handlers to catch critical errors but not test failures
 process.on('uncaughtException', (err) => {
@@ -287,6 +287,7 @@ async function runPoolTests() {
             // Load only the assigned test file
             mocha.files = [testIdentifier]
             await loadTests(mocha)
+            filterSuites(mocha.suite.suites, test => !skipTests.includes(test.uid))
 
             if (mocha.suite.total() > 0) {
               // Run only the tests in the current mocha suite
@@ -423,16 +424,14 @@ async function filterTests() {
   mocha.files = files
   await loadTests(mocha)
 
-  // Recursively filter tests in all suites (including nested ones)
-  const filterSuiteTests = (suite) => {
-    suite.tests = suite.tests.filter(test => tests.indexOf(test.uid) >= 0)
-    for (const childSuite of suite.suites) {
-      filterSuiteTests(childSuite)
-    }
-  }
+  filterSuites(mocha.suite.suites, test => tests.indexOf(test.uid) >= 0)
+}
 
-  for (const suite of mocha.suite.suites) {
-    filterSuiteTests(suite)
+// Recursively filter tests in all suites (including nested ones)
+function filterSuites(suites, keep) {
+  for (const suite of suites) {
+    suite.tests = suite.tests.filter(keep)
+    filterSuites(suite.suites, keep)
   }
 }
 

lib/workers.js

@@ -68,11 +68,12 @@ const withoutEmptyGroups = groups => {
   return nonEmpty.length ? nonEmpty : groups.slice(0, 1)
 }
 
-const createWorker = (workerObject, isPoolMode = false) => {
+const createWorker = (workerObject, isPoolMode = false, skipTests = []) => {
   const worker = new Worker(pathToWorker, {
     workerData: {
       options: simplifyObject(workerObject.options),
       tests: workerObject.tests,
+      skipTests,
       testRoot: workerObject.testRoot,
       workerIndex: workerObject.workerIndex + 1,
       poolMode: isPoolMode,
@@ -224,6 +225,7 @@ class WorkerObject {
    */
   constructor(workerIndex) {
     this.workerIndex = workerIndex
+    this.serial = false
     this.options = {}
     this.tests = []
     this.testRoot = getTestRoot()
@@ -299,6 +301,7 @@ class Workers extends EventEmitter {
     this.testGroups = []
     this.testPool = []
     this.testPoolInitialized = false
+    this.serialTests = []
     this.isPoolMode = config.by === 'pool'
     this.activeWorkers = new Map()
     this.maxWorkers = numberOfWorkers // Track original worker count for pool mode
@@ -323,9 +326,21 @@ class Workers extends EventEmitter {
 
   async _initWorkers(numberOfWorkers, config) {
     await this.splitTestsByGroups(numberOfWorkers, config)
+    this.serialTests = await this.createGroupOfSerialTests()
+    if (this.serialTests.length && !this.isPoolMode) {
+      this.testGroups = this.testGroups.map(group => group.filter(uid => !this.serialTests.includes(uid))).filter(group => group.length)
+    }
     // For function-based grouping, use the actual number of test groups created
     const actualNumberOfWorkers = isFunction(config.by) ? this.testGroups.length : numberOfWorkers
     this.workers = createWorkerObjects(this.testGroups, this.codecept.config, getTestRoot(config.testConfig), config.options, config.selectedRuns)
+    if (this.serialTests.length) {
+      const serialWorkers = createWorkerObjects([this.serialTests], this.codecept.config, getTestRoot(config.testConfig), config.options, config.selectedRuns)
+      for (const worker of serialWorkers) {
+        worker.workerIndex = this.workers.length
+        worker.serial = true
+        this.workers.push(worker)
+      }
+    }
     this.numberOfWorkers = this.workers.length
   }
 
@@ -408,6 +423,32 @@ class Workers extends EventEmitter {
     return groups
   }
 
+  /**
+   * Collects tests matching `serial` config pattern.
+   * These tests are excluded from parallel workers and run one by one after them.
+   *
+   * @returns {Promise<string[]>}
+   */
+  async createGroupOfSerialTests() {
+    const pattern = this.codecept?.config.serial
+    if (!pattern) return []
+    const serial = new RegExp(pattern)
+
+    Container.createMocha(this.codecept.config.mocha || {}, this.options)
+    const mocha = Container.mocha()
+    mocha.files = this.codecept.testFiles
+    await loadTests(mocha, { reload: true })
+
+    const group = []
+    mocha.suite.eachTest(test => {
+      if (test && serial.test(test.fullTitle()) && matchesGrep(mocha, test)) group.push(test.uid)
+    })
+
+    mocha.unloadFiles()
+
+    return group
+  }
+
   /**
    * @param {Number} numberOfWorkers
    */
@@ -534,7 +575,8 @@ class Workers extends EventEmitter {
       : 0
 
     for (const worker of this.workers) {
-      const workerThread = createWorker(worker, this.isPoolMode)
+      if (worker.serial) continue
+      const workerThread = createWorker(worker, this.isPoolMode, this.serialTests)
       this._listenWorkerEvents(workerThread)
       workerThreads.push(workerThread)
 
@@ -545,6 +587,8 @@ class Workers extends EventEmitter {
       }
     }
 
+    if (!workerThreads.length) this._runSerialWorker()
+
     recorder.add('workers started', () => {
       // Workers are already running, this is just a placeholder step
     })
@@ -556,6 +600,14 @@ class Workers extends EventEmitter {
     })
   }
 
+  _runSerialWorker() {
+    const parallelWorkers = this.workers.filter(worker => !worker.serial)
+    if (this.closedWorkers < parallelWorkers.length) return
+    const worker = this.workers[this.closedWorkers]
+    if (!worker) return
+    this._listenWorkerEvents(createWorker(worker))
+  }
+
   /**
    * @returns {Array<WorkerObject>}
    */
@@ -787,14 +839,10 @@ class Workers extends EventEmitter {
         process.exitCode = 1
       }
 
-      if (this.isPoolMode) {
-        // Pool mode: finish when all workers have exited and no more tests
-        if (this.closedWorkers === this.numberOfWorkers) {
-          this._finishRun()
-        }
-      } else if (this.closedWorkers === this.numberOfWorkers) {
-        // Regular mode: finish when all original workers have exited
+      if (this.closedWorkers === this.numberOfWorkers) {
         this._finishRun()
+      } else {
+        this._runSerialWorker()
       }
     })
   }

test/data/sandbox/codecept.workers-serial.conf.js

@@ -0,0 +1,12 @@
+export const config = {
+  tests: './workers-serial/*_test.js',
+  timeout: 10000,
+  output: './output',
+  helpers: {
+    FileSystem: {},
+  },
+  include: {},
+  serial: '@serial',
+  mocha: {},
+  name: 'sandbox',
+};

test/data/sandbox/workers-serial/a_serial_test.js

@@ -0,0 +1,11 @@
+import { track } from './timeline.js'
+
+Feature('Serial lane @serial')
+
+Scenario('serial one', async () => {
+  await track('serial one')
+})
+
+Scenario('serial two', async () => {
+  await track('serial two')
+})

test/data/sandbox/workers-serial/b_parallel_test.js

@@ -0,0 +1,19 @@
+import { track } from './timeline.js'
+
+Feature('Parallel lane')
+
+Scenario('parallel one', async () => {
+  await track('parallel one')
+})
+
+Scenario('parallel two', async () => {
+  await track('parallel two')
+})
+
+Scenario('serial three @serial', async () => {
+  await track('serial three')
+})
+
+Scenario('parallel three', async () => {
+  await track('parallel three')
+})

test/data/sandbox/workers-serial/timeline.js

@@ -0,0 +1,10 @@
+import fs from 'fs'
+import { fileURLToPath } from 'url'
+
+export const timelineFile = fileURLToPath(new URL('../output/serial_timeline.log', import.meta.url))
+
+export async function track(name, duration = 300) {
+  fs.appendFileSync(timelineFile, `start ${name}\n`)
+  await new Promise(resolve => setTimeout(resolve, duration))
+  fs.appendFileSync(timelineFile, `end ${name}\n`)
+}

test/runner/run_workers_test.js

@@ -591,4 +591,67 @@ describe('CodeceptJS Workers Runner', function () {
       done()
     })
   })
+
+  describe('serial tests', () => {
+    const timelineFile = path.join(codecept_dir, 'output', 'serial_timeline.log')
+    const serial_run = `${codecept_run_glob('codecept.workers-serial.conf.js')} 2`
+
+    const timeline = () => fs.readFileSync(timelineFile, 'utf8').trim().split('\n')
+
+    const expectSerialAfterParallel = (events, serialTests) => {
+      const serialEvents = serialTests.flatMap(name => [`start ${name}`, `end ${name}`])
+      expect(events.slice(-serialEvents.length)).toEqual(serialEvents)
+      expect(events.slice(0, -serialEvents.length).filter(e => e.includes('serial'))).toEqual([])
+    }
+
+    beforeEach(() => {
+      fs.mkdirSync(path.dirname(timelineFile), { recursive: true })
+      fs.rmSync(timelineFile, { force: true })
+    })
+
+    after(() => fs.rmSync(timelineFile, { force: true }))
+
+    for (const by of ['test', 'suite', 'pool']) {
+      it(`should run serial tests one by one after parallel ones with --by ${by}`, function (done) {
+        exec(`${serial_run} --by ${by}`, (err, stdout) => {
+          expect(stdout).toContain('3 tests (serial)')
+          expect(stdout).toContain('OK  | 6 passed')
+          const events = timeline()
+          expect(events).toHaveLength(12)
+          expectSerialAfterParallel(events, ['serial three', 'serial one', 'serial two'])
+          expect(err).toEqual(null)
+          done()
+        })
+      })
+    }
+
+    it('should apply grep to serial tests', function (done) {
+      exec(`${serial_run} --grep "serial one|parallel"`, (err, stdout) => {
+        expect(stdout).toContain('1 test (serial)')
+        expect(stdout).toContain('OK  | 4 passed')
+        expectSerialAfterParallel(timeline(), ['serial one'])
+        expect(err).toEqual(null)
+        done()
+      })
+    })
+
+    it('should run only serial tests when no parallel tests match grep', function (done) {
+      exec(`${serial_run} --grep "@serial"`, (err, stdout) => {
+        expect(stdout).toContain('Worker 1: 3 tests (serial)')
+        expect(stdout).not.toContain('Worker 2:')
+        expect(stdout).toContain('OK  | 3 passed')
+        expect(err).toEqual(null)
+        done()
+      })
+    })
+
+    it('should not start a serial worker when no serial tests match grep', function (done) {
+      exec(`${serial_run} --grep "parallel one"`, (err, stdout) => {
+        expect(stdout).not.toContain('(serial)')
+        expect(stdout).toContain('OK  | 1 passed')
+        expect(err).toEqual(null)
+        done()
+      })
+    })
+  })
 })

typings/index.d.ts

@@ -132,6 +132,15 @@ declare namespace CodeceptJS {
      * ```
      */
     grep?: string
+    /**
+     * Pattern for tests that must not run in parallel.
+     * In `run-workers` matching tests run one by one after all parallel workers finish.
+     *
+     * ```js
+     * serial: '@serial'
+     * ```
+     */
+    serial?: string | RegExp
     /**
      * Enable and configure helpers:
      *