Skip to content

armgit / mocha

Public Imported from GitHub
0

☕️ Classic, reliable, trusted test framework for Node.js and the browser

Allow to retry failed test from test context for #1773

Long Ho7d394ab0bf11
diff --git a/bin/_mocha b/bin/_mocha
index f8068835..d31154af 100755
--- a/bin/_mocha
+++ b/bin/_mocha
@@ -92,6 +92,7 @@ program
.option('--prof', 'log statistical profiling information')
.option('--recursive', 'include sub directories')
.option('--reporters', 'display available reporters')
+ .option('--retries <times>', 'set numbers of time to retry a failed test case')
.option('--throw-deprecation', 'throw an exception anytime a deprecated function is used')
.option('--trace', 'trace function calls')
.option('--trace-deprecation', 'show stack traces on deprecations')
@@ -284,6 +285,10 @@ if (program.delay) mocha.delay();
mocha.globals(globals);
+// --retries
+
+if (program.retries) mocha.suite.retries(program.retries);
+
// custom compiler support
var extensions = ['js'];
diff --git a/lib/context.js b/lib/context.js
index 0b0242fe..4e7247b8 100644
--- a/lib/context.js
+++ b/lib/context.js
@@ -76,6 +76,21 @@ Context.prototype.skip = function() {
return this;
};
+/**
+ * Allow a number of retries on failed tests
+ *
+ * @api private
+ * @param {number} n
+ * @return {Context} self
+ */
+Context.prototype.retries = function(n) {
+ if (!arguments.length) {
+ return this.runnable().retries();
+ }
+ this.runnable().retries(n);
+ return this;
+};
+
/**
* Inspect the context void of `._runnable`.
*
diff --git a/lib/interfaces/bdd.js b/lib/interfaces/bdd.js
index 253f24e4..97f55008 100644
--- a/lib/interfaces/bdd.js
+++ b/lib/interfaces/bdd.js
@@ -106,5 +106,12 @@ module.exports = function(suite) {
context.xit = context.xspecify = context.it.skip = function(title) {
context.it(title);
};
+
+ /**
+ * Number of attempts to retry.
+ */
+ context.it.retries = function(n) {
+ context.retries(n);
+ };
});
};
diff --git a/lib/interfaces/common.js b/lib/interfaces/common.js
index 4a21c16c..3e38030d 100644
--- a/lib/interfaces/common.js
+++ b/lib/interfaces/common.js
@@ -70,6 +70,15 @@ module.exports = function(suites, context) {
*/
skip: function(title) {
context.test(title);
+ },
+
+ /**
+ * Number of retry attempts
+ *
+ * @param {string} n
+ */
+ retries: function(n) {
+ context.retries(n);
}
}
};
diff --git a/lib/interfaces/qunit.js b/lib/interfaces/qunit.js
index be7f50fb..b79dcc68 100644
--- a/lib/interfaces/qunit.js
+++ b/lib/interfaces/qunit.js
@@ -89,5 +89,6 @@ module.exports = function(suite) {
};
context.test.skip = common.test.skip;
+ context.test.retries = common.test.retries;
});
};
diff --git a/lib/interfaces/tdd.js b/lib/interfaces/tdd.js
index fb22a791..8cb211ba 100644
--- a/lib/interfaces/tdd.js
+++ b/lib/interfaces/tdd.js
@@ -101,5 +101,6 @@ module.exports = function(suite) {
};
context.test.skip = common.test.skip;
+ context.test.retries = common.test.retries;
});
};
diff --git a/lib/mocha.js b/lib/mocha.js
index eb6c8d07..97cf57dc 100644
--- a/lib/mocha.js
+++ b/lib/mocha.js
@@ -62,6 +62,7 @@ function image(name) {
* - `reporter` reporter instance, defaults to `mocha.reporters.spec`
* - `globals` array of accepted globals
* - `timeout` timeout in milliseconds
+ * - `retries` number of times to retry failed tests
* - `bail` bail on the first test failure
* - `slow` milliseconds to wait before considering a test slow
* - `ignoreLeaks` ignore global leaks
@@ -88,6 +89,9 @@ function Mocha(options) {
if (typeof options.timeout !== 'undefined' && options.timeout !== null) {
this.timeout(options.timeout);
}
+ if (typeof options.retries !== 'undefined' && options.retries !== null) {
+ this.retries(options.retries);
+ }
this.useColors(options.useColors);
if (options.enableTimeouts !== null) {
this.enableTimeouts(options.enableTimeouts);
@@ -372,6 +376,18 @@ Mocha.prototype.timeout = function(timeout) {
return this;
};
+/**
+ * Set the number of times to retry failed tests.
+ *
+ * @param {Number} retry times
+ * @return {Mocha}
+ * @api public
+ */
+Mocha.prototype.retries = function(n) {
+ this.suite.retries(n);
+ return this;
+};
+
/**
* Set slowness threshold in milliseconds.
*
diff --git a/lib/reporters/json-cov.js b/lib/reporters/json-cov.js
index ed444e41..5a32569f 100644
--- a/lib/reporters/json-cov.js
+++ b/lib/reporters/json-cov.js
@@ -144,6 +144,7 @@ function coverage(filename, data) {
function clean(test) {
return {
duration: test.duration,
+ currentRetry: test.currentRetry(),
fullTitle: test.fullTitle(),
title: test.title
};
diff --git a/lib/reporters/json-stream.js b/lib/reporters/json-stream.js
index c8f0e113..02399e7d 100644
--- a/lib/reporters/json-stream.js
+++ b/lib/reporters/json-stream.js
@@ -54,6 +54,7 @@ function clean(test) {
return {
title: test.title,
fullTitle: test.fullTitle(),
- duration: test.duration
+ duration: test.duration,
+ currentRetry: test.currentRetry()
};
}
diff --git a/lib/reporters/json.js b/lib/reporters/json.js
index 35b760e2..cd9ec286 100644
--- a/lib/reporters/json.js
+++ b/lib/reporters/json.js
@@ -69,6 +69,7 @@ function clean(test) {
title: test.title,
fullTitle: test.fullTitle(),
duration: test.duration,
+ currentRetry: test.currentRetry(),
err: errorJSON(test.err || {})
};
}
diff --git a/lib/runnable.js b/lib/runnable.js
index 2bc21248..5e1ab767 100644
--- a/lib/runnable.js
+++ b/lib/runnable.js
@@ -52,6 +52,8 @@ function Runnable(title, fn) {
this._enableTimeouts = true;
this.timedOut = false;
this._trace = new Error('done() called multiple times');
+ this._retries = -1;
+ this._currentRetry = 0;
}
/**
@@ -128,6 +130,30 @@ Runnable.prototype.skip = function() {
throw new Pending();
};
+/**
+ * Set number of retries.
+ *
+ * @api private
+ */
+Runnable.prototype.retries = function(n) {
+ if (!arguments.length) {
+ return this._retries;
+ }
+ this._retries = n;
+};
+
+/**
+ * Get current retry
+ *
+ * @api private
+ */
+Runnable.prototype.currentRetry = function(n) {
+ if (!arguments.length) {
+ return this._currentRetry;
+ }
+ this._currentRetry = n;
+};
+
/**
* Return the full title generated by recursively concatenating the parent's
* full title.
diff --git a/lib/runner.js b/lib/runner.js
index d7656cda..c69ecadd 100644
--- a/lib/runner.js
+++ b/lib/runner.js
@@ -529,8 +529,16 @@ Runner.prototype.runTests = function(suite, fn) {
test = self.test;
if (err) {
+ var retry = test.currentRetry();
if (err instanceof Pending) {
self.emit('pending', test);
+ } else if (retry < test.retries()) {
+ test.currentRetry(retry + 1);
+ tests.unshift(test);
+
+ // Early return + hook trigger so that it doesn't
+ // increment the count wrong
+ return self.hookUp('afterEach', next);
} else {
self.fail(test, err);
}
diff --git a/lib/suite.js b/lib/suite.js
index 7834e284..8e6f393e 100644
--- a/lib/suite.js
+++ b/lib/suite.js
@@ -60,6 +60,7 @@ function Suite(title, parentContext) {
this._enableTimeouts = true;
this._slow = 75;
this._bail = false;
+ this._retries = -1;
this.delayed = false;
}
@@ -79,6 +80,7 @@ Suite.prototype.clone = function() {
debug('clone');
suite.ctx = this.ctx;
suite.timeout(this.timeout());
+ suite.retries(this.retries());
suite.enableTimeouts(this.enableTimeouts());
suite.slow(this.slow());
suite.bail(this.bail());
@@ -107,6 +109,22 @@ Suite.prototype.timeout = function(ms) {
return this;
};
+/**
+ * Set number of times to retry a failed test.
+ *
+ * @api private
+ * @param {number|string} n
+ * @return {Suite|number} for chaining
+ */
+Suite.prototype.retries = function(n) {
+ if (!arguments.length) {
+ return this._retries;
+ }
+ debug('retries %d', n);
+ this._retries = parseInt(n, 10) || 0;
+ return this;
+};
+
/**
* Set timeout to `enabled`.
*
@@ -179,6 +197,7 @@ Suite.prototype.beforeAll = function(title, fn) {
var hook = new Hook(title, fn);
hook.parent = this;
hook.timeout(this.timeout());
+ hook.retries(this.retries());
hook.enableTimeouts(this.enableTimeouts());
hook.slow(this.slow());
hook.ctx = this.ctx;
@@ -208,6 +227,7 @@ Suite.prototype.afterAll = function(title, fn) {
var hook = new Hook(title, fn);
hook.parent = this;
hook.timeout(this.timeout());
+ hook.retries(this.retries());
hook.enableTimeouts(this.enableTimeouts());
hook.slow(this.slow());
hook.ctx = this.ctx;
@@ -237,6 +257,7 @@ Suite.prototype.beforeEach = function(title, fn) {
var hook = new Hook(title, fn);
hook.parent = this;
hook.timeout(this.timeout());
+ hook.retries(this.retries());
hook.enableTimeouts(this.enableTimeouts());
hook.slow(this.slow());
hook.ctx = this.ctx;
@@ -266,6 +287,7 @@ Suite.prototype.afterEach = function(title, fn) {
var hook = new Hook(title, fn);
hook.parent = this;
hook.timeout(this.timeout());
+ hook.retries(this.retries());
hook.enableTimeouts(this.enableTimeouts());
hook.slow(this.slow());
hook.ctx = this.ctx;
@@ -284,6 +306,7 @@ Suite.prototype.afterEach = function(title, fn) {
Suite.prototype.addSuite = function(suite) {
suite.parent = this;
suite.timeout(this.timeout());
+ suite.retries(this.retries());
suite.enableTimeouts(this.enableTimeouts());
suite.slow(this.slow());
suite.bail(this.bail());
@@ -302,6 +325,7 @@ Suite.prototype.addSuite = function(suite) {
Suite.prototype.addTest = function(test) {
test.parent = this;
test.timeout(this.timeout());
+ test.retries(this.retries());
test.enableTimeouts(this.enableTimeouts());
test.slow(this.slow());
test.ctx = this.ctx;
diff --git a/test/integration/fixtures/options/retries.js b/test/integration/fixtures/options/retries.js
new file mode 100644
index 00000000..b2f76e12
--- /dev/null
+++ b/test/integration/fixtures/options/retries.js
@@ -0,0 +1,5 @@
+describe('retries', function() {
+ it('should fail', function () {
+ throw new Error('retry failure');
+ });
+});
diff --git a/test/integration/fixtures/retries/early-pass.js b/test/integration/fixtures/retries/early-pass.js
new file mode 100644
index 00000000..0010d120
--- /dev/null
+++ b/test/integration/fixtures/retries/early-pass.js
@@ -0,0 +1,11 @@
+describe('retries', function() {
+ this.retries(1);
+ var times = 0;
+
+ it('should pass after 1 retry', function() {
+ times++;
+ if (times !== 2) {
+ throw new Error('retry error ' + times);
+ }
+ });
+});
diff --git a/test/integration/fixtures/retries/hooks.js b/test/integration/fixtures/retries/hooks.js
new file mode 100644
index 00000000..47f6c6a1
--- /dev/null
+++ b/test/integration/fixtures/retries/hooks.js
@@ -0,0 +1,25 @@
+describe('retries', function() {
+ var times = 0;
+ before(function () {
+ console.log('before');
+ });
+
+ after(function () {
+ console.log('after');
+ });
+
+ beforeEach(function() {
+ console.log('before each', times);
+ });
+
+ afterEach(function () {
+ console.log('after each', times);
+ });
+
+ it('should allow override and run appropriate hooks', function(){
+ this.retries(4);
+ console.log('TEST', times);
+ times++;
+ throw new Error('retry error');
+ });
+});
diff --git a/test/integration/fixtures/retries/nested.js b/test/integration/fixtures/retries/nested.js
new file mode 100644
index 00000000..43c40cb2
--- /dev/null
+++ b/test/integration/fixtures/retries/nested.js
@@ -0,0 +1,9 @@
+describe('retries', function() {
+ this.retries(3);
+ describe('nested', function () {
+ it('should fail after only 1 retry', function(){
+ this.retries(1);
+ throw new Error('retry error');
+ });
+ });
+});
diff --git a/test/integration/options.js b/test/integration/options.js
index 52eff4b4..1e25b6ef 100644
--- a/test/integration/options.js
+++ b/test/integration/options.js
@@ -151,4 +151,20 @@ describe('options', function() {
});
});
});
+
+ describe('--retries', function() {
+ it('retries after a certain threshold', function (done) {
+ args = ['--retries', '3'];
+ run('options/retries.js', args, function(err, res) {
+ assert(!err);
+ assert.equal(res.stats.pending, 0);
+ assert.equal(res.stats.passes, 0);
+ assert.equal(res.stats.tests, 1);
+ assert.equal(res.tests[0].currentRetry, 3);
+ assert.equal(res.stats.failures, 1);
+ assert.equal(res.code, 1);
+ done();
+ });
+ })
+ });
});
diff --git a/test/integration/retries.js b/test/integration/retries.js
new file mode 100644
index 00000000..4e3b5297
--- /dev/null
+++ b/test/integration/retries.js
@@ -0,0 +1,72 @@
+var assert = require('assert');
+var helpers = require('./helpers');
+var args = [];
+
+describe('retries', function() {
+ this.timeout(1000);
+
+ it('are ran in correct order', function(done) {
+ helpers.runMocha('retries/hooks.js', args, function(err, res) {
+ var lines, expected;
+
+ assert(!err);
+
+ lines = res.output.split(/[\n․]+/).map(function(line) {
+ return line.trim();
+ }).filter(function(line) {
+ return line.length;
+ }).slice(0, -1);
+
+ expected = [
+ 'before',
+ 'before each 0',
+ 'TEST 0',
+ 'after each 1',
+ 'before each 1',
+ 'TEST 1',
+ 'after each 2',
+ 'before each 2',
+ 'TEST 2',
+ 'after each 3',
+ 'before each 3',
+ 'TEST 3',
+ 'after each 4',
+ 'before each 4',
+ 'TEST 4',
+ 'after each 5',
+ 'after'
+ ];
+
+ expected.forEach(function(line, i) {
+ assert.equal(lines[i], line);
+ });
+
+ assert.equal(res.code, 1);
+ done();
+ });
+ });
+
+ it('should exit early if test passes', function (done) {
+ helpers.runMochaJSON('retries/early-pass.js', args, function(err, res) {
+ assert(!err);
+ assert.equal(res.stats.passes, 1);
+ assert.equal(res.stats.failures, 0);
+ assert.equal(res.tests[0].currentRetry, 1);
+ assert.equal(res.stats.tests, 1);
+ assert.equal(res.code, 0);
+ done();
+ });
+ });
+
+ it('should let test override', function (done) {
+ helpers.runMochaJSON('retries/nested.js', args, function(err, res) {
+ assert(!err);
+ assert.equal(res.stats.passes, 0);
+ assert.equal(res.stats.failures, 1);
+ assert.equal(res.stats.tests, 1);
+ assert.equal(res.tests[0].currentRetry, 1);
+ assert.equal(res.code, 1);
+ done();
+ });
+ })
+});
diff --git a/test/runnable.js b/test/runnable.js
index 6d66e310..6323cfa7 100644
--- a/test/runnable.js
+++ b/test/runnable.js
@@ -89,6 +89,14 @@ describe('Runnable(title, fn)', function(){
})
})
+ describe('#retries(n)', function(){
+ it('should set the number of retries', function(){
+ var run = new Runnable;
+ run.retries(1);
+ run.retries().should.equal(1);
+ })
+ })
+
describe('.run(fn)', function(){
describe('when .pending', function(){
it('should not invoke the callback', function(done){