Skip to content

Open-Source API Development Ecosystem • https://hoppscotch.io • Offline, On-Prem & Cloud • Web, Desktop & CLI • Open-Source Alternative to Postman, Insomnia

feat: collection CLI runner with iterations and data feed (#4475)

shobanrajmb78cd57884c1
diff --git a/packages/hoppscotch-cli/README.md b/packages/hoppscotch-cli/README.md
index 5eac16511..28a1efdb5 100644
--- a/packages/hoppscotch-cli/README.md
+++ b/packages/hoppscotch-cli/README.md
@@ -28,31 +28,50 @@ hopp [options or commands] arguments
- Displays the help text
3. #### **`hopp test [options] <file_path>`**
+
- Interactive CLI to accept Hoppscotch collection JSON path
- Parses the collection JSON and executes each requests
- Executes pre-request script.
- Outputs the response of each request.
- Executes and outputs test-script response.
- #### Options:
+ #### Options:
+
+ ##### `-e <file_path>` / `--env <file_path>`
+
+ - Accepts path to env.json with contents in below format:
+
+ ```json
+ {
+ "ENV1": "value1",
+ "ENV2": "value2"
+ }
+ ```
+
+ - You can now access those variables using `pw.env.get('<var_name>')`
- ##### `-e <file_path>` / `--env <file_path>`
+ Taking the above example, `pw.env.get("ENV1")` will return `"value1"`
- - Accepts path to env.json with contents in below format:
+ ##### `--iteration-count <no_of_iterations>`
- ```json
- {
- "ENV1":"value1",
- "ENV2":"value2"
- }
- ```
+ - Accepts the number of iterations to run the collection
- - You can now access those variables using `pw.env.get('<var_name>')`
+ ##### `--iteration-data <file_path>`
- Taking the above example, `pw.env.get("ENV1")` will return `"value1"`
+ - Accepts the path to a CSV file with contents in the below format:
+
+ ```text
+ key1,key2,key3
+ value1,value2,value3
+ value4,value5,value6
+ ```
+
+ For every iteration the values will be replaced with the respective keys in the environment. For iteration 1 the value1,value2,value3 will be replaced and for iteration 2 value4,value5,value6 will be replaced and so on.
## Install
+
- Before you install Hoppscotch CLI you need to make sure you have the dependencies it requires to run.
+
- **Windows & macOS**: You will need `node-gyp` installed. Find instructions here: https://github.com/nodejs/node-gyp
- **Debian/Ubuntu derivatives**:
```sh
@@ -75,7 +94,6 @@ hopp [options or commands] arguments
sudo dnf install python3 make gcc gcc-c++ zlib-devel brotli-devel openssl-devel libuv-devel
```
-
- Once the dependencies are installed, install [@hoppscotch/cli](https://www.npmjs.com/package/@hoppscotch/cli) from npm by running:
```
npm i -g @hoppscotch/cli
@@ -112,39 +130,39 @@ Please note we have a code of conduct, please follow it in all your interactions
1. After cloning the repository, execute the following commands:
- ```bash
- pnpm install
- pnpm run build
- ```
+ ```bash
+ pnpm install
+ pnpm run build
+ ```
2. In order to test locally, you can use two types of package linking:
- 1. The 'pnpm exec' way (preferred since it does not hamper your original installation of the CLI):
+ 1. The 'pnpm exec' way (preferred since it does not hamper your original installation of the CLI):
- ```bash
- pnpm link @hoppscotch/cli
+ ```bash
+ pnpm link @hoppscotch/cli
- // Then to use or test the CLI:
- pnpm exec hopp
+ // Then to use or test the CLI:
+ pnpm exec hopp
- // After testing, to remove the package linking:
- pnpm rm @hoppscotch/cli
- ```
+ // After testing, to remove the package linking:
+ pnpm rm @hoppscotch/cli
+ ```
- 2. The 'global' way (warning: this might override the globally installed CLI, if exists):
+ 2. The 'global' way (warning: this might override the globally installed CLI, if exists):
- ```bash
- sudo pnpm link --global
+ ```bash
+ sudo pnpm link --global
- // Then to use or test the CLI:
- hopp
+ // Then to use or test the CLI:
+ hopp
- // After testing, to remove the package linking:
- sudo pnpm rm --global @hoppscotch/cli
- ```
+ // After testing, to remove the package linking:
+ sudo pnpm rm --global @hoppscotch/cli
+ ```
3. To use the Typescript watch scripts:
- ```bash
- pnpm run dev
- ```
+ ```bash
+ pnpm run dev
+ ```
diff --git a/packages/hoppscotch-cli/package.json b/packages/hoppscotch-cli/package.json
index 65ed6a94b..3f957b930 100644
--- a/packages/hoppscotch-cli/package.json
+++ b/packages/hoppscotch-cli/package.json
@@ -1,6 +1,6 @@
{
"name": "@hoppscotch/cli",
- "version": "0.12.0",
+ "version": "0.13.0",
"description": "A CLI to run Hoppscotch test scripts in CI environments.",
"homepage": "https://hoppscotch.io",
"type": "module",
@@ -51,7 +51,8 @@
"qs": "6.13.0",
"verzod": "0.2.3",
"xmlbuilder2": "3.1.1",
- "zod": "3.23.8"
+ "zod": "3.23.8",
+ "papaparse": "5.4.1"
},
"devDependencies": {
"@hoppscotch/data": "workspace:^",
@@ -64,6 +65,7 @@
"qs": "6.11.2",
"tsup": "8.3.0",
"typescript": "5.6.3",
- "vitest": "2.1.2"
+ "vitest": "2.1.2",
+ "@types/papaparse": "5.3.14"
}
}
diff --git a/packages/hoppscotch-cli/src/__tests__/e2e/commands/__snapshots__/test.spec.ts.snap b/packages/hoppscotch-cli/src/__tests__/e2e/commands/__snapshots__/test.spec.ts.snap
index 5c07416c8..070d36bd2 100644
--- a/packages/hoppscotch-cli/src/__tests__/e2e/commands/__snapshots__/test.spec.ts.snap
+++ b/packages/hoppscotch-cli/src/__tests__/e2e/commands/__snapshots__/test.spec.ts.snap
@@ -3,6 +3,92 @@
exports[`hopp test [options] <file_path_or_id> > Test\`hopp test <file_path_or_id> --env <file_path_or_id> --reporter-junit [path] > Generates a JUnit report at the default path 1`] = `
"<?xml version="1.0" encoding="UTF-8"?>
<testsuites tests="76" failures="2" errors="66" time="time">
+ <testsuite name="test-junit-report-export/assertions/error" time="time" timestamp="timestamp" tests="22" failures="0" errors="22">
+ <testcase name="\`toBeLevelxxx()\` error scenarios - Expected 200-level status but could not parse value 'foo'" classname="test-junit-report-export/assertions/error">
+ <error message="Expected 200-level status but could not parse value 'foo'"/>
+ </testcase>
+ <testcase name="\`toBeLevelxxx()\` error scenarios - Expected 200-level status but could not parse value 'foo'" classname="test-junit-report-export/assertions/error">
+ <error message="Expected 200-level status but could not parse value 'foo'"/>
+ </testcase>
+ <testcase name="\`toBeType()\` error scenarios - Argument for toBeType should be &quot;string&quot;, &quot;boolean&quot;, &quot;number&quot;, &quot;object&quot;, &quot;undefined&quot;, &quot;bigint&quot;, &quot;symbol&quot; or &quot;function&quot;" classname="test-junit-report-export/assertions/error">
+ <error message="Argument for toBeType should be &quot;string&quot;, &quot;boolean&quot;, &quot;number&quot;, &quot;object&quot;, &quot;undefined&quot;, &quot;bigint&quot;, &quot;symbol&quot; or &quot;function&quot;"/>
+ </testcase>
+ <testcase name="\`toBeType()\` error scenarios - Argument for toBeType should be &quot;string&quot;, &quot;boolean&quot;, &quot;number&quot;, &quot;object&quot;, &quot;undefined&quot;, &quot;bigint&quot;, &quot;symbol&quot; or &quot;function&quot;" classname="test-junit-report-export/assertions/error">
+ <error message="Argument for toBeType should be &quot;string&quot;, &quot;boolean&quot;, &quot;number&quot;, &quot;object&quot;, &quot;undefined&quot;, &quot;bigint&quot;, &quot;symbol&quot; or &quot;function&quot;"/>
+ </testcase>
+ <testcase name="\`toBeType()\` error scenarios - Argument for toBeType should be &quot;string&quot;, &quot;boolean&quot;, &quot;number&quot;, &quot;object&quot;, &quot;undefined&quot;, &quot;bigint&quot;, &quot;symbol&quot; or &quot;function&quot;" classname="test-junit-report-export/assertions/error">
+ <error message="Argument for toBeType should be &quot;string&quot;, &quot;boolean&quot;, &quot;number&quot;, &quot;object&quot;, &quot;undefined&quot;, &quot;bigint&quot;, &quot;symbol&quot; or &quot;function&quot;"/>
+ </testcase>
+ <testcase name="\`toBeType()\` error scenarios - Argument for toBeType should be &quot;string&quot;, &quot;boolean&quot;, &quot;number&quot;, &quot;object&quot;, &quot;undefined&quot;, &quot;bigint&quot;, &quot;symbol&quot; or &quot;function&quot;" classname="test-junit-report-export/assertions/error">
+ <error message="Argument for toBeType should be &quot;string&quot;, &quot;boolean&quot;, &quot;number&quot;, &quot;object&quot;, &quot;undefined&quot;, &quot;bigint&quot;, &quot;symbol&quot; or &quot;function&quot;"/>
+ </testcase>
+ <testcase name="\`toBeType()\` error scenarios - Argument for toBeType should be &quot;string&quot;, &quot;boolean&quot;, &quot;number&quot;, &quot;object&quot;, &quot;undefined&quot;, &quot;bigint&quot;, &quot;symbol&quot; or &quot;function&quot;" classname="test-junit-report-export/assertions/error">
+ <error message="Argument for toBeType should be &quot;string&quot;, &quot;boolean&quot;, &quot;number&quot;, &quot;object&quot;, &quot;undefined&quot;, &quot;bigint&quot;, &quot;symbol&quot; or &quot;function&quot;"/>
+ </testcase>
+ <testcase name="\`toBeType()\` error scenarios - Argument for toBeType should be &quot;string&quot;, &quot;boolean&quot;, &quot;number&quot;, &quot;object&quot;, &quot;undefined&quot;, &quot;bigint&quot;, &quot;symbol&quot; or &quot;function&quot;" classname="test-junit-report-export/assertions/error">
+ <error message="Argument for toBeType should be &quot;string&quot;, &quot;boolean&quot;, &quot;number&quot;, &quot;object&quot;, &quot;undefined&quot;, &quot;bigint&quot;, &quot;symbol&quot; or &quot;function&quot;"/>
+ </testcase>
+ <testcase name="\`toBeType()\` error scenarios - Argument for toBeType should be &quot;string&quot;, &quot;boolean&quot;, &quot;number&quot;, &quot;object&quot;, &quot;undefined&quot;, &quot;bigint&quot;, &quot;symbol&quot; or &quot;function&quot;" classname="test-junit-report-export/assertions/error">
+ <error message="Argument for toBeType should be &quot;string&quot;, &quot;boolean&quot;, &quot;number&quot;, &quot;object&quot;, &quot;undefined&quot;, &quot;bigint&quot;, &quot;symbol&quot; or &quot;function&quot;"/>
+ </testcase>
+ <testcase name="\`toBeType()\` error scenarios - Argument for toBeType should be &quot;string&quot;, &quot;boolean&quot;, &quot;number&quot;, &quot;object&quot;, &quot;undefined&quot;, &quot;bigint&quot;, &quot;symbol&quot; or &quot;function&quot;" classname="test-junit-report-export/assertions/error">
+ <error message="Argument for toBeType should be &quot;string&quot;, &quot;boolean&quot;, &quot;number&quot;, &quot;object&quot;, &quot;undefined&quot;, &quot;bigint&quot;, &quot;symbol&quot; or &quot;function&quot;"/>
+ </testcase>
+ <testcase name="\`toBeType()\` error scenarios - Argument for toBeType should be &quot;string&quot;, &quot;boolean&quot;, &quot;number&quot;, &quot;object&quot;, &quot;undefined&quot;, &quot;bigint&quot;, &quot;symbol&quot; or &quot;function&quot;" classname="test-junit-report-export/assertions/error">
+ <error message="Argument for toBeType should be &quot;string&quot;, &quot;boolean&quot;, &quot;number&quot;, &quot;object&quot;, &quot;undefined&quot;, &quot;bigint&quot;, &quot;symbol&quot; or &quot;function&quot;"/>
+ </testcase>
+ <testcase name="\`toBeType()\` error scenarios - Argument for toBeType should be &quot;string&quot;, &quot;boolean&quot;, &quot;number&quot;, &quot;object&quot;, &quot;undefined&quot;, &quot;bigint&quot;, &quot;symbol&quot; or &quot;function&quot;" classname="test-junit-report-export/assertions/error">
+ <error message="Argument for toBeType should be &quot;string&quot;, &quot;boolean&quot;, &quot;number&quot;, &quot;object&quot;, &quot;undefined&quot;, &quot;bigint&quot;, &quot;symbol&quot; or &quot;function&quot;"/>
+ </testcase>
+ <testcase name="\`toHaveLength()\` error scenarios - Expected toHaveLength to be called for an array or string" classname="test-junit-report-export/assertions/error">
+ <error message="Expected toHaveLength to be called for an array or string"/>
+ </testcase>
+ <testcase name="\`toHaveLength()\` error scenarios - Expected toHaveLength to be called for an array or string" classname="test-junit-report-export/assertions/error">
+ <error message="Expected toHaveLength to be called for an array or string"/>
+ </testcase>
+ <testcase name="\`toHaveLength()\` error scenarios - Expected toHaveLength to be called for an array or string" classname="test-junit-report-export/assertions/error">
+ <error message="Expected toHaveLength to be called for an array or string"/>
+ </testcase>
+ <testcase name="\`toHaveLength()\` error scenarios - Expected toHaveLength to be called for an array or string" classname="test-junit-report-export/assertions/error">
+ <error message="Expected toHaveLength to be called for an array or string"/>
+ </testcase>
+ <testcase name="\`toHaveLength()\` error scenarios - Argument for toHaveLength should be a number" classname="test-junit-report-export/assertions/error">
+ <error message="Argument for toHaveLength should be a number"/>
+ </testcase>
+ <testcase name="\`toHaveLength()\` error scenarios - Argument for toHaveLength should be a number" classname="test-junit-report-export/assertions/error">
+ <error message="Argument for toHaveLength should be a number"/>
+ </testcase>
+ <testcase name="\`toInclude() error scenarios\` - Expected toInclude to be called for an array or string" classname="test-junit-report-export/assertions/error">
+ <error message="Expected toInclude to be called for an array or string"/>
+ </testcase>
+ <testcase name="\`toInclude() error scenarios\` - Expected toInclude to be called for an array or string" classname="test-junit-report-export/assertions/error">
+ <error message="Expected toInclude to be called for an array or string"/>
+ </testcase>
+ <testcase name="\`toInclude() error scenarios\` - Argument for toInclude should not be null" classname="test-junit-report-export/assertions/error">
+ <error message="Argument for toInclude should not be null"/>
+ </testcase>
+ <testcase name="\`toInclude() error scenarios\` - Argument for toInclude should not be undefined" classname="test-junit-report-export/assertions/error">
+ <error message="Argument for toInclude should not be undefined"/>
+ </testcase>
+ </testsuite>
+ <testsuite name="test-junit-report-export/assertions/success" time="time" timestamp="timestamp" tests="5" failures="0" errors="0">
+ <testcase name="Status code is 200 - Expected '200' to be '200'" classname="test-junit-report-export/assertions/success"/>
+ <testcase name="Check headers - Expected 'application/json, text/plain, */*,image/webp' to be 'application/json, text/plain, */*,image/webp'" classname="test-junit-report-export/assertions/success"/>
+ <testcase name="Check headers - Expected 'echo.hoppscotch.io' to be 'echo.hoppscotch.io'" classname="test-junit-report-export/assertions/success"/>
+ <testcase name="Check headers - Expected 'undefined' to be 'undefined'" classname="test-junit-report-export/assertions/success"/>
+ <testcase name="Status code is 2xx - Expected '200' to be 200-level status" classname="test-junit-report-export/assertions/success"/>
+ </testsuite>
+ <testsuite name="test-junit-report-export/assertions/failure" time="time" timestamp="timestamp" tests="5" failures="2" errors="0">
+ <testcase name="Simulating failure - Status code is 200 - Expected '200' to not be '200'" classname="test-junit-report-export/assertions/failure">
+ <failure type="AssertionFailure" message="Expected '200' to not be '200'"/>
+ </testcase>
+ <testcase name="Simulating failure - Check headers - Expected 'application/json, text/plain, */*,image/webp' to not be 'application/json, text/plain, */*'" classname="test-junit-report-export/assertions/failure"/>
+ <testcase name="Simulating failure - Check headers - Expected 'echo.hoppscotch.io' to not be 'httpbin.org'" classname="test-junit-report-export/assertions/failure"/>
+ <testcase name="Simulating failure - Check headers - Expected 'undefined' to not be 'value'" classname="test-junit-report-export/assertions/failure"/>
+ <testcase name="Simulating failure - Status code is 2xx - Expected '200' to not be 200-level status" classname="test-junit-report-export/assertions/failure">
+ <failure type="AssertionFailure" message="Expected '200' to not be 200-level status"/>
+ </testcase>
+ </testsuite>
<testsuite name="test-junit-report-export/request-level-errors/invalid-url" time="time" timestamp="timestamp" tests="22" failures="0" errors="22">
<system-err><![CDATA[
REQUEST_ERROR - TypeError: Invalid URL]]></system-err>
@@ -150,6 +236,12 @@ exports[`hopp test [options] <file_path_or_id> > Test\`hopp test <file_path_or_i
<error message="Argument for toInclude should not be undefined"/>
</testcase>
</testsuite>
+</testsuites>"
+`;
+
+exports[`hopp test [options] <file_path_or_id> > Test\`hopp test <file_path_or_id> --env <file_path_or_id> --reporter-junit [path] > Generates a JUnit report at the specified path 1`] = `
+"<?xml version="1.0" encoding="UTF-8"?>
+<testsuites tests="76" failures="2" errors="66" time="time">
<testsuite name="test-junit-report-export/assertions/error" time="time" timestamp="timestamp" tests="22" failures="0" errors="22">
<testcase name="\`toBeLevelxxx()\` error scenarios - Expected 200-level status but could not parse value 'foo'" classname="test-junit-report-export/assertions/error">
<error message="Expected 200-level status but could not parse value 'foo'"/>
@@ -236,12 +328,6 @@ exports[`hopp test [options] <file_path_or_id> > Test\`hopp test <file_path_or_i
<failure type="AssertionFailure" message="Expected '200' to not be 200-level status"/>
</testcase>
</testsuite>
-</testsuites>"
-`;
-
-exports[`hopp test [options] <file_path_or_id> > Test\`hopp test <file_path_or_id> --env <file_path_or_id> --reporter-junit [path] > Generates a JUnit report at the specified path 1`] = `
-"<?xml version="1.0" encoding="UTF-8"?>
-<testsuites tests="76" failures="2" errors="66" time="time">
<testsuite name="test-junit-report-export/request-level-errors/invalid-url" time="time" timestamp="timestamp" tests="22" failures="0" errors="22">
<system-err><![CDATA[
REQUEST_ERROR - TypeError: Invalid URL]]></system-err>
@@ -389,92 +475,6 @@ exports[`hopp test [options] <file_path_or_id> > Test\`hopp test <file_path_or_i
<error message="Argument for toInclude should not be undefined"/>
</testcase>
</testsuite>
- <testsuite name="test-junit-report-export/assertions/error" time="time" timestamp="timestamp" tests="22" failures="0" errors="22">
- <testcase name="\`toBeLevelxxx()\` error scenarios - Expected 200-level status but could not parse value 'foo'" classname="test-junit-report-export/assertions/error">
- <error message="Expected 200-level status but could not parse value 'foo'"/>
- </testcase>
- <testcase name="\`toBeLevelxxx()\` error scenarios - Expected 200-level status but could not parse value 'foo'" classname="test-junit-report-export/assertions/error">
- <error message="Expected 200-level status but could not parse value 'foo'"/>
- </testcase>
- <testcase name="\`toBeType()\` error scenarios - Argument for toBeType should be &quot;string&quot;, &quot;boolean&quot;, &quot;number&quot;, &quot;object&quot;, &quot;undefined&quot;, &quot;bigint&quot;, &quot;symbol&quot; or &quot;function&quot;" classname="test-junit-report-export/assertions/error">
- <error message="Argument for toBeType should be &quot;string&quot;, &quot;boolean&quot;, &quot;number&quot;, &quot;object&quot;, &quot;undefined&quot;, &quot;bigint&quot;, &quot;symbol&quot; or &quot;function&quot;"/>
- </testcase>
- <testcase name="\`toBeType()\` error scenarios - Argument for toBeType should be &quot;string&quot;, &quot;boolean&quot;, &quot;number&quot;, &quot;object&quot;, &quot;undefined&quot;, &quot;bigint&quot;, &quot;symbol&quot; or &quot;function&quot;" classname="test-junit-report-export/assertions/error">
- <error message="Argument for toBeType should be &quot;string&quot;, &quot;boolean&quot;, &quot;number&quot;, &quot;object&quot;, &quot;undefined&quot;, &quot;bigint&quot;, &quot;symbol&quot; or &quot;function&quot;"/>
- </testcase>
- <testcase name="\`toBeType()\` error scenarios - Argument for toBeType should be &quot;string&quot;, &quot;boolean&quot;, &quot;number&quot;, &quot;object&quot;, &quot;undefined&quot;, &quot;bigint&quot;, &quot;symbol&quot; or &quot;function&quot;" classname="test-junit-report-export/assertions/error">
- <error message="Argument for toBeType should be &quot;string&quot;, &quot;boolean&quot;, &quot;number&quot;, &quot;object&quot;, &quot;undefined&quot;, &quot;bigint&quot;, &quot;symbol&quot; or &quot;function&quot;"/>
- </testcase>
- <testcase name="\`toBeType()\` error scenarios - Argument for toBeType should be &quot;string&quot;, &quot;boolean&quot;, &quot;number&quot;, &quot;object&quot;, &quot;undefined&quot;, &quot;bigint&quot;, &quot;symbol&quot; or &quot;function&quot;" classname="test-junit-report-export/assertions/error">
- <error message="Argument for toBeType should be &quot;string&quot;, &quot;boolean&quot;, &quot;number&quot;, &quot;object&quot;, &quot;undefined&quot;, &quot;bigint&quot;, &quot;symbol&quot; or &quot;function&quot;"/>
- </testcase>
- <testcase name="\`toBeType()\` error scenarios - Argument for toBeType should be &quot;string&quot;, &quot;boolean&quot;, &quot;number&quot;, &quot;object&quot;, &quot;undefined&quot;, &quot;bigint&quot;, &quot;symbol&quot; or &quot;function&quot;" classname="test-junit-report-export/assertions/error">
- <error message="Argument for toBeType should be &quot;string&quot;, &quot;boolean&quot;, &quot;number&quot;, &quot;object&quot;, &quot;undefined&quot;, &quot;bigint&quot;, &quot;symbol&quot; or &quot;function&quot;"/>
- </testcase>
- <testcase name="\`toBeType()\` error scenarios - Argument for toBeType should be &quot;string&quot;, &quot;boolean&quot;, &quot;number&quot;, &quot;object&quot;, &quot;undefined&quot;, &quot;bigint&quot;, &quot;symbol&quot; or &quot;function&quot;" classname="test-junit-report-export/assertions/error">
- <error message="Argument for toBeType should be &quot;string&quot;, &quot;boolean&quot;, &quot;number&quot;, &quot;object&quot;, &quot;undefined&quot;, &quot;bigint&quot;, &quot;symbol&quot; or &quot;function&quot;"/>
- </testcase>
- <testcase name="\`toBeType()\` error scenarios - Argument for toBeType should be &quot;string&quot;, &quot;boolean&quot;, &quot;number&quot;, &quot;object&quot;, &quot;undefined&quot;, &quot;bigint&quot;, &quot;symbol&quot; or &quot;function&quot;" classname="test-junit-report-export/assertions/error">
- <error message="Argument for toBeType should be &quot;string&quot;, &quot;boolean&quot;, &quot;number&quot;, &quot;object&quot;, &quot;undefined&quot;, &quot;bigint&quot;, &quot;symbol&quot; or &quot;function&quot;"/>
- </testcase>
- <testcase name="\`toBeType()\` error scenarios - Argument for toBeType should be &quot;string&quot;, &quot;boolean&quot;, &quot;number&quot;, &quot;object&quot;, &quot;undefined&quot;, &quot;bigint&quot;, &quot;symbol&quot; or &quot;function&quot;" classname="test-junit-report-export/assertions/error">
- <error message="Argument for toBeType should be &quot;string&quot;, &quot;boolean&quot;, &quot;number&quot;, &quot;object&quot;, &quot;undefined&quot;, &quot;bigint&quot;, &quot;symbol&quot; or &quot;function&quot;"/>
- </testcase>
- <testcase name="\`toBeType()\` error scenarios - Argument for toBeType should be &quot;string&quot;, &quot;boolean&quot;, &quot;number&quot;, &quot;object&quot;, &quot;undefined&quot;, &quot;bigint&quot;, &quot;symbol&quot; or &quot;function&quot;" classname="test-junit-report-export/assertions/error">
- <error message="Argument for toBeType should be &quot;string&quot;, &quot;boolean&quot;, &quot;number&quot;, &quot;object&quot;, &quot;undefined&quot;, &quot;bigint&quot;, &quot;symbol&quot; or &quot;function&quot;"/>
- </testcase>
- <testcase name="\`toBeType()\` error scenarios - Argument for toBeType should be &quot;string&quot;, &quot;boolean&quot;, &quot;number&quot;, &quot;object&quot;, &quot;undefined&quot;, &quot;bigint&quot;, &quot;symbol&quot; or &quot;function&quot;" classname="test-junit-report-export/assertions/error">
- <error message="Argument for toBeType should be &quot;string&quot;, &quot;boolean&quot;, &quot;number&quot;, &quot;object&quot;, &quot;undefined&quot;, &quot;bigint&quot;, &quot;symbol&quot; or &quot;function&quot;"/>
- </testcase>
- <testcase name="\`toHaveLength()\` error scenarios - Expected toHaveLength to be called for an array or string" classname="test-junit-report-export/assertions/error">
- <error message="Expected toHaveLength to be called for an array or string"/>
- </testcase>
- <testcase name="\`toHaveLength()\` error scenarios - Expected toHaveLength to be called for an array or string" classname="test-junit-report-export/assertions/error">
- <error message="Expected toHaveLength to be called for an array or string"/>
- </testcase>
- <testcase name="\`toHaveLength()\` error scenarios - Expected toHaveLength to be called for an array or string" classname="test-junit-report-export/assertions/error">
- <error message="Expected toHaveLength to be called for an array or string"/>
- </testcase>
- <testcase name="\`toHaveLength()\` error scenarios - Expected toHaveLength to be called for an array or string" classname="test-junit-report-export/assertions/error">
- <error message="Expected toHaveLength to be called for an array or string"/>
- </testcase>
- <testcase name="\`toHaveLength()\` error scenarios - Argument for toHaveLength should be a number" classname="test-junit-report-export/assertions/error">
- <error message="Argument for toHaveLength should be a number"/>
- </testcase>
- <testcase name="\`toHaveLength()\` error scenarios - Argument for toHaveLength should be a number" classname="test-junit-report-export/assertions/error">
- <error message="Argument for toHaveLength should be a number"/>
- </testcase>
- <testcase name="\`toInclude() error scenarios\` - Expected toInclude to be called for an array or string" classname="test-junit-report-export/assertions/error">
- <error message="Expected toInclude to be called for an array or string"/>
- </testcase>
- <testcase name="\`toInclude() error scenarios\` - Expected toInclude to be called for an array or string" classname="test-junit-report-export/assertions/error">
- <error message="Expected toInclude to be called for an array or string"/>
- </testcase>
- <testcase name="\`toInclude() error scenarios\` - Argument for toInclude should not be null" classname="test-junit-report-export/assertions/error">
- <error message="Argument for toInclude should not be null"/>
- </testcase>
- <testcase name="\`toInclude() error scenarios\` - Argument for toInclude should not be undefined" classname="test-junit-report-export/assertions/error">
- <error message="Argument for toInclude should not be undefined"/>
- </testcase>
- </testsuite>
- <testsuite name="test-junit-report-export/assertions/success" time="time" timestamp="timestamp" tests="5" failures="0" errors="0">
- <testcase name="Status code is 200 - Expected '200' to be '200'" classname="test-junit-report-export/assertions/success"/>
- <testcase name="Check headers - Expected 'application/json, text/plain, */*,image/webp' to be 'application/json, text/plain, */*,image/webp'" classname="test-junit-report-export/assertions/success"/>
- <testcase name="Check headers - Expected 'echo.hoppscotch.io' to be 'echo.hoppscotch.io'" classname="test-junit-report-export/assertions/success"/>
- <testcase name="Check headers - Expected 'undefined' to be 'undefined'" classname="test-junit-report-export/assertions/success"/>
- <testcase name="Status code is 2xx - Expected '200' to be 200-level status" classname="test-junit-report-export/assertions/success"/>
- </testsuite>
- <testsuite name="test-junit-report-export/assertions/failure" time="time" timestamp="timestamp" tests="5" failures="2" errors="0">
- <testcase name="Simulating failure - Status code is 200 - Expected '200' to not be '200'" classname="test-junit-report-export/assertions/failure">
- <failure type="AssertionFailure" message="Expected '200' to not be '200'"/>
- </testcase>
- <testcase name="Simulating failure - Check headers - Expected 'application/json, text/plain, */*,image/webp' to not be 'application/json, text/plain, */*'" classname="test-junit-report-export/assertions/failure"/>
- <testcase name="Simulating failure - Check headers - Expected 'echo.hoppscotch.io' to not be 'httpbin.org'" classname="test-junit-report-export/assertions/failure"/>
- <testcase name="Simulating failure - Check headers - Expected 'undefined' to not be 'value'" classname="test-junit-report-export/assertions/failure"/>
- <testcase name="Simulating failure - Status code is 2xx - Expected '200' to not be 200-level status" classname="test-junit-report-export/assertions/failure">
- <failure type="AssertionFailure" message="Expected '200' to not be 200-level status"/>
- </testcase>
- </testsuite>
</testsuites>"
`;
@@ -501,14 +501,6 @@ exports[`hopp test [options] <file_path_or_id> > Test\`hopp test <file_path_or_i
exports[`hopp test [options] <file_path_or_id> > Test\`hopp test <file_path_or_id> --env <file_path_or_id> --reporter-junit [path] > Generates a JUnit report for a collection with authorization/headers set at the collection level 1`] = `
"<?xml version="1.0" encoding="UTF-8"?>
<testsuites tests="12" failures="0" errors="0" time="time">
- <testsuite name="CollectionB/RequestA" time="time" timestamp="timestamp" tests="2" failures="0" errors="0">
- <testcase name="Correctly inherits auth and headers from the root collection - Expected 'Set at root collection' to be 'Set at root collection'" classname="CollectionB/RequestA"/>
- <testcase name="Correctly inherits auth and headers from the root collection - Expected 'Bearer BearerToken' to be 'Bearer BearerToken'" classname="CollectionB/RequestA"/>
- </testsuite>
- <testsuite name="CollectionB/FolderA/RequestB" time="time" timestamp="timestamp" tests="2" failures="0" errors="0">
- <testcase name="Correctly inherits auth and headers from the parent folder - Expected 'Set at root collection' to be 'Set at root collection'" classname="CollectionB/FolderA/RequestB"/>
- <testcase name="Correctly inherits auth and headers from the parent folder - Expected 'Bearer BearerToken' to be 'Bearer BearerToken'" classname="CollectionB/FolderA/RequestB"/>
- </testsuite>
<testsuite name="CollectionA/RequestA" time="time" timestamp="timestamp" tests="2" failures="0" errors="0">
<testcase name="Correctly inherits auth and headers from the root collection - Expected 'Set at root collection' to be 'Set at root collection'" classname="CollectionA/RequestA"/>
<testcase name="Correctly inherits auth and headers from the root collection - Expected 'Bearer BearerToken' to be 'Bearer BearerToken'" classname="CollectionA/RequestA"/>
@@ -525,5 +517,13 @@ exports[`hopp test [options] <file_path_or_id> > Test\`hopp test <file_path_or_i
<testcase name="Overrides auth and headers set at the parent folder - Expected 'Overriden at RequestD' to be 'Overriden at RequestD'" classname="CollectionA/FolderA/FolderB/FolderC/RequestD"/>
<testcase name="Overrides auth and headers set at the parent folder - Expected 'Basic dXNlcm5hbWU6cGFzc3dvcmQ=' to be 'Basic dXNlcm5hbWU6cGFzc3dvcmQ='" classname="CollectionA/FolderA/FolderB/FolderC/RequestD"/>
</testsuite>
+ <testsuite name="CollectionB/RequestA" time="time" timestamp="timestamp" tests="2" failures="0" errors="0">
+ <testcase name="Correctly inherits auth and headers from the root collection - Expected 'Set at root collection' to be 'Set at root collection'" classname="CollectionB/RequestA"/>
+ <testcase name="Correctly inherits auth and headers from the root collection - Expected 'Bearer BearerToken' to be 'Bearer BearerToken'" classname="CollectionB/RequestA"/>
+ </testsuite>
+ <testsuite name="CollectionB/FolderA/RequestB" time="time" timestamp="timestamp" tests="2" failures="0" errors="0">
+ <testcase name="Correctly inherits auth and headers from the parent folder - Expected 'Set at root collection' to be 'Set at root collection'" classname="CollectionB/FolderA/RequestB"/>
+ <testcase name="Correctly inherits auth and headers from the parent folder - Expected 'Bearer BearerToken' to be 'Bearer BearerToken'" classname="CollectionB/FolderA/RequestB"/>
+ </testsuite>
</testsuites>"
`;
diff --git a/packages/hoppscotch-cli/src/__tests__/e2e/commands/test.spec.ts b/packages/hoppscotch-cli/src/__tests__/e2e/commands/test.spec.ts
index ac89236ca..f386046ec 100644
--- a/packages/hoppscotch-cli/src/__tests__/e2e/commands/test.spec.ts
+++ b/packages/hoppscotch-cli/src/__tests__/e2e/commands/test.spec.ts
@@ -1,7 +1,7 @@
import { ExecException } from "child_process";
-import { afterAll, beforeAll, describe, expect, test } from "vitest";
import fs from "fs";
import path from "path";
+import { afterAll, beforeAll, describe, expect, test } from "vitest";
import { HoppErrorCode } from "../../../types/errors";
import { getErrorCode, getTestJsonFilePath, runCLI } from "../../utils";
@@ -181,6 +181,45 @@ describe("hopp test [options] <file_path_or_id>", { timeout: 100000 }, () => {
});
});
+ test("Ensures tests run in sequence order based on request path", async () => {
+ // Expected order of collection runs
+ const expectedOrder = [
+ "root-collection-request",
+ "folder-1/folder-1-request",
+ "folder-1/folder-11/folder-11-request",
+ "folder-1/folder-12/folder-12-request",
+ "folder-1/folder-13/folder-13-request",
+ "folder-2/folder-2-request",
+ "folder-2/folder-21/folder-21-request",
+ "folder-2/folder-22/folder-22-request",
+ "folder-2/folder-23/folder-23-request",
+ "folder-3/folder-3-request",
+ "folder-3/folder-31/folder-31-request",
+ "folder-3/folder-32/folder-32-request",
+ "folder-3/folder-33/folder-33-request",
+ ];
+
+ const normalizePath = (path: string) => path.replace(/\\/g, "/");
+
+ const extractRunningOrder = (stdout: string): string[] =>
+ [...stdout.matchAll(/Running:.*?\/(.*?)\r?\n/g)].map(
+ ([, path]) => normalizePath(path.replace(/\x1b\[\d+m/g, "")) // Remove ANSI codes and normalize paths
+ );
+
+ const args = `test ${getTestJsonFilePath(
+ "multiple-child-collections-auth-headers-coll.json",
+ "collection"
+ )}`;
+
+ const { stdout, error } = await runCLI(args);
+
+ // Verify the actual order matches the expected order
+ expect(extractRunningOrder(stdout)).toStrictEqual(expectedOrder);
+
+ // Ensure no errors occurred
+ expect(error).toBeNull();
+ });
+
describe("Test `hopp test <file_path_or_id> --env <file_path_or_id>` command:", () => {
describe("Supplied environment export file validations", () => {
describe("Argument parsing", () => {
@@ -235,12 +274,12 @@ describe("hopp test [options] <file_path_or_id>", { timeout: 100000 }, () => {
});
test("Successfully resolves values from the supplied environment export file", async () => {
- const TESTS_PATH = getTestJsonFilePath(
+ const COLL_PATH = getTestJsonFilePath(
"env-flag-tests-coll.json",
"collection"
);
const ENV_PATH = getTestJsonFilePath("env-flag-envs.json", "environment");
- const args = `test ${TESTS_PATH} --env ${ENV_PATH}`;
+ const args = `test ${COLL_PATH} --env ${ENV_PATH}`;
const { error } = await runCLI(args);
expect(error).toBeNull();
@@ -251,23 +290,23 @@ describe("hopp test [options] <file_path_or_id>", { timeout: 100000 }, () => {
"req-body-env-vars-coll.json",
"collection"
);
- const ENVS_PATH = getTestJsonFilePath(
+ const ENV_PATH = getTestJsonFilePath(
"req-body-env-vars-envs.json",
"environment"
);
- const args = `test ${COLL_PATH} --env ${ENVS_PATH}`;
+ const args = `test ${COLL_PATH} --env ${ENV_PATH}`;
const { error } = await runCLI(args);
expect(error).toBeNull();
});
test("Works with short `-e` flag", async () => {
- const TESTS_PATH = getTestJsonFilePath(
+ const COLL_PATH = getTestJsonFilePath(
"env-flag-tests-coll.json",
"collection"
);
const ENV_PATH = getTestJsonFilePath("env-flag-envs.json", "environment");
- const args = `test ${TESTS_PATH} -e ${ENV_PATH}`;
+ const args = `test ${COLL_PATH} -e ${ENV_PATH}`;
const { error } = await runCLI(args);
expect(error).toBeNull();
@@ -290,11 +329,8 @@ describe("hopp test [options] <file_path_or_id>", { timeout: 100000 }, () => {
"secret-envs-coll.json",
"collection"
);
- const ENVS_PATH = getTestJsonFilePath(
- "secret-envs.json",
- "environment"
- );
- const args = `test ${COLL_PATH} --env ${ENVS_PATH}`;
+ const ENV_PATH = getTestJsonFilePath("secret-envs.json", "environment");
+ const args = `test ${COLL_PATH} --env ${ENV_PATH}`;
const { error, stdout } = await runCLI(args, { env });
@@ -310,11 +346,11 @@ describe("hopp test [options] <file_path_or_id>", { timeout: 100000 }, () => {
"secret-envs-coll.json",
"collection"
);
- const ENVS_PATH = getTestJsonFilePath(
+ const ENV_PATH = getTestJsonFilePath(
"secret-supplied-values-envs.json",
"environment"
);
- const args = `test ${COLL_PATH} --env ${ENVS_PATH}`;
+ const args = `test ${COLL_PATH} --env ${ENV_PATH}`;
const { error, stdout } = await runCLI(args);
@@ -330,11 +366,11 @@ describe("hopp test [options] <file_path_or_id>", { timeout: 100000 }, () => {
"secret-envs-persistence-coll.json",
"collection"
);
- const ENVS_PATH = getTestJsonFilePath(
+ const ENV_PATH = getTestJsonFilePath(
"secret-supplied-values-envs.json",
"environment"
);
- const args = `test ${COLL_PATH} --env ${ENVS_PATH}`;
+ const args = `test ${COLL_PATH} --env ${ENV_PATH}`;
const { error, stdout } = await runCLI(args);
@@ -349,12 +385,12 @@ describe("hopp test [options] <file_path_or_id>", { timeout: 100000 }, () => {
"secret-envs-persistence-scripting-coll.json",
"collection"
);
- const ENVS_PATH = getTestJsonFilePath(
+ const ENV_PATH = getTestJsonFilePath(
"secret-envs-persistence-scripting-envs.json",
"environment"
);
- const args = `test ${COLL_PATH} --env ${ENVS_PATH}`;
+ const args = `test ${COLL_PATH} --env ${ENV_PATH}`;
const { error } = await runCLI(args);
expect(error).toBeNull();
@@ -372,12 +408,12 @@ describe("hopp test [options] <file_path_or_id>", { timeout: 100000 }, () => {
"request-vars-coll.json",
"collection"
);
- const ENVS_PATH = getTestJsonFilePath(
+ const ENV_PATH = getTestJsonFilePath(
"request-vars-envs.json",
"environment"
);
- const args = `test ${COLL_PATH} --env ${ENVS_PATH}`;
+ const args = `test ${COLL_PATH} --env ${ENV_PATH}`;
const { error, stdout } = await runCLI(args, { env });
expect(stdout).toContain(
@@ -399,12 +435,12 @@ describe("hopp test [options] <file_path_or_id>", { timeout: 100000 }, () => {
"aws-signature-auth-coll.json",
"collection"
);
- const ENVS_PATH = getTestJsonFilePath(
+ const ENV_PATH = getTestJsonFilePath(
"aws-signature-auth-envs.json",
"environment"
);
- const args = `test ${COLL_PATH} -e ${ENVS_PATH}`;
+ const args = `test ${COLL_PATH} -e ${ENV_PATH}`;
const { error } = await runCLI(args, { env });
expect(error).toBeNull();
@@ -417,14 +453,13 @@ describe("hopp test [options] <file_path_or_id>", { timeout: 100000 }, () => {
"digest-auth-success-coll.json",
"collection"
);
- const ENVS_PATH = getTestJsonFilePath(
+ const ENV_PATH = getTestJsonFilePath(
"digest-auth-envs.json",
"environment"
);
- const args = `test ${COLL_PATH} -e ${ENVS_PATH}`;
+ const args = `test ${COLL_PATH} -e ${ENV_PATH}`;
const { error } = await runCLI(args);
-
expect(error).toBeNull();
});
});
@@ -434,12 +469,12 @@ describe("hopp test [options] <file_path_or_id>", { timeout: 100000 }, () => {
"digest-auth-failure-coll.json",
"collection"
);
- const ENVS_PATH = getTestJsonFilePath(
+ const ENV_PATH = getTestJsonFilePath(
"digest-auth-envs.json",
"environment"
);
- const args = `test ${COLL_PATH} -e ${ENVS_PATH}`;
+ const args = `test ${COLL_PATH} -e ${ENV_PATH}`;
const { error } = await runCLI(args);
expect(error).toBeTruthy();
@@ -583,11 +618,11 @@ describe("hopp test [options] <file_path_or_id>", { timeout: 100000 }, () => {
});
test("Supports specifying collection file path along with environment ID", async () => {
- const TESTS_PATH = getTestJsonFilePath(
+ const COLL_PATH = getTestJsonFilePath(
"req-body-env-vars-coll.json",
"collection"
);
- const args = `test ${TESTS_PATH} --env ${REQ_BODY_ENV_VARS_ENVS_ID} --token ${PERSONAL_ACCESS_TOKEN} --server ${SERVER_URL}`;
+ const args = `test ${COLL_PATH} --env ${REQ_BODY_ENV_VARS_ENVS_ID} --token ${PERSONAL_ACCESS_TOKEN} --server ${SERVER_URL}`;
const { error } = await runCLI(args);
expect(error).toBeNull();
@@ -605,7 +640,7 @@ describe("hopp test [options] <file_path_or_id>", { timeout: 100000 }, () => {
});
test("Supports specifying both collection and environment file paths", async () => {
- const TESTS_PATH = getTestJsonFilePath(
+ const COLL_PATH = getTestJsonFilePath(
"req-body-env-vars-coll.json",
"collection"
);
@@ -613,7 +648,7 @@ describe("hopp test [options] <file_path_or_id>", { timeout: 100000 }, () => {
"req-body-env-vars-envs.json",
"environment"
);
- const args = `test ${TESTS_PATH} --env ${ENV_PATH} --token ${PERSONAL_ACCESS_TOKEN}`;
+ const args = `test ${COLL_PATH} --env ${ENV_PATH} --token ${PERSONAL_ACCESS_TOKEN}`;
const { error } = await runCLI(args);
expect(error).toBeNull();
@@ -644,9 +679,10 @@ describe("hopp test [options] <file_path_or_id>", { timeout: 100000 }, () => {
const COLL_PATH = getTestJsonFilePath("passes-coll.json", "collection");
- const invalidPath = process.platform === 'win32'
- ? 'Z:/non-existent-path/report.xml'
- : '/non-existent/report.xml';
+ const invalidPath =
+ process.platform === "win32"
+ ? "Z:/non-existent-path/report.xml"
+ : "/non-existent/report.xml";
const args = `test ${COLL_PATH} --reporter-junit ${invalidPath}`;
@@ -782,4 +818,139 @@ describe("hopp test [options] <file_path_or_id>", { timeout: 100000 }, () => {
expect(replaceDynamicValuesInStr(fileContents)).toMatchSnapshot();
});
});
+
+ describe("Test `hopp test <file> --iteration-count <no_of_iterations>` command:", () => {
+ const VALID_TEST_ARGS = `test ${getTestJsonFilePath("passes-coll.json", "collection")}`;
+
+ test("Errors with the code `INVALID_ARGUMENT` on not supplying an iteration count", async () => {
+ const args = `${VALID_TEST_ARGS} --iteration-count`;
+ const { stderr } = await runCLI(args);
+
+ const out = getErrorCode(stderr);
+ expect(out).toBe<HoppErrorCode>("INVALID_ARGUMENT");
+ });
+
+ test("Errors with the code `INVALID_ARGUMENT` on supplying an invalid iteration count", async () => {
+ const args = `${VALID_TEST_ARGS} --iteration-count NaN`;
+ const { stderr } = await runCLI(args);
+
+ const out = getErrorCode(stderr);
+ expect(out).toBe<HoppErrorCode>("INVALID_ARGUMENT");
+ });
+
+ test("Errors with the code `INVALID_ARGUMENT` on supplying an iteration count below `1`", async () => {
+ const args = `${VALID_TEST_ARGS} --iteration-count -5`;
+ const { stderr } = await runCLI(args);
+
+ const out = getErrorCode(stderr);
+ expect(out).toBe<HoppErrorCode>("INVALID_ARGUMENT");
+ });
+
+ test("Successfully executes all requests in the collection iteratively based on the specified iteration count", async () => {
+ const iterationCount = 3;
+ const args = `${VALID_TEST_ARGS} --iteration-count ${iterationCount}`;
+ const { error, stdout } = await runCLI(args);
+
+ // Logs iteration count in each pass
+ Array.from({ length: 3 }).forEach((_, idx) =>
+ expect(stdout).include(`Iteration: ${idx + 1}/${iterationCount}`)
+ );
+ expect(error).toBeNull();
+ });
+
+ test("Doesn't log iteration count if the value supplied is `1`", async () => {
+ const args = `${VALID_TEST_ARGS} --iteration-count 1`;
+ const { error, stdout } = await runCLI(args);
+
+ expect(stdout).not.include(`Iteration: 1/1`);
+
+ expect(error).toBeNull();
+ });
+ });
+
+ describe("Test `hopp test <file> --iteration-data <file_path>` command:", () => {
+ describe("Supplied data export file validations", () => {
+ const VALID_TEST_ARGS = `test ${getTestJsonFilePath("passes-coll.json", "collection")}`;
+
+ test("Errors with the code `INVALID_ARGUMENT` if no file is supplied", async () => {
+ const args = `${VALID_TEST_ARGS} --iteration-data`;
+ const { stderr } = await runCLI(args);
+
+ const out = getErrorCode(stderr);
+ expect(out).toBe<HoppErrorCode>("INVALID_ARGUMENT");
+ });
+
+ test("Errors with the code `INVALID_DATA_FILE_TYPE` if the supplied data file doesn't end with the `.csv` extension", async () => {
+ const args = `${VALID_TEST_ARGS} --iteration-data ${getTestJsonFilePath(
+ "notjson-coll.txt",
+ "collection"
+ )}`;
+ const { stderr } = await runCLI(args);
+
+ const out = getErrorCode(stderr);
+ expect(out).toBe<HoppErrorCode>("INVALID_DATA_FILE_TYPE");
+ });
+
+ test("Errors with the code `FILE_NOT_FOUND` if the supplied data export file doesn't exist", async () => {
+ const args = `${VALID_TEST_ARGS} --iteration-data notfound.csv`;
+ const { stderr } = await runCLI(args);
+
+ const out = getErrorCode(stderr);
+ expect(out).toBe<HoppErrorCode>("FILE_NOT_FOUND");
+ });
+ });
+
+ test("Prioritizes values from the supplied data export file over environment variables with relevant fallbacks for missing entries", async () => {
+ const COLL_PATH = getTestJsonFilePath(
+ "iteration-data-tests-coll.json",
+ "collection"
+ );
+ const ITERATION_DATA_PATH = getTestJsonFilePath(
+ "iteration-data-export.csv",
+ "environment"
+ );
+ const ENV_PATH = getTestJsonFilePath(
+ "iteration-data-envs.json",
+ "environment"
+ );
+ const args = `test ${COLL_PATH} --iteration-data ${ITERATION_DATA_PATH} -e ${ENV_PATH}`;
+
+ const { error, stdout } = await runCLI(args);
+
+ const iterationCount = 3;
+
+ // Even though iteration count is not supplied, it will be inferred from the iteration data size
+ Array.from({ length: iterationCount }).forEach((_, idx) =>
+ expect(stdout).include(`Iteration: ${idx + 1}/${iterationCount}`)
+ );
+
+ expect(error).toBeNull();
+ });
+
+ test("Iteration count takes priority if supplied instead of inferring from the iteration data size", async () => {
+ const COLL_PATH = getTestJsonFilePath(
+ "iteration-data-tests-coll.json",
+ "collection"
+ );
+ const ITERATION_DATA_PATH = getTestJsonFilePath(
+ "iteration-data-export.csv",
+ "environment"
+ );
+ const ENV_PATH = getTestJsonFilePath(
+ "iteration-data-envs.json",
+ "environment"
+ );
+
+ const iterationCount = 5;
+ const args = `test ${COLL_PATH} --iteration-data ${ITERATION_DATA_PATH} -e ${ENV_PATH} --iteration-count ${iterationCount}`;
+
+ const { error, stdout } = await runCLI(args);
+
+ Array.from({ length: iterationCount }).forEach((_, idx) =>
+ expect(stdout).include(`Iteration: ${idx + 1}/${iterationCount}`)
+ );
+
+ expect(error).toBeNull();
+ });
+ });
});
diff --git a/packages/hoppscotch-cli/src/__tests__/e2e/fixtures/collections/iteration-data-tests-coll.json b/packages/hoppscotch-cli/src/__tests__/e2e/fixtures/collections/iteration-data-tests-coll.json
new file mode 100644
index 000000000..d70aa8453
--- /dev/null
+++ b/packages/hoppscotch-cli/src/__tests__/e2e/fixtures/collections/iteration-data-tests-coll.json
@@ -0,0 +1,23 @@
+{
+ "v": 1,
+ "name": "iteration-data-tests-coll",
+ "folders": [],
+ "requests": [
+ {
+ "v": "3",
+ "endpoint": "<<URL>>",
+ "name": "test1",
+ "params": [],
+ "headers": [],
+ "method": "POST",
+ "auth": { "authType": "none", "authActive": true },
+ "preRequestScript": "",
+ "testScript": "// Iteration data is prioritised over environment variables \n const { data, headers } = pw.response.body;\n pw.expect(headers['host']).toBe('echo.hoppscotch.io')\n // Falls back to environment variables for missing entries in data export\n pw.expect(data).toInclude('overriden-body-key-at-environment')\n pw.expect(data).toInclude('body_value')",
+ "body": {
+ "contentType": "application/json",
+ "body": "{\n \"<<BODY_KEY>>\":\"<<BODY_VALUE>>\"\n}"
+ },
+ "requestVariables": []
+ }
+ ]
+}
diff --git a/packages/hoppscotch-cli/src/__tests__/e2e/fixtures/environments/iteration-data-envs.json b/packages/hoppscotch-cli/src/__tests__/e2e/fixtures/environments/iteration-data-envs.json
new file mode 100644
index 000000000..bea639355
--- /dev/null
+++ b/packages/hoppscotch-cli/src/__tests__/e2e/fixtures/environments/iteration-data-envs.json
@@ -0,0 +1,18 @@
+{
+ "v": 0,
+ "name": "Iteration data environments",
+ "variables": [
+ {
+ "key": "URL",
+ "value": "https://httpbin.org/get"
+ },
+ {
+ "key": "BODY_KEY",
+ "value": "overriden-body-key-at-environment"
+ },
+ {
+ "key": "BODY_VALUE",
+ "value": "overriden-body-value-at-environment"
+ }
+ ]
+}
diff --git a/packages/hoppscotch-cli/src/__tests__/e2e/fixtures/environments/iteration-data-export.csv b/packages/hoppscotch-cli/src/__tests__/e2e/fixtures/environments/iteration-data-export.csv
new file mode 100644
index 000000000..5caf994ed
--- /dev/null
+++ b/packages/hoppscotch-cli/src/__tests__/e2e/fixtures/environments/iteration-data-export.csv
@@ -0,0 +1,4 @@
+URL,BODY_KEY,BODY_VALUE
+https://echo.hoppscotch.io/1,,body_value1
+https://echo.hoppscotch.io/2,,body_value2
+https://echo.hoppscotch.io/3,,body_value3
diff --git a/packages/hoppscotch-cli/src/commands/test.ts b/packages/hoppscotch-cli/src/commands/test.ts
index 6143f5baf..0291fa31f 100644
--- a/packages/hoppscotch-cli/src/commands/test.ts
+++ b/packages/hoppscotch-cli/src/commands/test.ts
@@ -1,7 +1,14 @@
+import fs from "fs";
+import { isSafeInteger } from "lodash-es";
+import Papa from "papaparse";
+import path from "path";
+
import { handleError } from "../handlers/error";
import { parseDelayOption } from "../options/test/delay";
import { parseEnvsData } from "../options/test/env";
+import { IterationDataItem } from "../types/collections";
import { TestCmdEnvironmentOptions, TestCmdOptions } from "../types/commands";
+import { error } from "../types/errors";
import { HoppEnvs } from "../types/request";
import { isHoppCLIError } from "../utils/checks";
import {
@@ -13,16 +20,79 @@ import { parseCollectionData } from "../utils/mutators";
export const test = (pathOrId: string, options: TestCmdOptions) => async () => {
try {
- const delay = options.delay ? parseDelayOption(options.delay) : 0;
+ const { delay, env, iterationCount, iterationData, reporterJunit } =
+ options;
+
+ if (
+ iterationCount !== undefined &&
+ (iterationCount < 1 || !isSafeInteger(iterationCount))
+ ) {
+ throw error({
+ code: "INVALID_ARGUMENT",
+ data: "The value must be a positive integer",
+ });
+ }
+
+ const resolvedDelay = delay ? parseDelayOption(delay) : 0;
- const envs = options.env
+ const envs = env
? await parseEnvsData(options as TestCmdEnvironmentOptions)
: <HoppEnvs>{ global: [], selected: [] };
+ let parsedIterationData: unknown[] | null = null;
+ let transformedIterationData: IterationDataItem[][] | undefined;
+
const collections = await parseCollectionData(pathOrId, options);
- const report = await collectionsRunner({ collections, envs, delay });
- const hasSucceeded = collectionsRunnerResult(report, options.reporterJunit);
+ if (iterationData) {
+ // Check file existence
+ if (!fs.existsSync(iterationData)) {
+ throw error({ code: "FILE_NOT_FOUND", path: iterationData });
+ }
+
+ // Check the file extension
+ if (path.extname(iterationData) !== ".csv") {
+ throw error({
+ code: "INVALID_DATA_FILE_TYPE",
+ data: iterationData,
+ });
+ }
+
+ const csvData = fs.readFileSync(iterationData, "utf8");
+ parsedIterationData = Papa.parse(csvData, { header: true }).data;
+
+ // Transform data into the desired format
+ transformedIterationData = parsedIterationData
+ .map((item) => {
+ const iterationDataItem = item as Record<string, unknown>;
+ const keys = Object.keys(iterationDataItem);
+
+ return (
+ keys
+ // Ignore keys with empty string values
+ .filter((key) => iterationDataItem[key] !== "")
+ .map(
+ (key) =>
+ <IterationDataItem>{
+ key: key,
+ value: iterationDataItem[key],
+ secret: false,
+ }
+ )
+ );
+ })
+ // Ignore items that result in an empty array
+ .filter((item) => item.length > 0);
+ }
+
+ const report = await collectionsRunner({
+ collections,
+ envs,
+ delay: resolvedDelay,
+ iterationData: transformedIterationData,
+ iterationCount,
+ });
+ const hasSucceeded = collectionsRunnerResult(report, reporterJunit);
collectionsRunnerExit(hasSucceeded);
} catch (e) {
diff --git a/packages/hoppscotch-cli/src/handlers/error.ts b/packages/hoppscotch-cli/src/handlers/error.ts
index 65918136e..72be3accf 100644
--- a/packages/hoppscotch-cli/src/handlers/error.ts
+++ b/packages/hoppscotch-cli/src/handlers/error.ts
@@ -65,6 +65,9 @@ export const handleError = <T extends HoppErrorCode>(error: HoppError<T>) => {
case "INVALID_FILE_TYPE":
ERROR_MSG = `Please provide file of extension type .json: ${error.data}`;
break;
+ case "INVALID_DATA_FILE_TYPE":
+ ERROR_MSG = `Please provide file of extension type .csv: ${error.data}`;
+ break;
case "REQUEST_ERROR":
case "TEST_SCRIPT_ERROR":
case "PRE_REQUEST_SCRIPT_ERROR":
diff --git a/packages/hoppscotch-cli/src/index.ts b/packages/hoppscotch-cli/src/index.ts
index ebd20c01f..7b92a47a0 100644
--- a/packages/hoppscotch-cli/src/index.ts
+++ b/packages/hoppscotch-cli/src/index.ts
@@ -69,6 +69,15 @@ program
"--reporter-junit [path]",
"generate JUnit report optionally specifying the path"
)
+ .option(
+ "--iteration-count <no_of_iterations>",
+ "number of iterations to run the test",
+ parseInt
+ )
+ .option(
+ "--iteration-data <file_path>",
+ "path to a CSV file for data-driven testing"
+ )
.allowExcessArguments(false)
.allowUnknownOption(false)
.description("running hoppscotch collection.json file")
diff --git a/packages/hoppscotch-cli/src/types/collections.ts b/packages/hoppscotch-cli/src/types/collections.ts
index f5483e2d4..08c6ccb75 100644
--- a/packages/hoppscotch-cli/src/types/collections.ts
+++ b/packages/hoppscotch-cli/src/types/collections.ts
@@ -1,10 +1,15 @@
import { HoppCollection } from "@hoppscotch/data";
-import { HoppEnvs } from "./request";
+import { HoppEnvPair, HoppEnvs } from "./request";
export type CollectionRunnerParam = {
collections: HoppCollection[];
envs: HoppEnvs;
delay?: number;
+ iterationData?: IterationDataItem[][];
+ iterationCount?: number;
};
export type HoppCollectionFileExt = "json";
+
+// Indicates the shape each iteration data entry gets transformed into
+export type IterationDataItem = Extract<HoppEnvPair, { value: string }>;
diff --git a/packages/hoppscotch-cli/src/types/commands.ts b/packages/hoppscotch-cli/src/types/commands.ts
index 71fd51f6b..e353cb7d3 100644
--- a/packages/hoppscotch-cli/src/types/commands.ts
+++ b/packages/hoppscotch-cli/src/types/commands.ts
@@ -4,6 +4,8 @@ export type TestCmdOptions = {
token?: string;
server?: string;
reporterJunit?: string;
+ iterationCount?: number;
+ iterationData?: string;
};
// Consumed in the collection `file_path_or_id` argument action handler
diff --git a/packages/hoppscotch-cli/src/types/errors.ts b/packages/hoppscotch-cli/src/types/errors.ts
index 354b2da4f..6122ff797 100644
--- a/packages/hoppscotch-cli/src/types/errors.ts
+++ b/packages/hoppscotch-cli/src/types/errors.ts
@@ -26,6 +26,7 @@ type HoppErrors = {
MALFORMED_ENV_FILE: HoppErrorPath & HoppErrorData;
BULK_ENV_FILE: HoppErrorPath & HoppErrorData;
INVALID_FILE_TYPE: HoppErrorData;
+ INVALID_DATA_FILE_TYPE: HoppErrorData;
TOKEN_EXPIRED: HoppErrorData;
TOKEN_INVALID: HoppErrorData;
INVALID_ID: HoppErrorData;
diff --git a/packages/hoppscotch-cli/src/types/request.ts b/packages/hoppscotch-cli/src/types/request.ts
index 21dffbd1c..7aa90d0a9 100644
--- a/packages/hoppscotch-cli/src/types/request.ts
+++ b/packages/hoppscotch-cli/src/types/request.ts
@@ -18,7 +18,7 @@ export type HoppEnvs = {
selected: HoppEnvPair[];
};
-export type CollectionStack = {
+export type CollectionQueue = {
path: string;
collection: HoppCollection;
};
diff --git a/packages/hoppscotch-cli/src/utils/collections.ts b/packages/hoppscotch-cli/src/utils/collections.ts
index 9a6ef5ef1..5c81d5d7d 100644
--- a/packages/hoppscotch-cli/src/utils/collections.ts
+++ b/packages/hoppscotch-cli/src/utils/collections.ts
@@ -7,7 +7,7 @@ import { round } from "lodash-es";
import { CollectionRunnerParam } from "../types/collections";
import {
- CollectionStack,
+ CollectionQueue,
HoppEnvs,
ProcessRequestParams,
RequestReport,
@@ -35,7 +35,7 @@ import {
} from "./request";
import { getTestMetrics } from "./test";
-const { WARN, FAIL } = exceptionColors;
+const { WARN, FAIL, INFO } = exceptionColors;
/**
* Processes each requests within collections to prints details of subsequent requests,
@@ -43,93 +43,134 @@ const { WARN, FAIL } = exceptionColors;
* @param param Data of hopp-collection with hopp-requests, envs to be processed.
* @returns List of report for each processed request.
*/
+
export const collectionsRunner = async (
param: CollectionRunnerParam
): Promise<RequestReport[]> => {
- const envs: HoppEnvs = param.envs;
- const delay = param.delay ?? 0;
- const requestsReport: RequestReport[] = [];
- const collectionStack: CollectionStack[] = getCollectionStack(
- param.collections
- );
+ const { collections, envs, delay, iterationCount, iterationData } = param;
- while (collectionStack.length) {
- // Pop out top-most collection from stack to be processed.
- const { collection, path } = <CollectionStack>collectionStack.pop();
+ const resolvedDelay = delay ?? 0;
- // Processing each request in collection
- for (const request of collection.requests) {
- const _request = preProcessRequest(
- request as HoppRESTRequest,
- collection
- );
- const requestPath = `${path}/${_request.name}`;
- const processRequestParams: ProcessRequestParams = {
- path: requestPath,
- request: _request,
- envs,
- delay,
- };
+ const requestsReport: RequestReport[] = [];
+ const collectionQueue = getCollectionQueue(collections);
- // Request processing initiated message.
- log(WARN(`\nRunning: ${chalk.bold(requestPath)}`));
+ // If iteration count is not supplied, it should be based on the size of iteration data if in scope
+ const resolvedIterationCount = iterationCount ?? iterationData?.length ?? 1;
- // Processing current request.
- const result = await processRequest(processRequestParams)();
+ const originalSelectedEnvs = [...envs.selected];
- // Updating global & selected envs with new envs from processed-request output.
- const { global, selected } = result.envs;
- envs.global = global;
- envs.selected = selected;
+ for (let count = 0; count < resolvedIterationCount; count++) {
+ if (resolvedIterationCount > 1) {
+ log(INFO(`\nIteration: ${count + 1}/${resolvedIterationCount}`));
+ }
- // Storing current request's report.
- const requestReport = result.report;
- requestsReport.push(requestReport);
+ // Reset `envs` to the original value at the start of each iteration
+ envs.selected = [...originalSelectedEnvs];
+
+ if (iterationData) {
+ // Ensure last item is picked if the iteration count exceeds size of the iteration data
+ const iterationDataItem =
+ iterationData[Math.min(count, iterationData.length - 1)];
+
+ // Ensure iteration data takes priority over supplied environment variables
+ envs.selected = envs.selected
+ .filter(
+ (envPair) =>
+ !iterationDataItem.some((dataPair) => dataPair.key === envPair.key)
+ )
+ .concat(iterationDataItem);
}
- // Pushing remaining folders realted collection to stack.
- for (const folder of collection.folders) {
- const updatedFolder: HoppCollection = { ...folder };
-
- if (updatedFolder.auth?.authType === "inherit") {
- updatedFolder.auth = collection.auth;
- }
-
- if (collection.headers?.length) {
- // Filter out header entries present in the parent collection under the same name
- // This ensures the folder headers take precedence over the collection headers
- const filteredHeaders = collection.headers.filter(
- (collectionHeaderEntries) => {
- return !updatedFolder.headers.some(
- (folderHeaderEntries) =>
- folderHeaderEntries.key === collectionHeaderEntries.key
- );
- }
- );
- updatedFolder.headers.push(...filteredHeaders);
- }
-
- collectionStack.push({
- path: `${path}/${updatedFolder.name}`,
- collection: updatedFolder,
- });
+ for (const { collection, path } of collectionQueue) {
+ await processCollection(
+ collection,
+ path,
+ envs,
+ resolvedDelay,
+ requestsReport
+ );
}
}
return requestsReport;
};
+const processCollection = async (
+ collection: HoppCollection,
+ path: string,
+ envs: HoppEnvs,
+ delay: number,
+ requestsReport: RequestReport[]
+) => {
+ // Process each request in the collection
+ for (const request of collection.requests) {
+ const _request = preProcessRequest(request as HoppRESTRequest, collection);
+ const requestPath = `${path}/${_request.name}`;
+ const processRequestParams: ProcessRequestParams = {
+ path: requestPath,
+ request: _request,
+ envs,
+ delay,
+ };
+
+ // Request processing initiated message.
+ log(WARN(`\nRunning: ${chalk.bold(requestPath)}`));
+
+ // Processing current request.
+ const result = await processRequest(processRequestParams)();
+
+ // Updating global & selected envs with new envs from processed-request output.
+ const { global, selected } = result.envs;
+ envs.global = global;
+ envs.selected = selected;
+
+ // Storing current request's report.
+ const requestReport = result.report;
+ requestsReport.push(requestReport);
+ }
+
+ // Process each folder in the collection
+ for (const folder of collection.folders) {
+ const updatedFolder: HoppCollection = { ...folder };
+
+ if (updatedFolder.auth?.authType === "inherit") {
+ updatedFolder.auth = collection.auth;
+ }
+
+ if (collection.headers?.length) {
+ // Filter out header entries present in the parent collection under the same name
+ // This ensures the folder headers take precedence over the collection headers
+ const filteredHeaders = collection.headers.filter(
+ (collectionHeaderEntries) => {
+ return !updatedFolder.headers.some(
+ (folderHeaderEntries) =>
+ folderHeaderEntries.key === collectionHeaderEntries.key
+ );
+ }
+ );
+ updatedFolder.headers.push(...filteredHeaders);
+ }
+
+ await processCollection(
+ updatedFolder,
+ `${path}/${updatedFolder.name}`,
+ envs,
+ delay,
+ requestsReport
+ );
+ }
+};
/**
* Transforms collections to generate collection-stack which describes each collection's
* path within collection & the collection itself.
* @param collections Hopp-collection objects to be mapped to collection-stack type.
* @returns Mapped collections to collection-stack.
*/
-const getCollectionStack = (collections: HoppCollection[]): CollectionStack[] =>
+const getCollectionQueue = (collections: HoppCollection[]): CollectionQueue[] =>
pipe(
collections,
A.map(
- (collection) => <CollectionStack>{ collection, path: collection.name }
+ (collection) => <CollectionQueue>{ collection, path: collection.name }
)
);
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index c0f6dc292..f1b708ae8 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -395,6 +395,9 @@ importers:
lodash-es:
specifier: 4.17.21
version: 4.17.21
+ papaparse:
+ specifier: 5.4.1
+ version: 5.4.1
qs:
specifier: 6.13.0
version: 6.13.0
@@ -420,6 +423,9 @@ importers:
'@types/lodash-es':
specifier: 4.17.12
version: 4.17.12
+ '@types/papaparse':
+ specifier: 5.3.14
+ version: 5.3.14
'@types/qs':
specifier: 6.9.16
version: 6.9.16
@@ -5148,6 +5154,9 @@ packages:
resolution: {integrity: sha512-xOEii1m7jw7mIKjufDkolpz7VlyqptUmm/YFPtLJCybrPCuLhN+WYgNpulQ/CXo7wtEW7x4uGon2v89+6g/pcA==}
+ '@types/[email protected]':
+ resolution: {integrity: sha512-LxJ4iEFcpqc6METwp9f6BV6VVc43m6MfH0VqFosHvrUgfXiFe6ww7R3itkOQ+TCK6Y+Iv/+RnnvtRZnkc5Kc9g==}
+
resolution: {integrity: sha512-/nMfiPK2E6GKttwBzwj0Wjaot8eHrM57hnWxu52o6becr5/kXlH/4yE2v2rh234WGvSgEEzIII02Nc5oC5xEHA==}
@@ -9691,6 +9700,9 @@ packages:
resolution: {integrity: sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==}
+ resolution: {integrity: sha512-HipMsgJkZu8br23pW15uvo6sib6wne/4woLZPlFf3rpDyMe9ywEXUsuD7+6K9PRkJlVT51j/sCOYDKGGS3ZJrw==}
+
resolution: {integrity: sha512-RXlj7zCYokReqWpOPH9oYivUzLYZ5vAPIfEmCTNViosC78F8F0H9y7T7gG2M39ymgutxF5gcFEsyZQSph9Bp3A==}
@@ -16956,6 +16968,10 @@ snapshots:
'@types/[email protected]': {}
+ '@types/[email protected]':
+ dependencies:
+ '@types/node': 22.7.6
+
dependencies:
'@types/express': 5.0.0
@@ -23374,6 +23390,8 @@ snapshots:
+
dependencies:
dot-case: 3.0.4