Open-Source API Development Ecosystem • https://hoppscotch.io • Offline, On-Prem & Cloud • Web, Desktop & CLI • Open-Source Alternative to Postman, Insomnia
TypeScript apiapi-clientapi-restapi-testingdeveloper-toolsgraphqlhttphttp-clientpwarestrest-apispatestingtesting-toolstoolsvuevuejswebsocket
fix(common): preserve JSON bodies in curl parser (#5768)
Jiawen Geng63d05dd5fc67
diff --git a/packages/hoppscotch-common/src/helpers/curl/__tests__/curlparser.spec.js b/packages/hoppscotch-common/src/helpers/curl/__tests__/curlparser.spec.jsindex a8ffa7455..601ab82cf 100644--- a/packages/hoppscotch-common/src/helpers/curl/__tests__/curlparser.spec.js+++ b/packages/hoppscotch-common/src/helpers/curl/__tests__/curlparser.spec.js@@ -1060,6 +1060,105 @@ data2: {"type":"test2","typeId":"123"}`,]describe("Parse curl command to Hopp REST Request", () => {+ test("parses json body with semicolon-only headers", () => {+ const command = String.raw`curl 'https://echo.hoppscotch.io/api/process/insert' \+ -H 'Authorization-OAuth2;' \+ -H 'Authorization-OAuth2-Client;' \+ -H 'Authorization-OAuth2-Refresh;' \+ -H 'Content-Type: application/json;charset=UTF-8' \+ --data-raw '{"insertProcessDto":{"name":"测hi退回"},"formSaveDTO":{"formProps":"{\"list\":[]}"}}' \+ --insecure`++ const actual = parseCurlToHoppRESTReq(command)++ expect(actual.method).toBe("POST")+ expect(actual.endpoint).toBe(+ "https://echo.hoppscotch.io/api/process/insert"+ )+ expect(actual.body.contentType).toBe("application/json")++ const parsedBody = JSON.parse(actual.body.body)+ expect(parsedBody.insertProcessDto.name).toBe("测hi退回")+ expect(JSON.parse(parsedBody.formSaveDTO.formProps)).toEqual({ list: [] })+ })++ test("parses json body containing escaped XML", () => {+ const command = String.raw`curl 'https://echo.hoppscotch.io/api/process/insert' \+ -H 'Authorization-OAuth2;' \+ -H 'Authorization-OAuth2-Client;' \+ -H 'Authorization-OAuth2-Refresh;' \+ -H 'Content-Type: application/json;charset=UTF-8' \+ --data-raw '{"insertProcessDto":{"bpmnXmlString":"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<bpmn2:definitions xmlns:bpmn2=\"http://www.omg.org/spec/BPMN/20100524/MODEL\"></bpmn2:definitions>"},"formSaveDTO":{"formProps":"{\"list\":[]}"}}' \+ --insecure`++ const actual = parseCurlToHoppRESTReq(command)++ expect(actual.method).toBe("POST")+ expect(actual.endpoint).toBe(+ "https://echo.hoppscotch.io/api/process/insert"+ )+ expect(actual.body.contentType).toBe("application/json")++ const parsedBody = JSON.parse(actual.body.body)+ expect(parsedBody.insertProcessDto.bpmnXmlString).toContain("<?xml")+ expect(parsedBody.insertProcessDto.bpmnXmlString).toContain(+ "bpmn2:definitions"+ )+ expect(JSON.parse(parsedBody.formSaveDTO.formProps)).toEqual({ list: [] })+ })++ test("does not drop JSON body for -d with many headers", () => {+ const command = `curl 'https://echo.hoppscotch.io/api/chat/completions' -d '{"response_format":{"type":"json_object"},"messages":[{"content":"Translate array of texts from en into zh and return JSON result with the same array length, do not add any additional text, and do not return code blocks, such as: {\\"translations\\": [\\"translation of input text 1\\", ...]}","role":"system"},{"content":"[\\"Epic Games CEO Tim Sweeney argues banning Twitter over its ability to AI-generate pornographic images of minors is just '''gatekeepers''' attempting to '''censor all of their political opponents'''\\"]","role":"user"}],"model":"gpt-translate","temperature":0.30000001192092896}' -H ':authority: echo.hoppscotch.io' -H 'accept: */*' -H 'content-type: application/json' -H 'accept-language: en-US;q=1.0, zh-Hans-US;q=0.9' -H 'authorization: Bearer <redacted>' -H 'accept-encoding: br;q=1.0, gzip;q=0.9, deflate;q=0.8' -H 'user-agent: TranslationExtension/1.14.5 (org.lesslab.relingo.TranslationExtension; build:104; iOS 26.2.0) Alamofire/5.10.2' -H 'priority: u=3, i' --compressed`++ const actual = parseCurlToHoppRESTReq(command)++ expect(actual.method).toBe("POST")+ expect(actual.endpoint).toBe(+ "https://echo.hoppscotch.io/api/chat/completions"+ )+ expect(actual.body.contentType).toBe("application/json")++ const parsedBody = JSON.parse(actual.body.body)+ expect(parsedBody.model).toBe("gpt-translate")+ expect(parsedBody.temperature).toBe(0.30000001192092896)+ expect(parsedBody.response_format).toEqual({ type: "json_object" })+ expect(parsedBody.messages).toHaveLength(2)+ expect(parsedBody.messages[0].role).toBe("system")+ expect(parsedBody.messages[0].content).toBe(+ `Translate array of texts from en into zh and return JSON result with the same array length, do not add any additional text, and do not return code blocks, such as: {"translations": ["translation of input text 1", ...]}`+ )+ expect(parsedBody.messages[1].role).toBe("user")+ expect(parsedBody.messages[1].content).toBe(+ `["Epic Games CEO Tim Sweeney argues banning Twitter over its ability to AI-generate pornographic images of minors is just '''gatekeepers''' attempting to '''censor all of their political opponents'''"]`+ )+ })++ test("preserves -d POST without -G as form-urlencoded", () => {+ const command = `curl -X POST 'https://example.com/submit' -d 'name=alice&role=admin'`++ const actual = parseCurlToHoppRESTReq(command)++ expect(actual.method).toBe("POST")+ expect(actual.endpoint).toBe("https://example.com/submit")+ expect(actual.body.contentType).toBe("application/x-www-form-urlencoded")+ expect(actual.body.body).toBe("name: alice\nrole: admin")+ })++ test("does not intercept -d inside a quoted header value", () => {+ const command = `curl 'https://example.com/api' -H 'X-Custom: -d {"fake":1}' -d '{"real":true}'`++ const actual = parseCurlToHoppRESTReq(command)++ expect(actual.method).toBe("POST")+ expect(actual.endpoint).toBe("https://example.com/api")+ expect(actual.body.contentType).toBe("application/json")+ expect(JSON.parse(actual.body.body)).toEqual({ real: true })++ const customHeader = actual.headers.find((h) => h.key === "X-Custom")+ expect(customHeader).toBeDefined()+ expect(customHeader.value).toBe(`-d {"fake":1}`)+ })+for (const [i, { command, response }] of samples.entries()) {test(`for sample #${i + 1}:\n\n${command}`, () => {const actual = parseCurlToHoppRESTReq(command)diff --git a/packages/hoppscotch-common/src/helpers/curl/curlparser.ts b/packages/hoppscotch-common/src/helpers/curl/curlparser.tsindex f3d06f472..811f560fd 100644--- a/packages/hoppscotch-common/src/helpers/curl/curlparser.ts+++ b/packages/hoppscotch-common/src/helpers/curl/curlparser.ts@@ -22,7 +22,11 @@ import {import { getDefaultRESTRequest } from "../rest/default"import { getBody, getFArgumentMultipartData } from "./sub_helpers/body"import { getMethod } from "./sub_helpers/method"-import { preProcessCurlCommand } from "./sub_helpers/preproc"+import {+ preProcessCurlCommand,+ replaceJSONDataArgsWithPlaceholders,+ restoreJSONDataArgsFromPlaceholders,+} from "./sub_helpers/preproc"import { getQueries } from "./sub_helpers/queries"import { concatParams, getURLObject } from "./sub_helpers/url"import { HOPP_ENVIRONMENT_REGEX } from "../environment-regex"@@ -42,7 +46,13 @@ export const parseCurlCommand = (curlCommand: string) => {curlCommand = preProcessCurlCommand(curlCommand)- const args: parser.Arguments = parser(curlCommand)+ const { curlCommand: sanitizedCurlCommand, extractedJSONData } =+ replaceJSONDataArgsWithPlaceholders(curlCommand)++ const args: parser.Arguments = restoreJSONDataArgsFromPlaceholders(+ parser(sanitizedCurlCommand),+ extractedJSONData+ )const parsedArguments = pipe(args,@@ -116,10 +126,30 @@ export const parseCurlCommand = (curlCommand: string) => {Array.from(urlObject.searchParams.entries()))- const stringToPair = flow(- decodeURIComponent,- (pair) => <[string, string]>pair.split("=", 2)- )+ const safeDecodeURIComponent = (value: string) => {+ if (!value.includes("%")) return value+ if (/%(?![0-9A-Fa-f]{2})/.test(value)) return value++ try {+ return decodeURIComponent(value)+ } catch {+ return value+ }+ }++ const stringToPair = (pair: string): [string, string] => {+ const decodedPair = safeDecodeURIComponent(pair)+ const eqIndex = decodedPair.indexOf("=")++ if (eqIndex === -1) return [decodedPair, ""]++ return [decodedPair.slice(0, eqIndex), decodedPair.slice(eqIndex + 1)]+ }++ const shouldParseDataAsQueries = objHasProperty(+ "G",+ "boolean"+ )(parsedArguments)const pairs = pipe(rawData,O.fromPredicate(Array.isArray),@@ -127,14 +157,14 @@ export const parseCurlCommand = (curlCommand: string) => {O.alt(() =>pipe(rawData,- O.fromPredicate((s) => s.length > 0),- O.map(() => [stringToPair(rawData as string)])+ O.fromPredicate((s) => shouldParseDataAsQueries && s.length > 0),+ O.map((s) => [stringToPair(s)]))),O.getOrElseW(() => undefined))- if (objHasProperty("G", "boolean")(parsedArguments) && !!pairs) {+ if (shouldParseDataAsQueries && !!pairs) {const newQueries = getQueries(pairs)queries = [...queries, ...newQueries.queries]danglingParams = [...danglingParams, ...newQueries.danglingParams]diff --git a/packages/hoppscotch-common/src/helpers/curl/sub_helpers/preproc.ts b/packages/hoppscotch-common/src/helpers/curl/sub_helpers/preproc.tsindex e79d8a164..bed95e1c4 100644--- a/packages/hoppscotch-common/src/helpers/curl/sub_helpers/preproc.ts+++ b/packages/hoppscotch-common/src/helpers/curl/sub_helpers/preproc.ts@@ -68,3 +68,233 @@ export const preProcessCurlCommand = (curlCommand: string) =>O.map(flow(paperCuts, replaceLongOptions, prescreenXArgs)),O.getOrElse(() => ""))++const JSON_DATA_PLACEHOLDER_PREFIX = "__HOPP_CURL_JSON_DATA_"+const JSON_DATA_PLACEHOLDER_SUFFIX = "__"++const isWhitespace = (ch: string | undefined) =>+ ch === " " || ch === "\t" || ch === "\n" || ch === "\r"++const scanJSONValueEnd = (input: string, startIndex: number) => {+ const start = input[startIndex]+ if (start !== "{" && start !== "[") return null++ let depth = 0+ let inString = false+ let escaped = false++ for (let i = startIndex; i < input.length; i++) {+ const ch = input[i]++ if (inString) {+ if (escaped) {+ escaped = false+ continue+ }++ if (ch === "\\") {+ escaped = true+ continue+ }++ if (ch === '"') {+ inString = false+ }++ continue+ }++ if (ch === '"') {+ inString = true+ continue+ }++ if (ch === "{" || ch === "[") {+ depth+++ continue+ }++ if (ch === "}" || ch === "]") {+ depth--+ if (depth === 0) return i+ }+ }++ return null+}++const getJSONDataPlaceholder = (index: number) =>+ `${JSON_DATA_PLACEHOLDER_PREFIX}${index}${JSON_DATA_PLACEHOLDER_SUFFIX}`++const isJSONDataPlaceholder = (value: unknown): value is string =>+ typeof value === "string" &&+ value.startsWith(JSON_DATA_PLACEHOLDER_PREFIX) &&+ value.endsWith(JSON_DATA_PLACEHOLDER_SUFFIX)++const placeholderIndex = (value: string) => {+ const match = value.match(+ new RegExp(+ `^${JSON_DATA_PLACEHOLDER_PREFIX}(\\d+)${JSON_DATA_PLACEHOLDER_SUFFIX}$`+ )+ )+ return match ? Number(match[1]) : null+}++/**+ * Some exported curl commands wrap the request body in single quotes but+ * contain unescaped single quotes (e.g. `'''text'''`), which breaks the+ * argument tokenizer. Replace JSON bodies passed via `-d`/`--data*` with+ * placeholders before yargs-parser sees them, then restore them after.+ */+export const replaceJSONDataArgsWithPlaceholders = (curlCommand: string) => {+ const dataFlags = [+ "--data-raw",+ "--data-binary",+ "--data-ascii",+ "--data",+ "-d",+ ]+ const extractedJSONData: string[] = []++ let output = ""+ let i = 0+ let shellQuote: '"' | "'" | null = null++ while (i < curlCommand.length) {+ const ch = curlCommand[i]++ // Inside a top-level shell-quoted argument, copy verbatim and watch+ // for the close. Skip flag detection so an embedded `-d`/`--data`+ // inside e.g. a header value doesn't get intercepted as a data flag.+ if (shellQuote !== null) {+ if (+ ch === shellQuote &&+ (shellQuote === "'" || curlCommand[i - 1] !== "\\")+ ) {+ shellQuote = null+ }+ output += ch+ i+++ continue+ }++ const isBoundaryBefore = i === 0 || isWhitespace(curlCommand[i - 1])+ if (!isBoundaryBefore) {+ if (ch === '"' || ch === "'") shellQuote = ch+ output += ch+ i+++ continue+ }++ const flag = dataFlags.find((f) => curlCommand.startsWith(f, i))++ if (!flag) {+ if (ch === '"' || ch === "'") shellQuote = ch+ output += ch+ i+++ continue+ }++ const afterFlag = curlCommand[i + flag.length]+ const isBoundaryAfter =+ afterFlag === undefined ||+ isWhitespace(afterFlag) ||+ afterFlag === "=" ||+ afterFlag === "'" ||+ afterFlag === '"'++ if (!isBoundaryAfter) {+ output += curlCommand[i]+ i+++ continue+ }++ // Try to parse the value part (supports `-d VALUE`, `-d'VALUE'`, `--data=VALUE`, etc.)+ let j = i + flag.length++ if (curlCommand[j] === "=") j+++ while (isWhitespace(curlCommand[j])) j++++ const maybeQuote = curlCommand[j]+ const quoted = maybeQuote === "'" || maybeQuote === '"'+ const quoteChar = quoted ? maybeQuote : null+ if (quoted) j+++ while (isWhitespace(curlCommand[j])) j++++ const valueStart = j+ const endIndex = scanJSONValueEnd(curlCommand, valueStart)++ if (endIndex === null) {+ output += curlCommand[i]+ i+++ continue+ }++ const jsonBody = curlCommand.slice(valueStart, endIndex + 1)+ const placeholder = getJSONDataPlaceholder(extractedJSONData.length)+ extractedJSONData.push(jsonBody)++ if (output.length > 0 && !isWhitespace(output[output.length - 1]))+ output += " "++ output += `-d ${placeholder}`++ let nextIndex = endIndex + 1+ if (quoteChar && curlCommand[nextIndex] === quoteChar) nextIndex++++ if (+ nextIndex < curlCommand.length &&+ !isWhitespace(curlCommand[nextIndex])+ ) {+ output += " "+ }++ i = nextIndex+ }++ return {+ curlCommand: output.trim(),+ extractedJSONData,+ }+}++const DATA_ARG_KEYS = ["d", "data"] as const++const restorePlaceholder = (+ value: unknown,+ extractedJSONData: string[]+): unknown => {+ if (typeof value === "string") {+ if (!isJSONDataPlaceholder(value)) return value+ const idx = placeholderIndex(value)+ if (idx === null) return value+ return extractedJSONData[idx] ?? value+ }++ if (Array.isArray(value)) {+ return value.map((v) => restorePlaceholder(v, extractedJSONData))+ }++ return value+}++export const restoreJSONDataArgsFromPlaceholders = <T>(+ parsedArguments: T,+ extractedJSONData: string[]+): T => {+ if (extractedJSONData.length === 0) return parsedArguments+ if (!parsedArguments || typeof parsedArguments !== "object") {+ return parsedArguments+ }++ const args = parsedArguments as Record<string, unknown>+ const restored: Record<string, unknown> = { ...args }++ for (const key of DATA_ARG_KEYS) {+ if (key in restored) {+ restored[key] = restorePlaceholder(restored[key], extractedJSONData)+ }+ }++ return restored as T+}