Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
feat(workspace): read Node request, superagent, needle, AngularJS $http and Locust client calls (#4545)
Raghav Chamadiya514fa1a1fee6
diff --git a/docs/reference/WORKSPACE_CONTRACTS.md b/docs/reference/WORKSPACE_CONTRACTS.mdindex 7edce6690..860e18a13 100644--- a/docs/reference/WORKSPACE_CONTRACTS.md+++ b/docs/reference/WORKSPACE_CONTRACTS.md@@ -104,7 +104,9 @@ its URI version (`enableVersioning`, `@Version`), then the `@Controller` prefix.axios, ky, got or ofetch instance is read with the instance's `baseURL` / `prefixUrl`, also whenanother file imports the instance. An Angular `HttpClient` call is read on any receiver typedor injected as `HttpClient`, with `environment.apiUrl` folded from `src/environments/` and class-fields built on it; a base an interceptor prepends is not read. A client call+fields built on it; a base an interceptor prepends is not read. A Node `request` call reads a URL+or an options object held in a local (`var options = { uri, method }`), and a base from a+`module.exports = { ... }` module it requires (`endpoints.cartsUrl`). A client callwhose base URL is an unresolved placeholder (`fetch(\`${API_BASE}/users\`)`) matcheson the host-relative path; the link is **exact** when exactly one workspace serviceprovides that path (`unique_path`) and a lower-confidence **candidate** when the targetdiff --git a/docs/scale/WORKSPACES.md b/docs/scale/WORKSPACES.mdindex 2e2a93945..eb3848593 100644--- a/docs/scale/WORKSPACES.md+++ b/docs/scale/WORKSPACES.md@@ -138,8 +138,8 @@ Scans source files for HTTP routes, gRPC services, database tables, message topi| Language | Providers (routes served) | Consumers (calls made) ||----------|---------------------------|------------------------|-| JS / TS | Express, Hono, Fastify, Koa, Elysia; NestJS controllers; Next.js App Router; Remix | `fetch`; axios, ky, got, ofetch and their instances; Angular `HttpClient`; HTTP-named wrappers |-| Python | FastAPI, Flask, Django | requests, httpx (aiohttp through wrappers the index confirms) |+| JS / TS | Express, Hono, Fastify, Koa, Elysia; NestJS controllers; Next.js App Router; Remix | `fetch`; axios, ky, got, ofetch and their instances; Angular `HttpClient`; AngularJS `$http`; request, superagent, needle; HTTP-named wrappers |+| Python | FastAPI, Flask, Django | requests, httpx (aiohttp through wrappers the index confirms); Locust user `client` calls || PHP | Laravel | Guzzle, Laravel `Http` || Java / Kotlin | Spring, JAX-RS, Micronaut | Feign (`@RequestLine`, and `@FeignClient` with Spring MVC annotations, its `name` read as the host), Spring `@HttpExchange` interfaces, Spring `WebClient` and `RestClient`, `RestTemplate` (also through a getter or a `RequestEntity`), `java.net.http`, Apache HttpClient (`new HttpPost(url)`, with a base URL passed to a helper's constructor in another file), MicroProfile `@RegisterRestClient` interfaces, Ktor (Kotlin) || Go | gin, echo, chi, net/http | net/http |diff --git a/packages/core/src/repowise/core/workspace/extractors/http/commonjs.py b/packages/core/src/repowise/core/workspace/extractors/http/commonjs.pynew file mode 100644index 000000000..ab57bd62a--- /dev/null+++ b/packages/core/src/repowise/core/workspace/extractors/http/commonjs.py@@ -0,0 +1,74 @@+"""The string members a CommonJS module exports, for the files that ``require`` it.++Older Node services keep their service URLs in one module+(``module.exports = { cartsUrl: util.format("http://carts%s/carts", domain) }``)+and every route file reads them as ``endpoints.cartsUrl``. The members travel+through the orchestrator's mount pass keyed by module path, and a file binding+the module (``var endpoints = require("../endpoints")``) has+``endpoints.cartsUrl`` settled to the member's text. A member that does not+resolve is left out, so the name stays a ``${endpoints.cartsUrl}`` base token.+"""++from __future__ import annotations++import json+import posixpath+import re+from typing import TYPE_CHECKING++from ..langs import JS_EXTENSION_RE+from ..strings import JS_SYNTAX, object_members, resolve_string, string_constants++if TYPE_CHECKING:+ from collections.abc import Iterable++ from ..base import ScanContext++_KEY = "commonjs-exports:"+# Present when any file of the repo exports members, so a file is only looked up then.+_ANY = "commonjs-exports"+_EXPORTS_RE = re.compile(r"module\.exports\s*=\s*\{")+++def _module_path(rel_path: str) -> str:+ path = JS_EXTENSION_RE.sub("", rel_path)+ return path.removesuffix("/index") if path.endswith("/index") else path+++def exports_mounts(ctx: ScanContext) -> dict[str, str]:+ """The string members *ctx*'s ``module.exports = {...}`` holds, as one mount entry."""+ if "module.exports" not in ctx.content:+ return {}+ m = _EXPORTS_RE.search(ctx.content)+ if m is None or ctx.in_comment(m.start()):+ return {}+ constants = string_constants(ctx.content, JS_SYNTAX)+ members: dict[str, str] = {}+ for key, value in object_members(ctx.content, m.end() - 1, JS_SYNTAX).items():+ text = resolve_string(value, JS_SYNTAX, constants)+ if text is not None:+ members[key] = text+ if not members:+ return {}+ return {_KEY + _module_path(ctx.rel_path): json.dumps(members, sort_keys=True), _ANY: ""}+++def imported_exports(ctx: ScanContext, imports: Iterable[tuple[str, str, str]]) -> dict[str, str]:+ """``local.member -> text`` for each module *ctx* binds by a relative ``require`` or default import.++ *imports* is ``(local, imported, specifier)`` per binding.+ """+ if _ANY not in ctx.mounts:+ return {}+ out: dict[str, str] = {}+ for local, imported, spec in imports:+ if imported != "default" or not spec.startswith("."):+ continue+ path = posixpath.normpath(posixpath.join(posixpath.dirname(ctx.rel_path), spec))+ value = ctx.mounts.get(_KEY + _module_path(path))+ if value is not None:+ out.update((f"{local}.{member}", text) for member, text in json.loads(value).items())+ return out+++__all__ = ["exports_mounts", "imported_exports"]diff --git a/packages/core/src/repowise/core/workspace/extractors/http/js_clients.py b/packages/core/src/repowise/core/workspace/extractors/http/js_clients.pyindex 0b531404d..9250b99f6 100644--- a/packages/core/src/repowise/core/workspace/extractors/http/js_clients.py+++ b/packages/core/src/repowise/core/workspace/extractors/http/js_clients.py@@ -1,7 +1,8 @@"""JavaScript / TypeScript HTTP consumer dialect.Covers direct ``fetch`` calls, the client libraries of :mod:`.node_clients`-(axios, ky, got, ofetch and their instances), plus wrapper calls whose first+(axios, ky, got, ofetch, request, superagent, needle, AngularJS ``$http``+and their instances), plus wrapper calls whose firstargument is a concrete URL literal, e.g. ``fetchJSON(`${BASE}/path`, { method:"POST" })``. Wrapper detection is signal-gated (the callee name looks HTTP-ish,or the call carries a ``method:`` option) so ordinary `/`-prefixed string@@ -21,6 +22,7 @@ from ..langs import JS_TSfrom ..strings import JS_SYNTAX, literal_span, string_constantsfrom .angular import environment_mountsfrom .client_calls import ClientCallMatch, consumer_contracts, matches_in+from .commonjs import exports_mountsfrom .node_clients import client_calls, client_mountsif TYPE_CHECKING:@@ -128,7 +130,7 @@ class JsClientsDialect:requires = ("fetch", "'/", '"/', "`/", "${", "http", "axios", "import", "require", "export")def collect_mounts(self, ctx: ScanContext) -> dict[str, str]:- return client_mounts(ctx) | environment_mounts(ctx)+ return client_mounts(ctx) | environment_mounts(ctx) | exports_mounts(ctx)def extract(self, ctx: ScanContext) -> list[Contract]:content = ctx.contentdiff --git a/packages/core/src/repowise/core/workspace/extractors/http/node_clients.py b/packages/core/src/repowise/core/workspace/extractors/http/node_clients.pyindex 64029e1ff..be3d9e5c9 100644--- a/packages/core/src/repowise/core/workspace/extractors/http/node_clients.py+++ b/packages/core/src/repowise/core/workspace/extractors/http/node_clients.py@@ -1,4 +1,5 @@-"""HTTP client libraries for JS / TS: axios, ky, got, ofetch and Angular's HttpClient.+"""HTTP client libraries for JS / TS: axios, ky, got, ofetch, Angular's HttpClient,+Node's request, superagent and needle, and AngularJS ``$http``.A call goes straight through the library (``axios.post(url)``, ``ky.get(url)``,``got(url, { method })``) or through an instance created with a base@@ -27,7 +28,7 @@ from __future__ import annotationsimport refrom dataclasses import dataclass-from functools import lru_cache+from functools import lru_cache, partialfrom typing import TYPE_CHECKINGfrom repowise.core.workspace.origins import ContractOrigin@@ -35,13 +36,23 @@ from repowise.core.workspace.origins import ContractOriginfrom ..base import line_atfrom ..calls import Call, call_sites, file_strings, receiver_atfrom ..langs import JS_EXTENSION_RE, JS_TS-from ..strings import NAME_RE, Arg, call_arguments, js_class_fields, select_argument+from ..strings import (+ NAME_RE,+ Arg,+ call_arguments,+ js_class_fields,+ js_local_value,+ select_argument,+)from .angular import ANGULAR_HTTP, http_client_receivers, imported_environmentfrom .client_calls import VERBS, is_rooted_url, method_from_argument+from .commonjs import imported_exportsfrom .dialect import build_consumer_contractfrom .wrappers import GENERIC_ARGSif TYPE_CHECKING:+ from collections.abc import Callable+from repowise.core.workspace.contracts import Contractfrom ..base import ScanContext@@ -63,6 +74,10 @@ class _Library:typed: bool = Falserequest: str = "" # `request({ url, method })` ("config") or `request('GET', url)` ("method")body_first: bool = False # post / put / patch take a body before their options+ url_keys: tuple[str, ...] = ("url",) # the option a config object names its URL by+ # Names the client is reachable by with no import: a browser global, or+ # what a framework injects by name (AngularJS `$http`).+ globals: tuple[str, ...] = ()@propertydef package(self) -> str:@@ -70,7 +85,10 @@ class _Library:_LIBRARY_LIST = (- _Library("axios", ("create",), "baseURL", request="config", body_first=True),+ # A global axios: a script tag, Laravel's `window.axios`, Nuxt's `$axios`.+ _Library(+ "axios", ("create",), "baseURL", request="config", body_first=True, globals=("axios", "$axios")+ ),_Library("ky", ("create", "extend"), "prefixUrl"),_Library("got", ("extend",), "prefixUrl"),_Library("ofetch", ("create",), "baseURL", verbs=False, exports=("ofetch", "$fetch")),@@ -82,17 +100,25 @@ _LIBRARY_LIST = (request="method",body_first=True,),+ # `request(url, cb)`, `request({ uri, method }, cb)`, `request.post(url, cb)`.+ _Library("request", url_keys=("uri", "url")),+ _Library("request-promise", url_keys=("uri", "url")),+ _Library("superagent"),+ _Library("needle", request="method", body_first=True),+ # AngularJS injects `$http` by name; no import names it. Vue's+ # vue-resource and Nuxt's http module call theirs `$http` too.+ _Library("angularjs", exports=(), request="config", body_first=True, globals=("$http",)),)_LIBRARIES = {lib.package: lib for lib in _LIBRARY_LIST}# A library's specifier in quotes, so `ky` is not read out of every word holding it._SPECIFIERS = tuple(f"{q}{package}{q}" for package in _LIBRARIES for q in "'\"")-# axios is also a global (a script tag, Laravel's `window.axios`, Nuxt's `$axios`).-_AXIOS_GLOBALS = ("axios.", "axios(")+_GLOBALS = tuple((g, lib) for lib in _LIBRARY_LIST for g in lib.globals)# Qualifiers a client is called through: its own class, the browser global, Vue._QUALIFIER_RE = re.compile(r"(?:this|window|Vue)\s*\.\s*$")-_URL_KEY = Arg(keys=("url",))_METHOD_KEY = Arg(keys=("method",))+# A local a call's URL or options may be held in; a dotted name is a member, not a local.+_LOCAL_NAME_RE = re.compile(r"[A-Za-z_$][\w$]*")_MOUNT_NAMED = "http-client:"_MOUNT_DEFAULT = "http-client-default:"@@ -115,7 +141,8 @@ _IMPORT_RE = re.compile()# `const x = require('m')`, the binding read back from the `require`._REQUIRE_RE = re.compile(r"""require(?<![\w$]require)\s*\(\s*['"](?P<spec>[^'"]+)['"]\s*\)""")-_REQUIRE_BIND_RE = re.compile(r"(?:const|let|var)\s+(?P<default>[A-Za-z_$][\w$]*)\s*=\s*$")+# A declaration's first binding, or a later one in a comma list (`, request = require(...)`).+_REQUIRE_BIND_RE = re.compile(r"(?:(?:const|let|var)\s|,)\s*(?P<default>[A-Za-z_$][\w$]*)\s*=\s*$")# What `<client>.create(` is assigned to, read back from the call on its line._FACTORY_BIND_RE = re.compile(r"(?:(?P<default>export\s+default)|(?P<export>export\s+)?(?:const|let|var)\s+(?P<name>[A-Za-z_$][\w$]*)"@@ -248,11 +275,9 @@ def _file_clients(value = ctx.mounts.get(_key(_module_name(spec), imported))if value is not None:clients[local] = _decode(value)- axios = _LIBRARIES["axios"]- if "axios" not in clients and any(g in content for g in _AXIOS_GLOBALS):- clients["axios"] = _Client(axios, "", direct=True)- if "$axios" in content:- clients.setdefault("$axios", _Client(axios, "", direct=True))+ for name, library in _GLOBALS:+ if name not in clients and (f"{name}." in content or f"{name}(" in content):+ clients[name] = _Client(library, "", direct=True)exported: dict[str, _Client] = {}if not clients or (".create" not in content and ".extend" not in content):return _FileClients(clients, exported)@@ -308,17 +333,30 @@ def client_mounts(ctx: ScanContext) -> dict[str, str]:_BODY_VERBS = frozenset({"post", "put", "patch"})+def _local(content: str, pos: int, raw: str) -> str:+ """*raw*, or the value it holds at *pos* when it is a plain local name."""+ if _LOCAL_NAME_RE.fullmatch(raw):+ return js_local_value(content, raw, pos) or raw+ return raw++def _request(- verb: str | None, args: list[str], client: _Client, strings: FileStrings+ verb: str | None,+ args: list[str],+ client: _Client,+ strings: FileStrings,+ local: Callable[[str], str],) -> tuple[str, str] | None:"""``(method, url)`` a call names: ``x.post(url)``, ``x(url, { method })``,``x({ url, method })``, ``x.request('GET', url)``.A base in the call's own options (``axios.get(url, { baseURL })``) wins over- the client's, as it does at run time.+ the client's, as it does at run time. *local* reads a local name the+ config object or the URL is held in (``request(options, cb)``)."""if not args:return None+ args = [local(args[0]), *args[1:]]library = client.libraryif verb is not None and verb != "request":url_raws, method = args[:1], verb.upper()@@ -332,11 +370,11 @@ def _request(config = args[0].startswith("{")if verb == "request" and not config:return None # `request` takes a config object- url_raws = select_argument(args[:1], _URL_KEY) if config else args[:1]+ url_raws = select_argument(args[:1], Arg(keys=library.url_keys)) if config else args[:1]options = args[:1] if config else args[1:2]methods = select_argument(options, _METHOD_KEY)method = (method_from_argument(methods[0]) if methods else None) or "GET"- url = strings.text(url_raws[0]) if url_raws else None+ url = strings.text(local(url_raws[0])) if url_raws else Noneif url is None:return Nonebase = _base(options, library, strings)@@ -387,7 +425,7 @@ def client_calls(ctx: ScanContext) -> tuple[list[Contract], frozenset[str]]:content = ctx.contentif not (any(s in content for s in _SPECIFIERS)- or "axios" in content+ or any(name in content for name, _ in _GLOBALS)or (_MOUNT_ANY in ctx.mounts and ("import" in content or "require" in content))):return [], frozenset()@@ -396,6 +434,7 @@ def client_calls(ctx: ScanContext) -> tuple[list[Contract], frozenset[str]]:return [], frozenset()imports = _imports(content)strings.settled.update(imported_environment(ctx, imports))+ strings.settled.update(imported_exports(ctx, imports))clients = _file_clients(ctx, imports, strings).callable# `ofetch(url)` is a fetch call the fetch reader already reads.names = tuple(n for n, c in clients.items() if not (c.direct and not c.library.verbs))@@ -415,7 +454,7 @@ def client_calls(ctx: ScanContext) -> tuple[list[Contract], frozenset[str]]:continueif verb == "request" and not library.request:continue- request = _request(verb, args, client, strings)+ request = _request(verb, args, client, strings, partial(_local, content, m.start()))if request is None:continuemethod, url = requestdiff --git a/packages/core/src/repowise/core/workspace/extractors/http/python_clients.py b/packages/core/src/repowise/core/workspace/extractors/http/python_clients.pyindex df3c64aaa..de89cea56 100644--- a/packages/core/src/repowise/core/workspace/extractors/http/python_clients.py+++ b/packages/core/src/repowise/core/workspace/extractors/http/python_clients.py@@ -1,4 +1,10 @@-"""Python HTTP consumer dialect: ``requests`` and ``httpx``."""+"""Python HTTP consumer dialect: ``requests``, ``httpx`` and Locust's user client.++A Locust task calls the system under test through its user's client+(``self.client.get("/cart")``, ``l.client.post("/cart/empty")``); the path is+relative to the host the run targets, so only a rooted path is kept. Read+only in a file that imports ``locust``, where ``<x>.client`` is that client.+"""from __future__ import annotations@@ -7,7 +13,7 @@ from collections.abc import Iteratorfrom typing import TYPE_CHECKINGfrom ..langs import PYTHON-from ..strings import PYTHON_SYNTAX+from ..strings import PYTHON_SYNTAX, call_argumentsfrom .client_calls import ClientCallMatch, consumer_contracts, matches_infrom .dialect import METHODS@@ -22,15 +28,35 @@ _REQUESTS_RE = re.compile(re.IGNORECASE,)+_LOCUST_IMPORT_RE = re.compile(r"^[ \t]*(?:from|import)[ \t]+locust\b", re.MULTILINE)+# `self.client.get(` / `l.client.post(`: the verb and the call's opening parenthesis.+_LOCUST_CALL_RE = re.compile(rf"\.client\s*\.\s*({METHODS}|head|options)\s*\(")+def requests_calls(content: str) -> Iterator[ClientCallMatch]:yield from matches_in(content, _REQUESTS_RE, client="requests", url_group=2, method_group=1)+def locust_calls(content: str) -> Iterator[ClientCallMatch]:+ """``<user>.client.<verb>(path, ...)`` in a locustfile; the path is the first argument."""+ if _LOCUST_IMPORT_RE.search(content) is None:+ return+ for m in _LOCUST_CALL_RE.finditer(content):+ args = call_arguments(content, m.end() - 1)+ if args:+ yield ClientCallMatch(+ client="locust", url=args[0], offset=m.start(), method=m.group(1).upper()+ )++class PythonClientsDialect:name = "python-clients"extensions = PYTHON- requires = ("requests", "httpx", "Requests", "Httpx", "REQUESTS", "HTTPX")+ requires = ("requests", "httpx", "Requests", "Httpx", "REQUESTS", "HTTPX", "locust")def extract(self, ctx: ScanContext) -> list[Contract]:- return consumer_contracts(ctx, requests_calls(ctx.content), PYTHON_SYNTAX)+ content = ctx.content+ out = consumer_contracts(ctx, requests_calls(content), PYTHON_SYNTAX)+ if "locust" in content:+ out += consumer_contracts(ctx, locust_calls(content), PYTHON_SYNTAX, rooted_only=True)+ return outdiff --git a/packages/core/src/repowise/core/workspace/extractors/strings.py b/packages/core/src/repowise/core/workspace/extractors/strings.pyindex 8fdfcf16a..fa8a89f39 100644--- a/packages/core/src/repowise/core/workspace/extractors/strings.py+++ b/packages/core/src/repowise/core/workspace/extractors/strings.py@@ -44,7 +44,8 @@ class StringSyntax:``None`` means the body is already in ``${expr}`` form.``format_heads`` name calls whose first argument is a template with- ``placeholder`` holes. ``unwrap_heads`` name calls whose single argument+ ``placeholder`` holes; ``format_method`` is a method called on such a+ template (Python's ``"/x/{}".format(id)``). ``unwrap_heads`` name calls whose single argument*is* the value. ``concat`` is the string-concatenation operator, empty whenthe language's dialects do not fold concatenation. ``assignment`` locates``name = rhs`` statements for constant folding; ``assignment_strip`` drops a@@ -65,6 +66,7 @@ class StringSyntax:refuse_prefixes: str = ""format_heads: tuple[str, ...] = ()placeholder: re.Pattern[str] | None = None+ format_method: str = ""unwrap_heads: tuple[str, ...] = ()concat: str = ""assignment: re.Pattern[str] | None = None@@ -208,6 +210,10 @@ _TEMPLATE_INTERP_RE = re.compile(r"\$\{([^{}]+)\}")# Dropped from an interpolated expression (``settings.base.rstrip('/')``) so a# base-token reader sees the attribute, not ``rstrip``._TRAILING_CALL_RE = re.compile(r"\.\s*\w+\s*\([^()]*\)\s*$")+# A dotted name converted to text by a no-argument method: `id.toString()`, `name.strip()`.+_CONVERSION_RE = re.compile(+ NAME_RE.pattern + r"\.\s*(?:toString|trim|toLowerCase|toUpperCase|strip|lower|upper)\s*\(\s*\)"+)@cache@@ -307,6 +313,8 @@ def _literal_text(def _format_template(text: str, syntax: StringSyntax, constants: dict[str, str]) -> str | None:"""A ``format!("...")``-style call as template text, or ``None``."""+ if not syntax.format_heads:+ return Nonem = _head_re(syntax.format_heads).match(text)if m is None or syntax.placeholder is None:return None@@ -317,12 +325,31 @@ def _format_template(text: str, syntax: StringSyntax, constants: dict[str, str])if parsed is None:return Nonebody = _literal_text(parsed, syntax, constants)- if body is None:- return None+ return _template_holes(body, syntax) if body is not None else None+++def _template_holes(body: str, syntax: StringSyntax) -> str:+ assert syntax.placeholder is not None# `%%` is a literal percent sign, not a hole.return syntax.placeholder.sub(lambda h: "%" if h.group() == "%%" else "${x}", body)+def _format_method(text: str, syntax: StringSyntax, constants: dict[str, str]) -> str | None:+ """A ``"/x/{}".format(id)``-style call as template text, or ``None``."""+ if not syntax.format_method or not text.endswith(")"):+ return None+ dot = text.rfind(syntax.format_method + "(")+ if dot <= 0 or syntax.placeholder is None:+ return None+ if match_paren(text, dot + len(syntax.format_method)) != len(text) - 1:+ return None # the call does not run to the end of the expression+ parsed = _parse_literal(text[:dot], syntax)+ if parsed is None or (syntax.refuse is not None and syntax.refuse.search(parsed[2])):+ return None # `{{` is a literal brace no later step can tell from a hole+ body = _literal_text(parsed, syntax, constants)+ return _template_holes(body, syntax) if body is not None else None++def _whole_call_arguments(text: str, paren: int) -> list[str] | None:"""The arguments of the call at *paren* when the call is all of *text*.@@ -346,13 +373,13 @@ def _unwrap(text: str, syntax: StringSyntax) -> str | None:def _concat(text: str, syntax: StringSyntax, constants: dict[str, str]) -> str | None:"""``a + "/x"`` as template text, or ``None`` when a piece is not readable."""- if syntax.concat not in text:+ if not syntax.concat or syntax.concat not in text:return Nonepieces = split_top_level(text, syntax.concat)if len(pieces) < 2:return Noneout: list[str] = []- for piece in pieces:+ for i, piece in enumerate(pieces):piece = piece.strip()parsed = _parse_literal(piece, syntax)if parsed is not None:@@ -360,7 +387,9 @@ def _concat(text: str, syntax: StringSyntax, constants: dict[str, str]) -> str |if body is None:return Noneout.append(body)- elif NAME_RE.fullmatch(piece):+ elif NAME_RE.fullmatch(piece) or (i and _CONVERSION_RE.fullmatch(piece)):+ # `id.toString()` past the first piece is the value of `id`; the+ # first piece is the base, and a call there stays refused.out.append(_fold(piece, syntax, constants))else:return None # a call or an operator: not settled in this file@@ -380,18 +409,14 @@ def resolve_string(parsed = _parse_literal(text, syntax)if parsed is not None:return _literal_text(parsed, syntax, constants)- if syntax.format_heads:- folded = _format_template(text, syntax, constants)+ for fold in (_format_template, _format_method, _concat):+ folded = fold(text, syntax, constants)if folded is not None:return foldedif syntax.unwrap_heads:inner = _unwrap(text, syntax)if inner is not None:return resolve_string(inner, syntax, constants)- if syntax.concat:- folded = _concat(text, syntax, constants)- if folded is not None:- return foldedif NAME_RE.fullmatch(text):# A bare name. Its value resolved without constants when it was# recorded, so this step cannot reach another bare name.@@ -610,6 +635,80 @@ _MULTI_ASSIGN_RE = re.compile()+def object_members(content: str, brace: int, syntax: StringSyntax) -> dict[str, str]:+ """``key -> value source`` for each string entry of the object literal opening at *brace*.++ The rules of a ``members`` constant: a nested object's keys are dotted, and+ an entry that is not a string expression is left out.+ """+ seen: dict[str, str | None] = {}+ _fold_members(content, brace, "", syntax, seen)+ return {name[1:]: value for name, value in seen.items() if value is not None}+++# How far above a use a JS local's declaration is looked for.+_JS_LOCAL_WINDOW = 2_000+++@cache+def _js_declaration_re(name: str) -> re.Pattern[str]:+ return re.compile(rf"(?:var|let|const)\s+{re.escape(name)}\s*=(?![=>])")+++@cache+def _js_rebinding_re(name: str) -> re.Pattern[str]:+ """An assignment to *name* or a member of it, or a parameter of that name."""+ n = re.escape(name)+ return re.compile(+ rf"(?<![\w$.]){n}\s*(?:\.\s*[\w$]+\s*|\[[^\]]*\]\s*)?[-+*/|&?]?=(?![=>])" # `x =`, `x.uri =`, `x['uri'] =`+ rf"|Object\.assign\s*\(\s*{n}(?![\w$])"+ rf"|function(?![\w$])[^(]*\([^)]*(?<![\w$.]){n}(?![\w$])[^)]*\)" # `function (err, x)`+ rf"|(?<![\w$.]){n}\s*=>|\([^()]*(?<![\w$.]){n}(?![\w$])[^()]*\)\s*=>" # `x =>`, `(a, x) =>`+ # A method shorthand `save(x) {`; a control header (`if (x) {`) is not one.+ rf"|(?<![\w$.])(?!(?:if|for|while|switch|catch|with)\b)[A-Za-z_$][\w$]*\s*"+ rf"\([^()]*(?<![\w$.]){n}(?![\w$])[^()]*\)\s*\{{"+ )+++# A line that continues the statement above it, or a statement left open by an operator.+_CONTINUES_RE = re.compile(r"\s*[-+*/.?:|&]")+++def js_local_value(content: str, name: str, pos: int) -> str | None:+ """The value JS local *name* holds at *pos*, as source text, else ``None``.++ The right-hand side of the nearest ``var`` / ``let`` / ``const``+ declaration above *pos*, when the block declaring it is still open at+ *pos* and nothing assigns the name in between: ``var options = {...};+ request(options)``, where the file's constants refuse a name each handler+ declares again. A statement continued on the next line, an assignment to+ the name or one of its members, and a parameter of the same name between+ the two all refuse. Ceiling: only the last :data:`_JS_LOCAL_WINDOW`+ characters are read.+ """+ decl: re.Match[str] | None = None+ for m in _js_declaration_re(name).finditer(content, max(0, pos - _JS_LOCAL_WINDOW), pos):+ if not on_comment_line(content, m.start()):+ decl = m+ if decl is None:+ return None+ _, ends = _scan(content, decl.end(), pos, sep="\n", first_sep_only=True)+ if not ends:+ return None # the statement runs into the use+ statement = content[decl.end() : ends[0]].rstrip()+ if not statement.endswith(";") and (+ _CONTINUES_RE.match(content, ends[0] + 1) or _CONTINUES_RE.match(statement[-1:] or ";")+ ):+ return None # the expression goes on past the line+ rhs = split_top_level(statement, ",")[0].strip().rstrip(";").strip()+ between = "{" + content[ends[0] : pos]+ if _scan(between, 0, len(between), closer="}")[0] >= 0:+ return None # the declaring block closed before the use+ if _js_rebinding_re(name).search(content, ends[0], pos):+ return None # reassigned, mutated or shadowed before the use+ return rhs or None++# A typed local declaration (`var uri = `, `string url = `, `Uri? u = `). The# word before the name is its type, unless it is a keyword (`else uri = x`)._LOCAL_DECL = r"\b(?P<type>\w+)[>\[\]?]*[ \t]+{name}[ \t]*=(?![=>])"@@ -813,6 +912,8 @@ def resolve_argument(# ---------------------------------------------------------------------------_C_FORMAT_PLACEHOLDER_RE = re.compile(r"%%|%[-+ #0]*\d*(?:\.\d+)?[a-zA-Z]")+# Node's `util.format` specifiers only, so `%20` in a URL stays text.+_NODE_FORMAT_PLACEHOLDER_RE = re.compile(r"%%|%[sdifjoOc]")_BRACE_PLACEHOLDER_RE = re.compile(r"\{[^}]*\}")_BRACE_INTERP_RE = re.compile(r"(?<!\{)\{([^{}]+)\}(?!\})")_ESCAPED_BRACE_RE = re.compile(r"\{\{|\}\}")@@ -823,6 +924,8 @@ _ESCAPED_BRACE_RE = re.compile(r"\{\{|\}\}")JS_SYNTAX = StringSyntax(quotes="\"'`",template_quotes="`",+ format_heads=("util.format",),+ placeholder=_NODE_FORMAT_PLACEHOLDER_RE,concat="+",assignment=re.compile(# The boundary is checked behind the literal, so the regex keeps its@@ -854,6 +957,9 @@ PYTHON_SYNTAX = StringSyntax(interpolation=_BRACE_INTERP_RE,refuse=_ESCAPED_BRACE_RE,refuse_prefixes="b",+ format_method=".format",+ placeholder=_BRACE_PLACEHOLDER_RE,+ concat="+",assignment=re.compile(r"^(?<![(\[{,]\n)[ \t]*(?P<own>(?:self|cls)\.)?"r"(?P<name>[A-Z_][A-Z0-9_]*(?=[ \t]*=(?!=))|[A-Za-z_]\w*(?=(?:[ \t]*:[^=\n]+)?[ \t]+=[ \t]))"@@ -985,9 +1091,11 @@ __all__ = ["enclosing_local","inline_names","js_class_fields",+ "js_local_value","literal_span","map_entry","match_paren",+ "object_members","on_comment_line","resolve_argument","resolve_string",diff --git a/tests/unit/workspace/test_client_calls.py b/tests/unit/workspace/test_client_calls.pyindex 3658574af..fcff47ddb 100644--- a/tests/unit/workspace/test_client_calls.py+++ b/tests/unit/workspace/test_client_calls.py@@ -118,6 +118,26 @@ class TestUnwrapAndConcat:def test_concat_with_a_call_is_refused(self):assert resolve_string('base + "/" + fmt(x)', GO_SYNTAX) is None+ def test_a_converted_name_past_the_base_is_a_parameter(self):+ expr = 'base + "/catalogue/" + req.body.id.toString()'+ assert resolve_string(expr, JS_SYNTAX) == "${base}/catalogue/${req.body.id}"+ assert resolve_string('getBase() + "/x"', JS_SYNTAX) is None+ assert resolve_string('base.trim() + "/x"', JS_SYNTAX) is None+ assert resolve_string('"/a/" + cfg.getBasePath()', JS_SYNTAX) is None++ def test_python_concat_and_format_method(self):+ assert resolve_string('"/product/" + product', PYTHON_SYNTAX) == "/product/${product}"+ expr = "'/api/rate/{}/{}'.format(item['sku'], randint(1, 5))"+ assert resolve_string(expr, PYTHON_SYNTAX) == "/api/rate/${x}/${x}"+ assert resolve_string("'/a/{}'.format(x).strip()", PYTHON_SYNTAX) is None+ assert resolve_string("tmpl.format(x)", PYTHON_SYNTAX) is None+ assert resolve_string('"/x/{{y}}".format(a)', PYTHON_SYNTAX) is None++ def test_node_util_format(self):+ expr = 'util.format("http://carts%s/carts", domain)'+ assert resolve_string(expr, JS_SYNTAX) == "http://carts${x}/carts"+ assert resolve_string('util.format("/a%20b/%s", x)', JS_SYNTAX) == "/a%20b/${x}"+def test_a_concat_of_two_names_carries_no_path(self):assert resolve_string("c.InfoAPIURL + id.Name", GO_SYNTAX) == "${c.InfoAPIURL}${id.Name}"@@ -316,3 +336,35 @@ class TestConsumerUrlShapes:def test_a_scheme_after_a_leading_interpolation_is_the_urls(self):assert absolute_host("${proto}://h.com/api/u") == "h.com"assert extract_path_from_url("${proto}://h.com/api/u") == "/api/u"+++class TestLocust:+ _FILE = (+ "from locust import HttpUser, task\n"+ "def browse(l):\n"+ " l.client.get('/product/' + product)\n"+ "class U(HttpUser):\n"+ " @task\n"+ " def load(self):\n"+ " self.client.get('/')\n"+ " self.client.put('/api/rate/{}/{}'.format(sku, 5), headers=h)\n"+ " self.client.post('/cart', {'id': 1})\n"+ " # self.client.get('/api/commented')\n"+ " self.client.get('relative')\n"+ )++ def _ids(self, content: str) -> list[str]:+ from repowise.core.workspace.extractors.http.python_clients import PythonClientsDialect++ ctx = ScanContext("load", "locustfile.py", ".py", content, {}, None)+ return sorted(c.contract_id for c in PythonClientsDialect().extract(ctx))++ def test_user_client_calls_are_consumers(self):+ assert self._ids(self._FILE) == [+ "http::GET::/product/{param}",+ "http::POST::/cart",+ "http::PUT::/api/rate/{param}/{param}",+ ]++ def test_a_client_attribute_without_locust_is_not_read(self):+ assert self._ids(self._FILE.replace("from locust import", "from mylib import")) == []diff --git a/tests/unit/workspace/test_node_clients.py b/tests/unit/workspace/test_node_clients.pyindex 4ac1e2814..b256a8096 100644--- a/tests/unit/workspace/test_node_clients.py+++ b/tests/unit/workspace/test_node_clients.py@@ -257,3 +257,109 @@ def test_an_instance_verb_with_a_method_option_is_not_a_wrapper_row(tmp_path: Pa)rows = [c for c in HttpExtractor().extract(_write(tmp_path, {"a.ts": text}), "web") if c.role == "consumer"]assert [(c.meta["client"], c.contract_id) for c in rows] == [("axios", "http::POST::/api/pay")]+++class TestRequestSuperagentNeedleAndHttp:+ def test_request_by_url_options_and_verbs(self, tmp_path: Path) -> None:+ text = (+ "var express = require('express')\n"+ " , request = require('request')\n"+ "app.get('/cart', function (req, res) {\n"+ " request(base + '/' + custId + '/items', function (e, r, b) {});\n"+ "});\n"+ "app.delete('/cart', function (req, res) {\n"+ " var options = {\n uri: base + '/' + custId,\n method: 'DELETE'\n };\n"+ " request(options, function (e, r, b) {});\n"+ "});\n"+ "app.post('/cart', function (req, res) {\n"+ " var options = { uri: base + '/items/' + req.body.id.toString(), method: 'POST' };\n"+ " request(options, cb);\n"+ "});\n"+ "request.put('/api/sizes', cb);\n"+ )+ assert _calls(_write(tmp_path, {"api/cart.js": text})) == {+ ("request", "GET", "/{param}/items", "base"),+ ("request", "POST", "/items/{param}", "base"),+ ("request", "PUT", "/api/sizes", ""),+ }++ def test_a_local_from_a_closed_block_or_reassigned_is_not_read(self, tmp_path: Path) -> None:+ text = (+ "const request = require('request');\n"+ "function a() { var options = { uri: '/api/a' };\n}\n"+ "function b(options) { request(options, cb); }\n"+ "function c() { var url = '/api/c';\n url = next;\n request(url, cb); }\n"+ )+ assert _calls(_write(tmp_path, {"a.js": text})) == set()++ def test_the_base_comes_from_a_required_module_exports(self, tmp_path: Path) -> None:+ endpoints = (+ "var util = require('util');\nvar domain = '';\n"+ "module.exports = {\n"+ " cartsUrl: util.format('http://carts%s/carts', domain),\n"+ " ordersUrl: 'http://orders',\n"+ "};\n"+ )+ cart = (+ "var request = require('request'), endpoints = require('../endpoints');\n"+ "request(endpoints.cartsUrl + '/' + custId + '/items', cb);\n"+ "request.post(endpoints.ordersUrl + '/orders', cb);\n"+ )+ repo = _write(tmp_path, {"api/endpoints.js": endpoints, "api/cart/index.js": cart})+ assert _calls(repo) == {+ ("request", "GET", "/carts/{param}/items", ""),+ ("request", "POST", "/orders", ""),+ }++ def test_superagent_and_needle(self, tmp_path: Path) -> None:+ text = (+ "import superagent from 'superagent';\nconst needle = require('needle');\n"+ "superagent.get('/api/a').end(cb);\n"+ "needle.post('/api/b', body, { json: true }, cb);\n"+ "needle.request('delete', '/api/c', null, cb);\n"+ )+ assert _calls(_write(tmp_path, {"a.js": text})) == {+ ("superagent", "GET", "/api/a", ""),+ ("needle", "POST", "/api/b", ""),+ ("needle", "DELETE", "/api/c", ""),+ }++ def test_request_needs_its_require(self, tmp_path: Path) -> None:+ text = "app.get('/x', (request, res) => { request('/api/nope'); request.get('/api/no'); });\n"+ assert {row for row in _calls(_write(tmp_path, {"a.js": text})) if row[0] == "request"} == set()++ def test_angularjs_http_by_config_and_verbs(self, tmp_path: Path) -> None:+ text = (+ "app.controller('cart', function($scope, $http) {\n"+ " $http({\n url: '/api/catalogue/products/' + category,\n method: 'GET'\n });\n"+ " var url = '/api/cart/add/' + user.id + '/' + $scope.sku;\n"+ " $http({ url: url, method: 'PUT' }).then(ok);\n"+ " $http.post('/api/user/login', creds);\n"+ " // $http.get('/api/commented');\n"+ "});\n"+ )+ assert _calls(_write(tmp_path, {"static/js/controller.js": text})) == {+ ("angularjs", "GET", "/api/catalogue/products/{param}", ""),+ ("angularjs", "PUT", "/api/cart/add/{param}/{param}", ""),+ ("angularjs", "POST", "/api/user/login", ""),+ }+++def test_a_node_request_call_links_to_the_service_it_names(tmp_path: Path) -> None:+ catalogue = _write(+ tmp_path / "catalogue",+ {"server.js": "const app = require('express')();\napp.get('/product/:sku', (req, res) => {});\n"},+ )+ cart = _write(+ tmp_path / "cart",+ {+ "server.js": (+ "const request = require('request');\n"+ "var catalogueHost = process.env.CATALOGUE_HOST || 'catalogue'\n"+ "request('http://' + catalogueHost + ':8080/product/' + sku, (err, res, body) => {});\n"+ )+ },+ )+ contracts = HttpExtractor().extract(catalogue, "catalogue") + HttpExtractor().extract(cart, "cart")+ links = {(link.provider_repo, link.consumer_repo, link.contract_id) for link in match_contracts(contracts)}+ assert links == {("catalogue", "cart", "http::GET::/product/{param}")}diff --git a/tests/unit/workspace/test_strings.py b/tests/unit/workspace/test_strings.pyindex 824c7599a..a7902617b 100644--- a/tests/unit/workspace/test_strings.py+++ b/tests/unit/workspace/test_strings.py@@ -190,3 +190,41 @@ class TestPythonConstants:)def test_syntax_for_suffix(suffix: str, expected: object) -> None:assert syntax_for_suffix(suffix) is expected+++class TestJsLocalValue:+ @staticmethod+ def _at(content: str, name: str = "options") -> str | None:+ from repowise.core.workspace.extractors.strings import js_local_value++ return js_local_value(content, name, content.rindex("request("))++ def test_the_nearest_declaration_in_an_open_block(self) -> None:+ text = "function f() {\n var options = {\n uri: '/a',\n };\n request(options, cb);\n}\n"+ assert self._at(text) == "{\n uri: '/a',\n }"++ @pytest.mark.parametrize(+ "between",+ [+ "options.method = 'DELETE';\n",+ "Object.assign(options, extra);\n",+ "options = other;\n",+ "items.forEach(function (err, options) {\n",+ "items.forEach((e, options) => {\n",+ "items.forEach(options => {\n",+ "options['uri'] = base + '/b';\n",+ "class S {\n save(options) {\n",+ "}\nfunction g() {\n",+ ],+ )+ def test_a_rebinding_or_closed_block_refuses(self, between: str) -> None:+ assert self._at("function f() {\n var options = { uri: '/a' };\n" + between + "request(options);\n") is None++ def test_a_statement_continued_on_the_next_line_refuses(self) -> None:+ assert self._at("var url = '/a'\n + '/b';\nrequest(url);\n", "url") is None+ assert self._at("var url = '/a' +\n '/b';\nrequest(url);\n", "url") is None+ assert self._at("var url = '/a'\nrequest(url);\n", "url") == "'/a'"++ def test_a_control_header_naming_the_local_does_not_shadow_it(self) -> None:+ text = "function f() {\n var options = { uri: '/a' };\n if (options) {\n request(options);\n"+ assert self._at(text) == "{ uri: '/a' }"