Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

feat(workspace): read Node request, superagent, needle, AngularJS $http and Locust client calls (#4545)

Raghav Chamadiya514fa1a1fee6
diff --git a/docs/reference/WORKSPACE_CONTRACTS.md b/docs/reference/WORKSPACE_CONTRACTS.md
index 7edce6690..860e18a13 100644
--- a/docs/reference/WORKSPACE_CONTRACTS.md
+++ b/docs/reference/WORKSPACE_CONTRACTS.md
@@ -104,7 +104,9 @@ its URI version (`enableVersioning`, `@Version`), then the `@Controller` prefix.
axios, ky, got or ofetch instance is read with the instance's `baseURL` / `prefixUrl`, also when
another file imports the instance. An Angular `HttpClient` call is read on any receiver typed
or injected as `HttpClient`, with `environment.apiUrl` folded from `src/environments/` and class
-fields built on it; a base an interceptor prepends is not read. A client call
+fields built on it; a base an interceptor prepends is not read. A Node `request` call reads a URL
+or an options object held in a local (`var options = { uri, method }`), and a base from a
+`module.exports = { ... }` module it requires (`endpoints.cartsUrl`). A client call
whose base URL is an unresolved placeholder (`fetch(\`${API_BASE}/users\`)`) matches
on the host-relative path; the link is **exact** when exactly one workspace service
provides that path (`unique_path`) and a lower-confidence **candidate** when the target
diff --git a/docs/scale/WORKSPACES.md b/docs/scale/WORKSPACES.md
index 2e2a93945..eb3848593 100644
--- a/docs/scale/WORKSPACES.md
+++ b/docs/scale/WORKSPACES.md
@@ -138,8 +138,8 @@ Scans source files for HTTP routes, gRPC services, database tables, message topi
| Language | Providers (routes served) | Consumers (calls made) |
|----------|---------------------------|------------------------|
-| JS / TS | Express, Hono, Fastify, Koa, Elysia; NestJS controllers; Next.js App Router; Remix | `fetch`; axios, ky, got, ofetch and their instances; Angular `HttpClient`; HTTP-named wrappers |
-| Python | FastAPI, Flask, Django | requests, httpx (aiohttp through wrappers the index confirms) |
+| JS / TS | Express, Hono, Fastify, Koa, Elysia; NestJS controllers; Next.js App Router; Remix | `fetch`; axios, ky, got, ofetch and their instances; Angular `HttpClient`; AngularJS `$http`; request, superagent, needle; HTTP-named wrappers |
+| Python | FastAPI, Flask, Django | requests, httpx (aiohttp through wrappers the index confirms); Locust user `client` calls |
| PHP | Laravel | Guzzle, Laravel `Http` |
| Java / Kotlin | Spring, JAX-RS, Micronaut | Feign (`@RequestLine`, and `@FeignClient` with Spring MVC annotations, its `name` read as the host), Spring `@HttpExchange` interfaces, Spring `WebClient` and `RestClient`, `RestTemplate` (also through a getter or a `RequestEntity`), `java.net.http`, Apache HttpClient (`new HttpPost(url)`, with a base URL passed to a helper's constructor in another file), MicroProfile `@RegisterRestClient` interfaces, Ktor (Kotlin) |
| Go | gin, echo, chi, net/http | net/http |
diff --git a/packages/core/src/repowise/core/workspace/extractors/http/commonjs.py b/packages/core/src/repowise/core/workspace/extractors/http/commonjs.py
new file mode 100644
index 000000000..ab57bd62a
--- /dev/null
+++ b/packages/core/src/repowise/core/workspace/extractors/http/commonjs.py
@@ -0,0 +1,74 @@
+"""The string members a CommonJS module exports, for the files that ``require`` it.
+
+Older Node services keep their service URLs in one module
+(``module.exports = { cartsUrl: util.format("http://carts%s/carts", domain) }``)
+and every route file reads them as ``endpoints.cartsUrl``. The members travel
+through the orchestrator's mount pass keyed by module path, and a file binding
+the module (``var endpoints = require("../endpoints")``) has
+``endpoints.cartsUrl`` settled to the member's text. A member that does not
+resolve is left out, so the name stays a ``${endpoints.cartsUrl}`` base token.
+"""
+
+from __future__ import annotations
+
+import json
+import posixpath
+import re
+from typing import TYPE_CHECKING
+
+from ..langs import JS_EXTENSION_RE
+from ..strings import JS_SYNTAX, object_members, resolve_string, string_constants
+
+if TYPE_CHECKING:
+ from collections.abc import Iterable
+
+ from ..base import ScanContext
+
+_KEY = "commonjs-exports:"
+# Present when any file of the repo exports members, so a file is only looked up then.
+_ANY = "commonjs-exports"
+_EXPORTS_RE = re.compile(r"module\.exports\s*=\s*\{")
+
+
+def _module_path(rel_path: str) -> str:
+ path = JS_EXTENSION_RE.sub("", rel_path)
+ return path.removesuffix("/index") if path.endswith("/index") else path
+
+
+def exports_mounts(ctx: ScanContext) -> dict[str, str]:
+ """The string members *ctx*'s ``module.exports = {...}`` holds, as one mount entry."""
+ if "module.exports" not in ctx.content:
+ return {}
+ m = _EXPORTS_RE.search(ctx.content)
+ if m is None or ctx.in_comment(m.start()):
+ return {}
+ constants = string_constants(ctx.content, JS_SYNTAX)
+ members: dict[str, str] = {}
+ for key, value in object_members(ctx.content, m.end() - 1, JS_SYNTAX).items():
+ text = resolve_string(value, JS_SYNTAX, constants)
+ if text is not None:
+ members[key] = text
+ if not members:
+ return {}
+ return {_KEY + _module_path(ctx.rel_path): json.dumps(members, sort_keys=True), _ANY: ""}
+
+
+def imported_exports(ctx: ScanContext, imports: Iterable[tuple[str, str, str]]) -> dict[str, str]:
+ """``local.member -> text`` for each module *ctx* binds by a relative ``require`` or default import.
+
+ *imports* is ``(local, imported, specifier)`` per binding.
+ """
+ if _ANY not in ctx.mounts:
+ return {}
+ out: dict[str, str] = {}
+ for local, imported, spec in imports:
+ if imported != "default" or not spec.startswith("."):
+ continue
+ path = posixpath.normpath(posixpath.join(posixpath.dirname(ctx.rel_path), spec))
+ value = ctx.mounts.get(_KEY + _module_path(path))
+ if value is not None:
+ out.update((f"{local}.{member}", text) for member, text in json.loads(value).items())
+ return out
+
+
+__all__ = ["exports_mounts", "imported_exports"]
diff --git a/packages/core/src/repowise/core/workspace/extractors/http/js_clients.py b/packages/core/src/repowise/core/workspace/extractors/http/js_clients.py
index 0b531404d..9250b99f6 100644
--- a/packages/core/src/repowise/core/workspace/extractors/http/js_clients.py
+++ b/packages/core/src/repowise/core/workspace/extractors/http/js_clients.py
@@ -1,7 +1,8 @@
"""JavaScript / TypeScript HTTP consumer dialect.
Covers direct ``fetch`` calls, the client libraries of :mod:`.node_clients`
-(axios, ky, got, ofetch and their instances), plus wrapper calls whose first
+(axios, ky, got, ofetch, request, superagent, needle, AngularJS ``$http``
+and their instances), plus wrapper calls whose first
argument is a concrete URL literal, e.g. ``fetchJSON(`${BASE}/path`, { method:
"POST" })``. Wrapper detection is signal-gated (the callee name looks HTTP-ish,
or the call carries a ``method:`` option) so ordinary `/`-prefixed string
@@ -21,6 +22,7 @@ from ..langs import JS_TS
from ..strings import JS_SYNTAX, literal_span, string_constants
from .angular import environment_mounts
from .client_calls import ClientCallMatch, consumer_contracts, matches_in
+from .commonjs import exports_mounts
from .node_clients import client_calls, client_mounts
if TYPE_CHECKING:
@@ -128,7 +130,7 @@ class JsClientsDialect:
requires = ("fetch", "'/", '"/', "`/", "${", "http", "axios", "import", "require", "export")
def collect_mounts(self, ctx: ScanContext) -> dict[str, str]:
- return client_mounts(ctx) | environment_mounts(ctx)
+ return client_mounts(ctx) | environment_mounts(ctx) | exports_mounts(ctx)
def extract(self, ctx: ScanContext) -> list[Contract]:
content = ctx.content
diff --git a/packages/core/src/repowise/core/workspace/extractors/http/node_clients.py b/packages/core/src/repowise/core/workspace/extractors/http/node_clients.py
index 64029e1ff..be3d9e5c9 100644
--- a/packages/core/src/repowise/core/workspace/extractors/http/node_clients.py
+++ b/packages/core/src/repowise/core/workspace/extractors/http/node_clients.py
@@ -1,4 +1,5 @@
-"""HTTP client libraries for JS / TS: axios, ky, got, ofetch and Angular's HttpClient.
+"""HTTP client libraries for JS / TS: axios, ky, got, ofetch, Angular's HttpClient,
+Node's request, superagent and needle, and AngularJS ``$http``.
A call goes straight through the library (``axios.post(url)``, ``ky.get(url)``,
``got(url, { method })``) or through an instance created with a base
@@ -27,7 +28,7 @@ from __future__ import annotations
import re
from dataclasses import dataclass
-from functools import lru_cache
+from functools import lru_cache, partial
from typing import TYPE_CHECKING
from repowise.core.workspace.origins import ContractOrigin
@@ -35,13 +36,23 @@ from repowise.core.workspace.origins import ContractOrigin
from ..base import line_at
from ..calls import Call, call_sites, file_strings, receiver_at
from ..langs import JS_EXTENSION_RE, JS_TS
-from ..strings import NAME_RE, Arg, call_arguments, js_class_fields, select_argument
+from ..strings import (
+ NAME_RE,
+ Arg,
+ call_arguments,
+ js_class_fields,
+ js_local_value,
+ select_argument,
+)
from .angular import ANGULAR_HTTP, http_client_receivers, imported_environment
from .client_calls import VERBS, is_rooted_url, method_from_argument
+from .commonjs import imported_exports
from .dialect import build_consumer_contract
from .wrappers import GENERIC_ARGS
if TYPE_CHECKING:
+ from collections.abc import Callable
+
from repowise.core.workspace.contracts import Contract
from ..base import ScanContext
@@ -63,6 +74,10 @@ class _Library:
typed: bool = False
request: str = "" # `request({ url, method })` ("config") or `request('GET', url)` ("method")
body_first: bool = False # post / put / patch take a body before their options
+ url_keys: tuple[str, ...] = ("url",) # the option a config object names its URL by
+ # Names the client is reachable by with no import: a browser global, or
+ # what a framework injects by name (AngularJS `$http`).
+ globals: tuple[str, ...] = ()
@property
def package(self) -> str:
@@ -70,7 +85,10 @@ class _Library:
_LIBRARY_LIST = (
- _Library("axios", ("create",), "baseURL", request="config", body_first=True),
+ # A global axios: a script tag, Laravel's `window.axios`, Nuxt's `$axios`.
+ _Library(
+ "axios", ("create",), "baseURL", request="config", body_first=True, globals=("axios", "$axios")
+ ),
_Library("ky", ("create", "extend"), "prefixUrl"),
_Library("got", ("extend",), "prefixUrl"),
_Library("ofetch", ("create",), "baseURL", verbs=False, exports=("ofetch", "$fetch")),
@@ -82,17 +100,25 @@ _LIBRARY_LIST = (
request="method",
body_first=True,
),
+ # `request(url, cb)`, `request({ uri, method }, cb)`, `request.post(url, cb)`.
+ _Library("request", url_keys=("uri", "url")),
+ _Library("request-promise", url_keys=("uri", "url")),
+ _Library("superagent"),
+ _Library("needle", request="method", body_first=True),
+ # AngularJS injects `$http` by name; no import names it. Vue's
+ # vue-resource and Nuxt's http module call theirs `$http` too.
+ _Library("angularjs", exports=(), request="config", body_first=True, globals=("$http",)),
)
_LIBRARIES = {lib.package: lib for lib in _LIBRARY_LIST}
# A library's specifier in quotes, so `ky` is not read out of every word holding it.
_SPECIFIERS = tuple(f"{q}{package}{q}" for package in _LIBRARIES for q in "'\"")
-# axios is also a global (a script tag, Laravel's `window.axios`, Nuxt's `$axios`).
-_AXIOS_GLOBALS = ("axios.", "axios(")
+_GLOBALS = tuple((g, lib) for lib in _LIBRARY_LIST for g in lib.globals)
# Qualifiers a client is called through: its own class, the browser global, Vue.
_QUALIFIER_RE = re.compile(r"(?:this|window|Vue)\s*\.\s*$")
-_URL_KEY = Arg(keys=("url",))
_METHOD_KEY = Arg(keys=("method",))
+# A local a call's URL or options may be held in; a dotted name is a member, not a local.
+_LOCAL_NAME_RE = re.compile(r"[A-Za-z_$][\w$]*")
_MOUNT_NAMED = "http-client:"
_MOUNT_DEFAULT = "http-client-default:"
@@ -115,7 +141,8 @@ _IMPORT_RE = re.compile(
)
# `const x = require('m')`, the binding read back from the `require`.
_REQUIRE_RE = re.compile(r"""require(?<![\w$]require)\s*\(\s*['"](?P<spec>[^'"]+)['"]\s*\)""")
-_REQUIRE_BIND_RE = re.compile(r"(?:const|let|var)\s+(?P<default>[A-Za-z_$][\w$]*)\s*=\s*$")
+# A declaration's first binding, or a later one in a comma list (`, request = require(...)`).
+_REQUIRE_BIND_RE = re.compile(r"(?:(?:const|let|var)\s|,)\s*(?P<default>[A-Za-z_$][\w$]*)\s*=\s*$")
# What `<client>.create(` is assigned to, read back from the call on its line.
_FACTORY_BIND_RE = re.compile(
r"(?:(?P<default>export\s+default)|(?P<export>export\s+)?(?:const|let|var)\s+(?P<name>[A-Za-z_$][\w$]*)"
@@ -248,11 +275,9 @@ def _file_clients(
value = ctx.mounts.get(_key(_module_name(spec), imported))
if value is not None:
clients[local] = _decode(value)
- axios = _LIBRARIES["axios"]
- if "axios" not in clients and any(g in content for g in _AXIOS_GLOBALS):
- clients["axios"] = _Client(axios, "", direct=True)
- if "$axios" in content:
- clients.setdefault("$axios", _Client(axios, "", direct=True))
+ for name, library in _GLOBALS:
+ if name not in clients and (f"{name}." in content or f"{name}(" in content):
+ clients[name] = _Client(library, "", direct=True)
exported: dict[str, _Client] = {}
if not clients or (".create" not in content and ".extend" not in content):
return _FileClients(clients, exported)
@@ -308,17 +333,30 @@ def client_mounts(ctx: ScanContext) -> dict[str, str]:
_BODY_VERBS = frozenset({"post", "put", "patch"})
+def _local(content: str, pos: int, raw: str) -> str:
+ """*raw*, or the value it holds at *pos* when it is a plain local name."""
+ if _LOCAL_NAME_RE.fullmatch(raw):
+ return js_local_value(content, raw, pos) or raw
+ return raw
+
+
def _request(
- verb: str | None, args: list[str], client: _Client, strings: FileStrings
+ verb: str | None,
+ args: list[str],
+ client: _Client,
+ strings: FileStrings,
+ local: Callable[[str], str],
) -> tuple[str, str] | None:
"""``(method, url)`` a call names: ``x.post(url)``, ``x(url, { method })``,
``x({ url, method })``, ``x.request('GET', url)``.
A base in the call's own options (``axios.get(url, { baseURL })``) wins over
- the client's, as it does at run time.
+ the client's, as it does at run time. *local* reads a local name the
+ config object or the URL is held in (``request(options, cb)``).
"""
if not args:
return None
+ args = [local(args[0]), *args[1:]]
library = client.library
if verb is not None and verb != "request":
url_raws, method = args[:1], verb.upper()
@@ -332,11 +370,11 @@ def _request(
config = args[0].startswith("{")
if verb == "request" and not config:
return None # `request` takes a config object
- url_raws = select_argument(args[:1], _URL_KEY) if config else args[:1]
+ url_raws = select_argument(args[:1], Arg(keys=library.url_keys)) if config else args[:1]
options = args[:1] if config else args[1:2]
methods = select_argument(options, _METHOD_KEY)
method = (method_from_argument(methods[0]) if methods else None) or "GET"
- url = strings.text(url_raws[0]) if url_raws else None
+ url = strings.text(local(url_raws[0])) if url_raws else None
if url is None:
return None
base = _base(options, library, strings)
@@ -387,7 +425,7 @@ def client_calls(ctx: ScanContext) -> tuple[list[Contract], frozenset[str]]:
content = ctx.content
if not (
any(s in content for s in _SPECIFIERS)
- or "axios" in content
+ or any(name in content for name, _ in _GLOBALS)
or (_MOUNT_ANY in ctx.mounts and ("import" in content or "require" in content))
):
return [], frozenset()
@@ -396,6 +434,7 @@ def client_calls(ctx: ScanContext) -> tuple[list[Contract], frozenset[str]]:
return [], frozenset()
imports = _imports(content)
strings.settled.update(imported_environment(ctx, imports))
+ strings.settled.update(imported_exports(ctx, imports))
clients = _file_clients(ctx, imports, strings).callable
# `ofetch(url)` is a fetch call the fetch reader already reads.
names = tuple(n for n, c in clients.items() if not (c.direct and not c.library.verbs))
@@ -415,7 +454,7 @@ def client_calls(ctx: ScanContext) -> tuple[list[Contract], frozenset[str]]:
continue
if verb == "request" and not library.request:
continue
- request = _request(verb, args, client, strings)
+ request = _request(verb, args, client, strings, partial(_local, content, m.start()))
if request is None:
continue
method, url = request
diff --git a/packages/core/src/repowise/core/workspace/extractors/http/python_clients.py b/packages/core/src/repowise/core/workspace/extractors/http/python_clients.py
index df3c64aaa..de89cea56 100644
--- a/packages/core/src/repowise/core/workspace/extractors/http/python_clients.py
+++ b/packages/core/src/repowise/core/workspace/extractors/http/python_clients.py
@@ -1,4 +1,10 @@
-"""Python HTTP consumer dialect: ``requests`` and ``httpx``."""
+"""Python HTTP consumer dialect: ``requests``, ``httpx`` and Locust's user client.
+
+A Locust task calls the system under test through its user's client
+(``self.client.get("/cart")``, ``l.client.post("/cart/empty")``); the path is
+relative to the host the run targets, so only a rooted path is kept. Read
+only in a file that imports ``locust``, where ``<x>.client`` is that client.
+"""
from __future__ import annotations
@@ -7,7 +13,7 @@ from collections.abc import Iterator
from typing import TYPE_CHECKING
from ..langs import PYTHON
-from ..strings import PYTHON_SYNTAX
+from ..strings import PYTHON_SYNTAX, call_arguments
from .client_calls import ClientCallMatch, consumer_contracts, matches_in
from .dialect import METHODS
@@ -22,15 +28,35 @@ _REQUESTS_RE = re.compile(
re.IGNORECASE,
)
+_LOCUST_IMPORT_RE = re.compile(r"^[ \t]*(?:from|import)[ \t]+locust\b", re.MULTILINE)
+# `self.client.get(` / `l.client.post(`: the verb and the call's opening parenthesis.
+_LOCUST_CALL_RE = re.compile(rf"\.client\s*\.\s*({METHODS}|head|options)\s*\(")
+
def requests_calls(content: str) -> Iterator[ClientCallMatch]:
yield from matches_in(content, _REQUESTS_RE, client="requests", url_group=2, method_group=1)
+def locust_calls(content: str) -> Iterator[ClientCallMatch]:
+ """``<user>.client.<verb>(path, ...)`` in a locustfile; the path is the first argument."""
+ if _LOCUST_IMPORT_RE.search(content) is None:
+ return
+ for m in _LOCUST_CALL_RE.finditer(content):
+ args = call_arguments(content, m.end() - 1)
+ if args:
+ yield ClientCallMatch(
+ client="locust", url=args[0], offset=m.start(), method=m.group(1).upper()
+ )
+
+
class PythonClientsDialect:
name = "python-clients"
extensions = PYTHON
- requires = ("requests", "httpx", "Requests", "Httpx", "REQUESTS", "HTTPX")
+ requires = ("requests", "httpx", "Requests", "Httpx", "REQUESTS", "HTTPX", "locust")
def extract(self, ctx: ScanContext) -> list[Contract]:
- return consumer_contracts(ctx, requests_calls(ctx.content), PYTHON_SYNTAX)
+ content = ctx.content
+ out = consumer_contracts(ctx, requests_calls(content), PYTHON_SYNTAX)
+ if "locust" in content:
+ out += consumer_contracts(ctx, locust_calls(content), PYTHON_SYNTAX, rooted_only=True)
+ return out
diff --git a/packages/core/src/repowise/core/workspace/extractors/strings.py b/packages/core/src/repowise/core/workspace/extractors/strings.py
index 8fdfcf16a..fa8a89f39 100644
--- a/packages/core/src/repowise/core/workspace/extractors/strings.py
+++ b/packages/core/src/repowise/core/workspace/extractors/strings.py
@@ -44,7 +44,8 @@ class StringSyntax:
``None`` means the body is already in ``${expr}`` form.
``format_heads`` name calls whose first argument is a template with
- ``placeholder`` holes. ``unwrap_heads`` name calls whose single argument
+ ``placeholder`` holes; ``format_method`` is a method called on such a
+ template (Python's ``"/x/{}".format(id)``). ``unwrap_heads`` name calls whose single argument
*is* the value. ``concat`` is the string-concatenation operator, empty when
the language's dialects do not fold concatenation. ``assignment`` locates
``name = rhs`` statements for constant folding; ``assignment_strip`` drops a
@@ -65,6 +66,7 @@ class StringSyntax:
refuse_prefixes: str = ""
format_heads: tuple[str, ...] = ()
placeholder: re.Pattern[str] | None = None
+ format_method: str = ""
unwrap_heads: tuple[str, ...] = ()
concat: str = ""
assignment: re.Pattern[str] | None = None
@@ -208,6 +210,10 @@ _TEMPLATE_INTERP_RE = re.compile(r"\$\{([^{}]+)\}")
# Dropped from an interpolated expression (``settings.base.rstrip('/')``) so a
# base-token reader sees the attribute, not ``rstrip``.
_TRAILING_CALL_RE = re.compile(r"\.\s*\w+\s*\([^()]*\)\s*$")
+# A dotted name converted to text by a no-argument method: `id.toString()`, `name.strip()`.
+_CONVERSION_RE = re.compile(
+ NAME_RE.pattern + r"\.\s*(?:toString|trim|toLowerCase|toUpperCase|strip|lower|upper)\s*\(\s*\)"
+)
@cache
@@ -307,6 +313,8 @@ def _literal_text(
def _format_template(text: str, syntax: StringSyntax, constants: dict[str, str]) -> str | None:
"""A ``format!("...")``-style call as template text, or ``None``."""
+ if not syntax.format_heads:
+ return None
m = _head_re(syntax.format_heads).match(text)
if m is None or syntax.placeholder is None:
return None
@@ -317,12 +325,31 @@ def _format_template(text: str, syntax: StringSyntax, constants: dict[str, str])
if parsed is None:
return None
body = _literal_text(parsed, syntax, constants)
- if body is None:
- return None
+ return _template_holes(body, syntax) if body is not None else None
+
+
+def _template_holes(body: str, syntax: StringSyntax) -> str:
+ assert syntax.placeholder is not None
# `%%` is a literal percent sign, not a hole.
return syntax.placeholder.sub(lambda h: "%" if h.group() == "%%" else "${x}", body)
+def _format_method(text: str, syntax: StringSyntax, constants: dict[str, str]) -> str | None:
+ """A ``"/x/{}".format(id)``-style call as template text, or ``None``."""
+ if not syntax.format_method or not text.endswith(")"):
+ return None
+ dot = text.rfind(syntax.format_method + "(")
+ if dot <= 0 or syntax.placeholder is None:
+ return None
+ if match_paren(text, dot + len(syntax.format_method)) != len(text) - 1:
+ return None # the call does not run to the end of the expression
+ parsed = _parse_literal(text[:dot], syntax)
+ if parsed is None or (syntax.refuse is not None and syntax.refuse.search(parsed[2])):
+ return None # `{{` is a literal brace no later step can tell from a hole
+ body = _literal_text(parsed, syntax, constants)
+ return _template_holes(body, syntax) if body is not None else None
+
+
def _whole_call_arguments(text: str, paren: int) -> list[str] | None:
"""The arguments of the call at *paren* when the call is all of *text*.
@@ -346,13 +373,13 @@ def _unwrap(text: str, syntax: StringSyntax) -> str | None:
def _concat(text: str, syntax: StringSyntax, constants: dict[str, str]) -> str | None:
"""``a + "/x"`` as template text, or ``None`` when a piece is not readable."""
- if syntax.concat not in text:
+ if not syntax.concat or syntax.concat not in text:
return None
pieces = split_top_level(text, syntax.concat)
if len(pieces) < 2:
return None
out: list[str] = []
- for piece in pieces:
+ for i, piece in enumerate(pieces):
piece = piece.strip()
parsed = _parse_literal(piece, syntax)
if parsed is not None:
@@ -360,7 +387,9 @@ def _concat(text: str, syntax: StringSyntax, constants: dict[str, str]) -> str |
if body is None:
return None
out.append(body)
- elif NAME_RE.fullmatch(piece):
+ elif NAME_RE.fullmatch(piece) or (i and _CONVERSION_RE.fullmatch(piece)):
+ # `id.toString()` past the first piece is the value of `id`; the
+ # first piece is the base, and a call there stays refused.
out.append(_fold(piece, syntax, constants))
else:
return None # a call or an operator: not settled in this file
@@ -380,18 +409,14 @@ def resolve_string(
parsed = _parse_literal(text, syntax)
if parsed is not None:
return _literal_text(parsed, syntax, constants)
- if syntax.format_heads:
- folded = _format_template(text, syntax, constants)
+ for fold in (_format_template, _format_method, _concat):
+ folded = fold(text, syntax, constants)
if folded is not None:
return folded
if syntax.unwrap_heads:
inner = _unwrap(text, syntax)
if inner is not None:
return resolve_string(inner, syntax, constants)
- if syntax.concat:
- folded = _concat(text, syntax, constants)
- if folded is not None:
- return folded
if NAME_RE.fullmatch(text):
# A bare name. Its value resolved without constants when it was
# recorded, so this step cannot reach another bare name.
@@ -610,6 +635,80 @@ _MULTI_ASSIGN_RE = re.compile(
)
+def object_members(content: str, brace: int, syntax: StringSyntax) -> dict[str, str]:
+ """``key -> value source`` for each string entry of the object literal opening at *brace*.
+
+ The rules of a ``members`` constant: a nested object's keys are dotted, and
+ an entry that is not a string expression is left out.
+ """
+ seen: dict[str, str | None] = {}
+ _fold_members(content, brace, "", syntax, seen)
+ return {name[1:]: value for name, value in seen.items() if value is not None}
+
+
+# How far above a use a JS local's declaration is looked for.
+_JS_LOCAL_WINDOW = 2_000
+
+
+@cache
+def _js_declaration_re(name: str) -> re.Pattern[str]:
+ return re.compile(rf"(?:var|let|const)\s+{re.escape(name)}\s*=(?![=>])")
+
+
+@cache
+def _js_rebinding_re(name: str) -> re.Pattern[str]:
+ """An assignment to *name* or a member of it, or a parameter of that name."""
+ n = re.escape(name)
+ return re.compile(
+ rf"(?<![\w$.]){n}\s*(?:\.\s*[\w$]+\s*|\[[^\]]*\]\s*)?[-+*/|&?]?=(?![=>])" # `x =`, `x.uri =`, `x['uri'] =`
+ rf"|Object\.assign\s*\(\s*{n}(?![\w$])"
+ rf"|function(?![\w$])[^(]*\([^)]*(?<![\w$.]){n}(?![\w$])[^)]*\)" # `function (err, x)`
+ rf"|(?<![\w$.]){n}\s*=>|\([^()]*(?<![\w$.]){n}(?![\w$])[^()]*\)\s*=>" # `x =>`, `(a, x) =>`
+ # A method shorthand `save(x) {`; a control header (`if (x) {`) is not one.
+ rf"|(?<![\w$.])(?!(?:if|for|while|switch|catch|with)\b)[A-Za-z_$][\w$]*\s*"
+ rf"\([^()]*(?<![\w$.]){n}(?![\w$])[^()]*\)\s*\{{"
+ )
+
+
+# A line that continues the statement above it, or a statement left open by an operator.
+_CONTINUES_RE = re.compile(r"\s*[-+*/.?:|&]")
+
+
+def js_local_value(content: str, name: str, pos: int) -> str | None:
+ """The value JS local *name* holds at *pos*, as source text, else ``None``.
+
+ The right-hand side of the nearest ``var`` / ``let`` / ``const``
+ declaration above *pos*, when the block declaring it is still open at
+ *pos* and nothing assigns the name in between: ``var options = {...};
+ request(options)``, where the file's constants refuse a name each handler
+ declares again. A statement continued on the next line, an assignment to
+ the name or one of its members, and a parameter of the same name between
+ the two all refuse. Ceiling: only the last :data:`_JS_LOCAL_WINDOW`
+ characters are read.
+ """
+ decl: re.Match[str] | None = None
+ for m in _js_declaration_re(name).finditer(content, max(0, pos - _JS_LOCAL_WINDOW), pos):
+ if not on_comment_line(content, m.start()):
+ decl = m
+ if decl is None:
+ return None
+ _, ends = _scan(content, decl.end(), pos, sep="\n", first_sep_only=True)
+ if not ends:
+ return None # the statement runs into the use
+ statement = content[decl.end() : ends[0]].rstrip()
+ if not statement.endswith(";") and (
+ _CONTINUES_RE.match(content, ends[0] + 1) or _CONTINUES_RE.match(statement[-1:] or ";")
+ ):
+ return None # the expression goes on past the line
+ rhs = split_top_level(statement, ",")[0].strip().rstrip(";").strip()
+ between = "{" + content[ends[0] : pos]
+ if _scan(between, 0, len(between), closer="}")[0] >= 0:
+ return None # the declaring block closed before the use
+ if _js_rebinding_re(name).search(content, ends[0], pos):
+ return None # reassigned, mutated or shadowed before the use
+ return rhs or None
+
+
# A typed local declaration (`var uri = `, `string url = `, `Uri? u = `). The
# word before the name is its type, unless it is a keyword (`else uri = x`).
_LOCAL_DECL = r"\b(?P<type>\w+)[>\[\]?]*[ \t]+{name}[ \t]*=(?![=>])"
@@ -813,6 +912,8 @@ def resolve_argument(
# ---------------------------------------------------------------------------
_C_FORMAT_PLACEHOLDER_RE = re.compile(r"%%|%[-+ #0]*\d*(?:\.\d+)?[a-zA-Z]")
+# Node's `util.format` specifiers only, so `%20` in a URL stays text.
+_NODE_FORMAT_PLACEHOLDER_RE = re.compile(r"%%|%[sdifjoOc]")
_BRACE_PLACEHOLDER_RE = re.compile(r"\{[^}]*\}")
_BRACE_INTERP_RE = re.compile(r"(?<!\{)\{([^{}]+)\}(?!\})")
_ESCAPED_BRACE_RE = re.compile(r"\{\{|\}\}")
@@ -823,6 +924,8 @@ _ESCAPED_BRACE_RE = re.compile(r"\{\{|\}\}")
JS_SYNTAX = StringSyntax(
quotes="\"'`",
template_quotes="`",
+ format_heads=("util.format",),
+ placeholder=_NODE_FORMAT_PLACEHOLDER_RE,
concat="+",
assignment=re.compile(
# The boundary is checked behind the literal, so the regex keeps its
@@ -854,6 +957,9 @@ PYTHON_SYNTAX = StringSyntax(
interpolation=_BRACE_INTERP_RE,
refuse=_ESCAPED_BRACE_RE,
refuse_prefixes="b",
+ format_method=".format",
+ placeholder=_BRACE_PLACEHOLDER_RE,
+ concat="+",
assignment=re.compile(
r"^(?<![(\[{,]\n)[ \t]*(?P<own>(?:self|cls)\.)?"
r"(?P<name>[A-Z_][A-Z0-9_]*(?=[ \t]*=(?!=))|[A-Za-z_]\w*(?=(?:[ \t]*:[^=\n]+)?[ \t]+=[ \t]))"
@@ -985,9 +1091,11 @@ __all__ = [
"enclosing_local",
"inline_names",
"js_class_fields",
+ "js_local_value",
"literal_span",
"map_entry",
"match_paren",
+ "object_members",
"on_comment_line",
"resolve_argument",
"resolve_string",
diff --git a/tests/unit/workspace/test_client_calls.py b/tests/unit/workspace/test_client_calls.py
index 3658574af..fcff47ddb 100644
--- a/tests/unit/workspace/test_client_calls.py
+++ b/tests/unit/workspace/test_client_calls.py
@@ -118,6 +118,26 @@ class TestUnwrapAndConcat:
def test_concat_with_a_call_is_refused(self):
assert resolve_string('base + "/" + fmt(x)', GO_SYNTAX) is None
+ def test_a_converted_name_past_the_base_is_a_parameter(self):
+ expr = 'base + "/catalogue/" + req.body.id.toString()'
+ assert resolve_string(expr, JS_SYNTAX) == "${base}/catalogue/${req.body.id}"
+ assert resolve_string('getBase() + "/x"', JS_SYNTAX) is None
+ assert resolve_string('base.trim() + "/x"', JS_SYNTAX) is None
+ assert resolve_string('"/a/" + cfg.getBasePath()', JS_SYNTAX) is None
+
+ def test_python_concat_and_format_method(self):
+ assert resolve_string('"/product/" + product', PYTHON_SYNTAX) == "/product/${product}"
+ expr = "'/api/rate/{}/{}'.format(item['sku'], randint(1, 5))"
+ assert resolve_string(expr, PYTHON_SYNTAX) == "/api/rate/${x}/${x}"
+ assert resolve_string("'/a/{}'.format(x).strip()", PYTHON_SYNTAX) is None
+ assert resolve_string("tmpl.format(x)", PYTHON_SYNTAX) is None
+ assert resolve_string('"/x/{{y}}".format(a)', PYTHON_SYNTAX) is None
+
+ def test_node_util_format(self):
+ expr = 'util.format("http://carts%s/carts", domain)'
+ assert resolve_string(expr, JS_SYNTAX) == "http://carts${x}/carts"
+ assert resolve_string('util.format("/a%20b/%s", x)', JS_SYNTAX) == "/a%20b/${x}"
+
def test_a_concat_of_two_names_carries_no_path(self):
assert resolve_string("c.InfoAPIURL + id.Name", GO_SYNTAX) == "${c.InfoAPIURL}${id.Name}"
@@ -316,3 +336,35 @@ class TestConsumerUrlShapes:
def test_a_scheme_after_a_leading_interpolation_is_the_urls(self):
assert absolute_host("${proto}://h.com/api/u") == "h.com"
assert extract_path_from_url("${proto}://h.com/api/u") == "/api/u"
+
+
+class TestLocust:
+ _FILE = (
+ "from locust import HttpUser, task\n"
+ "def browse(l):\n"
+ " l.client.get('/product/' + product)\n"
+ "class U(HttpUser):\n"
+ " @task\n"
+ " def load(self):\n"
+ " self.client.get('/')\n"
+ " self.client.put('/api/rate/{}/{}'.format(sku, 5), headers=h)\n"
+ " self.client.post('/cart', {'id': 1})\n"
+ " # self.client.get('/api/commented')\n"
+ " self.client.get('relative')\n"
+ )
+
+ def _ids(self, content: str) -> list[str]:
+ from repowise.core.workspace.extractors.http.python_clients import PythonClientsDialect
+
+ ctx = ScanContext("load", "locustfile.py", ".py", content, {}, None)
+ return sorted(c.contract_id for c in PythonClientsDialect().extract(ctx))
+
+ def test_user_client_calls_are_consumers(self):
+ assert self._ids(self._FILE) == [
+ "http::GET::/product/{param}",
+ "http::POST::/cart",
+ "http::PUT::/api/rate/{param}/{param}",
+ ]
+
+ def test_a_client_attribute_without_locust_is_not_read(self):
+ assert self._ids(self._FILE.replace("from locust import", "from mylib import")) == []
diff --git a/tests/unit/workspace/test_node_clients.py b/tests/unit/workspace/test_node_clients.py
index 4ac1e2814..b256a8096 100644
--- a/tests/unit/workspace/test_node_clients.py
+++ b/tests/unit/workspace/test_node_clients.py
@@ -257,3 +257,109 @@ def test_an_instance_verb_with_a_method_option_is_not_a_wrapper_row(tmp_path: Pa
)
rows = [c for c in HttpExtractor().extract(_write(tmp_path, {"a.ts": text}), "web") if c.role == "consumer"]
assert [(c.meta["client"], c.contract_id) for c in rows] == [("axios", "http::POST::/api/pay")]
+
+
+class TestRequestSuperagentNeedleAndHttp:
+ def test_request_by_url_options_and_verbs(self, tmp_path: Path) -> None:
+ text = (
+ "var express = require('express')\n"
+ " , request = require('request')\n"
+ "app.get('/cart', function (req, res) {\n"
+ " request(base + '/' + custId + '/items', function (e, r, b) {});\n"
+ "});\n"
+ "app.delete('/cart', function (req, res) {\n"
+ " var options = {\n uri: base + '/' + custId,\n method: 'DELETE'\n };\n"
+ " request(options, function (e, r, b) {});\n"
+ "});\n"
+ "app.post('/cart', function (req, res) {\n"
+ " var options = { uri: base + '/items/' + req.body.id.toString(), method: 'POST' };\n"
+ " request(options, cb);\n"
+ "});\n"
+ "request.put('/api/sizes', cb);\n"
+ )
+ assert _calls(_write(tmp_path, {"api/cart.js": text})) == {
+ ("request", "GET", "/{param}/items", "base"),
+ ("request", "POST", "/items/{param}", "base"),
+ ("request", "PUT", "/api/sizes", ""),
+ }
+
+ def test_a_local_from_a_closed_block_or_reassigned_is_not_read(self, tmp_path: Path) -> None:
+ text = (
+ "const request = require('request');\n"
+ "function a() { var options = { uri: '/api/a' };\n}\n"
+ "function b(options) { request(options, cb); }\n"
+ "function c() { var url = '/api/c';\n url = next;\n request(url, cb); }\n"
+ )
+ assert _calls(_write(tmp_path, {"a.js": text})) == set()
+
+ def test_the_base_comes_from_a_required_module_exports(self, tmp_path: Path) -> None:
+ endpoints = (
+ "var util = require('util');\nvar domain = '';\n"
+ "module.exports = {\n"
+ " cartsUrl: util.format('http://carts%s/carts', domain),\n"
+ " ordersUrl: 'http://orders',\n"
+ "};\n"
+ )
+ cart = (
+ "var request = require('request'), endpoints = require('../endpoints');\n"
+ "request(endpoints.cartsUrl + '/' + custId + '/items', cb);\n"
+ "request.post(endpoints.ordersUrl + '/orders', cb);\n"
+ )
+ repo = _write(tmp_path, {"api/endpoints.js": endpoints, "api/cart/index.js": cart})
+ assert _calls(repo) == {
+ ("request", "GET", "/carts/{param}/items", ""),
+ ("request", "POST", "/orders", ""),
+ }
+
+ def test_superagent_and_needle(self, tmp_path: Path) -> None:
+ text = (
+ "import superagent from 'superagent';\nconst needle = require('needle');\n"
+ "superagent.get('/api/a').end(cb);\n"
+ "needle.post('/api/b', body, { json: true }, cb);\n"
+ "needle.request('delete', '/api/c', null, cb);\n"
+ )
+ assert _calls(_write(tmp_path, {"a.js": text})) == {
+ ("superagent", "GET", "/api/a", ""),
+ ("needle", "POST", "/api/b", ""),
+ ("needle", "DELETE", "/api/c", ""),
+ }
+
+ def test_request_needs_its_require(self, tmp_path: Path) -> None:
+ text = "app.get('/x', (request, res) => { request('/api/nope'); request.get('/api/no'); });\n"
+ assert {row for row in _calls(_write(tmp_path, {"a.js": text})) if row[0] == "request"} == set()
+
+ def test_angularjs_http_by_config_and_verbs(self, tmp_path: Path) -> None:
+ text = (
+ "app.controller('cart', function($scope, $http) {\n"
+ " $http({\n url: '/api/catalogue/products/' + category,\n method: 'GET'\n });\n"
+ " var url = '/api/cart/add/' + user.id + '/' + $scope.sku;\n"
+ " $http({ url: url, method: 'PUT' }).then(ok);\n"
+ " $http.post('/api/user/login', creds);\n"
+ " // $http.get('/api/commented');\n"
+ "});\n"
+ )
+ assert _calls(_write(tmp_path, {"static/js/controller.js": text})) == {
+ ("angularjs", "GET", "/api/catalogue/products/{param}", ""),
+ ("angularjs", "PUT", "/api/cart/add/{param}/{param}", ""),
+ ("angularjs", "POST", "/api/user/login", ""),
+ }
+
+
+def test_a_node_request_call_links_to_the_service_it_names(tmp_path: Path) -> None:
+ catalogue = _write(
+ tmp_path / "catalogue",
+ {"server.js": "const app = require('express')();\napp.get('/product/:sku', (req, res) => {});\n"},
+ )
+ cart = _write(
+ tmp_path / "cart",
+ {
+ "server.js": (
+ "const request = require('request');\n"
+ "var catalogueHost = process.env.CATALOGUE_HOST || 'catalogue'\n"
+ "request('http://' + catalogueHost + ':8080/product/' + sku, (err, res, body) => {});\n"
+ )
+ },
+ )
+ contracts = HttpExtractor().extract(catalogue, "catalogue") + HttpExtractor().extract(cart, "cart")
+ links = {(link.provider_repo, link.consumer_repo, link.contract_id) for link in match_contracts(contracts)}
+ assert links == {("catalogue", "cart", "http::GET::/product/{param}")}
diff --git a/tests/unit/workspace/test_strings.py b/tests/unit/workspace/test_strings.py
index 824c7599a..a7902617b 100644
--- a/tests/unit/workspace/test_strings.py
+++ b/tests/unit/workspace/test_strings.py
@@ -190,3 +190,41 @@ class TestPythonConstants:
)
def test_syntax_for_suffix(suffix: str, expected: object) -> None:
assert syntax_for_suffix(suffix) is expected
+
+
+class TestJsLocalValue:
+ @staticmethod
+ def _at(content: str, name: str = "options") -> str | None:
+ from repowise.core.workspace.extractors.strings import js_local_value
+
+ return js_local_value(content, name, content.rindex("request("))
+
+ def test_the_nearest_declaration_in_an_open_block(self) -> None:
+ text = "function f() {\n var options = {\n uri: '/a',\n };\n request(options, cb);\n}\n"
+ assert self._at(text) == "{\n uri: '/a',\n }"
+
+ @pytest.mark.parametrize(
+ "between",
+ [
+ "options.method = 'DELETE';\n",
+ "Object.assign(options, extra);\n",
+ "options = other;\n",
+ "items.forEach(function (err, options) {\n",
+ "items.forEach((e, options) => {\n",
+ "items.forEach(options => {\n",
+ "options['uri'] = base + '/b';\n",
+ "class S {\n save(options) {\n",
+ "}\nfunction g() {\n",
+ ],
+ )
+ def test_a_rebinding_or_closed_block_refuses(self, between: str) -> None:
+ assert self._at("function f() {\n var options = { uri: '/a' };\n" + between + "request(options);\n") is None
+
+ def test_a_statement_continued_on_the_next_line_refuses(self) -> None:
+ assert self._at("var url = '/a'\n + '/b';\nrequest(url);\n", "url") is None
+ assert self._at("var url = '/a' +\n '/b';\nrequest(url);\n", "url") is None
+ assert self._at("var url = '/a'\nrequest(url);\n", "url") == "'/a'"
+
+ def test_a_control_header_naming_the_local_does_not_shadow_it(self) -> None:
+ text = "function f() {\n var options = { uri: '/a' };\n if (options) {\n request(options);\n"
+ assert self._at(text) == "{ uri: '/a' }"