:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import re
import ast
import builtins
from dataclasses import dataclass
from typing import Literal
from posthog.dataclasses import frozen
@dataclass(frozen=True)
class PythonGlobalsAnalysis:
used: list[str]
exported_with_types: list[dict[str, str]]
class LocalAssignmentCollector(ast.NodeVisitor):
def __init__(self) -> None:
self.locals: set[str] = set()
self.global_names: set[str] = set()
self.nonlocal_names: set[str] = set()
def visit_Global(self, node: ast.Global) -> None:
self.global_names.update(node.names)
def visit_Nonlocal(self, node: ast.Nonlocal) -> None:
self.nonlocal_names.update(node.names)
def visit_Name(self, node: ast.Name) -> None:
if isinstance(node.ctx, ast.Store):
self.locals.add(node.id)
def visit_FunctionDef(self, node: ast.FunctionDef) -> None:
self.locals.add(node.name)
def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef) -> None:
self.locals.add(node.name)
def visit_ClassDef(self, node: ast.ClassDef) -> None:
self.locals.add(node.name)
def visit_Lambda(self, node: ast.Lambda) -> None:
return
def visit_ListComp(self, node: ast.ListComp) -> None:
return
def visit_SetComp(self, node: ast.SetComp) -> None:
return
def visit_DictComp(self, node: ast.DictComp) -> None:
return
def visit_GeneratorExp(self, node: ast.GeneratorExp) -> None:
return
def visit_Import(self, node: ast.Import) -> None:
for alias in node.names:
name = alias.asname or alias.name.split(".")[0]
self.locals.add(name)
def visit_ImportFrom(self, node: ast.ImportFrom) -> None:
for alias in node.names:
name = alias.asname or alias.name
self.locals.add(name)
def collect_scope_locals(body: list[ast.stmt]) -> set[str]:
collector = LocalAssignmentCollector()
for statement in body:
collector.visit(statement)
locals_set = collector.locals - collector.global_names - collector.nonlocal_names
return locals_set
def collect_arg_names(arguments: ast.arguments) -> set[str]:
names = {arg.arg for arg in arguments.args}
names.update({arg.arg for arg in arguments.posonlyargs})
names.update({arg.arg for arg in arguments.kwonlyargs})
if arguments.vararg:
names.add(arguments.vararg.arg)
if arguments.kwarg:
names.add(arguments.kwarg.arg)
return names
ScopeKind = Literal["module", "function", "class", "lambda", "comprehension"]
@frozen
class Scope:
kind: ScopeKind
locals: set[str]
class GlobalAnalyzer(ast.NodeVisitor):
def __init__(self, module_locals: set[str], builtins: set[str]) -> None:
self.module_locals = module_locals
self.builtins = builtins
self.used: set[str] = set()
self.scopes: list[Scope] = [Scope(kind="module", locals=module_locals)]
# Module names bound so far in execution order. At module (exec) scope a name is a
# local *from the point it is bound onwards* — reading it before that reads the injected
# namespace, so a read-before-bind name (df.columns = ...; df = df.assign(...)) is a
# genuine external input, unlike function scope where any assignment makes the name local.
self.module_assigned: set[str] = set()
def is_global_name(self, name: str) -> bool:
current_scope = self.scopes[-1]
if current_scope.kind == "class":
if name in current_scope.locals:
return False
return True
for scope in reversed(self.scopes):
if name in scope.locals:
return scope.kind == "module"
return True
def visit_Name(self, node: ast.Name) -> None:
if isinstance(node.ctx, ast.Load):
# A module-scope name already bound before this read is a local, not an input.
if self.is_global_name(node.id) and node.id not in self.builtins and node.id not in self.module_assigned:
self.used.add(node.id)
def visit_Module(self, node: ast.Module) -> None:
for statement in node.body:
self._visit_module_statement(statement)
def _visit_module_statement(self, statement: ast.stmt) -> None:
# Walk top-level statements in execution order, visiting reads before recording the
# bindings they introduce, so a name read before it is bound counts as an input.
if isinstance(statement, ast.Assign):
self.visit(statement.value)
for target in statement.targets:
self.visit(target) # attribute/subscript targets still *read* their base object
self._record_bindings(statement.targets)
elif isinstance(statement, ast.AnnAssign):
if statement.value is not None:
self.visit(statement.value)
self.visit(statement.annotation)
self.visit(statement.target)
if statement.value is not None:
self._record_bindings([statement.target])
elif isinstance(statement, ast.AugAssign):
self.visit(statement.value)
self.visit(statement.target)
self._record_bindings([statement.target])
elif isinstance(statement, ast.Import):
self.module_assigned.update(alias.asname or alias.name.split(".")[0] for alias in statement.names)
elif isinstance(statement, ast.ImportFrom):
self.module_assigned.update(alias.asname or alias.name for alias in statement.names)
elif isinstance(statement, ast.FunctionDef | ast.AsyncFunctionDef | ast.ClassDef):
self.visit(statement)
self.module_assigned.add(statement.name)
elif isinstance(statement, ast.For | ast.AsyncFor):
self.visit(statement.iter)
self._record_bindings([statement.target]) # loop var is bound before the body runs
for child in [*statement.body, *statement.orelse]:
self._visit_module_statement(child)
elif isinstance(statement, ast.With | ast.AsyncWith):
for item in statement.items:
self.visit(item.context_expr)
if item.optional_vars is not None:
self._record_bindings([item.optional_vars])
for child in statement.body:
self._visit_module_statement(child)
elif isinstance(statement, ast.If | ast.While):
self.visit(statement.test)
for child in [*statement.body, *statement.orelse]:
self._visit_module_statement(child)
elif isinstance(statement, ast.Try):
for child in statement.body:
self._visit_module_statement(child)
for handler in statement.handlers:
if handler.type is not None:
self.visit(handler.type)
if handler.name is not None:
self.module_assigned.add(handler.name)
for child in handler.body:
self._visit_module_statement(child)
for child in [*statement.orelse, *statement.finalbody]:
self._visit_module_statement(child)
elif isinstance(statement, ast.Match):
self.visit(statement.subject)
for case in statement.cases:
# Capture patterns bind their names before the case body runs.
self.module_assigned.update(match_capture_names(case.pattern))
if case.guard is not None:
self.visit(case.guard)
for child in case.body:
self._visit_module_statement(child)
else:
self.visit(statement)
def visit_NamedExpr(self, node: ast.NamedExpr) -> None:
# A walrus binds its target in the enclosing scope; at module scope record it so a later
# read (print(rows) after `if (rows := ...)`) isn't mistaken for an external input.
self.visit(node.value)
if len(self.scopes) == 1:
self.module_assigned.update(extract_target_names(node.target))
self.visit(node.target)
def _record_bindings(self, targets: list[ast.expr]) -> None:
for target in targets:
self.module_assigned.update(extract_target_names(target))
def visit_FunctionDef(self, node: ast.FunctionDef) -> None:
self._visit_function_like(node, "function")
def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef) -> None:
self._visit_function_like(node, "function")
def _visit_function_like(self, node: ast.AST, kind: ScopeKind) -> None:
if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef):
for decorator in node.decorator_list:
self.visit(decorator)
for default in node.args.defaults:
self.visit(default)
for kw_default in node.args.kw_defaults:
if kw_default is not None:
self.visit(kw_default)
for arg in node.args.args + node.args.posonlyargs + node.args.kwonlyargs:
if arg.annotation:
self.visit(arg.annotation)
if node.args.vararg and node.args.vararg.annotation:
self.visit(node.args.vararg.annotation)
if node.args.kwarg and node.args.kwarg.annotation:
self.visit(node.args.kwarg.annotation)
if node.returns:
self.visit(node.returns)
if getattr(node, "type_params", None):
for param in node.type_params:
self.visit(param)
locals_set = collect_scope_locals(node.body)
locals_set.update(collect_arg_names(node.args))
self.scopes.append(Scope(kind=kind, locals=locals_set))
for statement in node.body:
self.visit(statement)
self.scopes.pop()
def visit_ClassDef(self, node: ast.ClassDef) -> None:
for base in node.bases:
self.visit(base)
for keyword in node.keywords:
self.visit(keyword)
for decorator in node.decorator_list:
self.visit(decorator)
if getattr(node, "type_params", None):
for param in node.type_params:
self.visit(param)
locals_set = collect_scope_locals(node.body)
self.scopes.append(Scope(kind="class", locals=locals_set))
for statement in node.body:
self.visit(statement)
self.scopes.pop()
def visit_Lambda(self, node: ast.Lambda) -> None:
locals_set = collect_arg_names(node.args)
self.scopes.append(Scope(kind="lambda", locals=locals_set))
self.visit(node.body)
self.scopes.pop()
def visit_ListComp(self, node: ast.ListComp) -> None:
self._visit_comprehension(node, "comprehension")
def visit_SetComp(self, node: ast.SetComp) -> None:
self._visit_comprehension(node, "comprehension")
def visit_DictComp(self, node: ast.DictComp) -> None:
self._visit_comprehension(node, "comprehension")
def visit_GeneratorExp(self, node: ast.GeneratorExp) -> None:
self._visit_comprehension(node, "comprehension")
def _visit_comprehension(self, node: ast.AST, kind: ScopeKind) -> None:
generators = getattr(node, "generators", [])
locals_set: set[str] = set()
for generator in generators:
locals_set.update(extract_target_names(generator.target))
self.scopes.append(Scope(kind=kind, locals=locals_set))
if isinstance(node, ast.DictComp):
self.visit(node.key)
self.visit(node.value)
elif hasattr(node, "elt"):
self.visit(node.elt)
for generator in generators:
self.visit(generator.iter)
for if_node in generator.ifs:
self.visit(if_node)
self.scopes.pop()
def extract_target_names(target: ast.AST) -> set[str]:
names: set[str] = set()
if isinstance(target, ast.Name):
names.add(target.id)
elif isinstance(target, ast.Tuple | ast.List):
for item in target.elts:
names.update(extract_target_names(item))
elif isinstance(target, ast.Starred):
names.update(extract_target_names(target.value))
return names
def match_capture_names(pattern: ast.pattern) -> set[str]:
"""Names a match-case pattern binds — captures are locals, not external frames."""
names: set[str] = set()
if isinstance(pattern, ast.MatchAs):
if pattern.name is not None:
names.add(pattern.name)
if pattern.pattern is not None:
names |= match_capture_names(pattern.pattern)
elif isinstance(pattern, ast.MatchStar):
if pattern.name is not None:
names.add(pattern.name)
elif isinstance(pattern, ast.MatchMapping):
if pattern.rest is not None:
names.add(pattern.rest)
for sub in pattern.patterns:
names |= match_capture_names(sub)
elif isinstance(pattern, ast.MatchSequence | ast.MatchOr):
for sub in pattern.patterns:
names |= match_capture_names(sub)
elif isinstance(pattern, ast.MatchClass):
for sub in [*pattern.patterns, *pattern.kwd_patterns]:
names |= match_capture_names(sub)
return names
def annotation_to_string(annotation: ast.AST) -> str:
if hasattr(ast, "unparse"):
return ast.unparse(annotation)
return "unknown"
def dotted_name(value: ast.AST) -> str | None:
if isinstance(value, ast.Name):
return value.id
if isinstance(value, ast.Attribute):
base = dotted_name(value.value)
if base:
return f"{base}.{value.attr}"
return value.attr
return None
def infer_value_type(value: ast.AST) -> str:
if isinstance(value, ast.Constant):
if value.value is None:
return "None"
return type(value.value).__name__
if isinstance(value, ast.List | ast.ListComp):
return "list"
if isinstance(value, ast.Tuple):
return "tuple"
if isinstance(value, ast.Dict | ast.DictComp):
return "dict"
if isinstance(value, ast.Set | ast.SetComp):
return "set"
if isinstance(value, ast.GeneratorExp):
return "generator"
if isinstance(value, ast.Call):
call_name = dotted_name(value.func)
if call_name in {"list", "dict", "set", "tuple", "int", "float", "str", "bool"}:
return call_name
return "unknown"
def collect_exported_types(body: list[ast.stmt]) -> dict[str, str]:
exported_types: dict[str, str] = {}
for statement in body:
if isinstance(statement, ast.Assign):
type_name = infer_value_type(statement.value)
for target in statement.targets:
for name in extract_target_names(target):
exported_types[name] = type_name
elif isinstance(statement, ast.AnnAssign):
type_name = annotation_to_string(statement.annotation)
if type_name == "unknown" and statement.value:
type_name = infer_value_type(statement.value)
for name in extract_target_names(statement.target):
exported_types[name] = type_name
elif isinstance(statement, ast.AugAssign):
for name in extract_target_names(statement.target):
exported_types.setdefault(name, "unknown")
elif isinstance(statement, ast.Import):
for alias in statement.names:
name = alias.asname or alias.name.split(".")[0]
exported_types.setdefault(name, "module")
elif isinstance(statement, ast.ImportFrom):
for alias in statement.names:
name = alias.asname or alias.name
exported_types.setdefault(name, "module")
return exported_types
# IPython syntax a cell may use: `%magic`, `%%cell_magic`, `!shell`, and `x = !cmd` / `x = %magic`.
# `[ \t]` rather than `\s`: a pattern that matches newlines retries at every blank line, which is quadratic.
_IPYTHON_SYNTAX = re.compile(r"^[ \t]*[%!]|=[ \t]*[%!]", re.MULTILINE)
# Magics whose argument or body is Python, so the names it reads still count as inputs.
_PYTHON_BODY_CELL_MAGICS = frozenset({"time", "timeit", "prun", "capture"})
_PYTHON_ARGUMENT_LINE_MAGIC = re.compile(r"^([ \t]*)%(timeit|time|prun)\b(.*)$", re.MULTILINE)
# The single-letter options each magic reads a value for, as in `-n 10` or `-n10`.
_MAGIC_VALUE_OPTIONS: dict[str, frozenset[str]] = {
"timeit": frozenset("nrp"),
"prun": frozenset("lsTD"),
}
# Names the transformed source calls that IPython provides at runtime, never a sibling frame.
_IPYTHON_RUNTIME_NAMES = frozenset({"get_ipython"})
def _magic_statement(magic: str, arguments: str) -> str:
"""The Python statement after a magic's options: `-n10 -r 3 df.sum()` gives `df.sum()`."""
value_options = _MAGIC_VALUE_OPTIONS.get(magic, frozenset())
rest = arguments.strip()
while rest.startswith("-"):
token, _, rest = rest.partition(" ")
rest = rest.lstrip()
letters = token[1:]
for position, letter in enumerate(letters):
if letter in value_options:
# The value is the rest of the token, or else the next token.
if position == len(letters) - 1:
_, _, rest = rest.partition(" ")
rest = rest.lstrip()
break
return rest
def to_plain_python(code: str) -> str:
"""Rewrite the IPython syntax in a cell as the plain Python the kernel would run.
Cells run through IPython, so `%pip install x` or `!ls` is valid there but a SyntaxError
to `ast.parse`, which would drop every input the rest of the cell reads.
"""
if not _IPYTHON_SYNTAX.search(code):
return code
# A loop, not recursion: cell magics can stack, and a cell may hold thousands of them.
setup_statements: list[str] = []
while (stripped := code.lstrip()).startswith("%%"):
header, _, code = stripped.partition("\n")
magic, _, arguments = header[2:].strip().partition(" ")
# Any other cell magic (%%bash, %%html, …) holds another language: there is no Python to read.
if magic not in _PYTHON_BODY_CELL_MAGICS:
return ""
# `%%timeit` runs the statement after its options once as setup, before the body.
if magic == "timeit" and (setup := _magic_statement(magic, arguments)):
setup_statements.append(setup)
if setup_statements:
code = "\n".join([*setup_statements, code])
code = _PYTHON_ARGUMENT_LINE_MAGIC.sub(
lambda match: match.group(1) + _magic_statement(match.group(2), match.group(3)), code
)
from IPython.core.inputtransformer2 import TransformerManager # noqa: PLC0415 — keeps IPython off the import path
return TransformerManager().transform_cell(code)
def analyze_python_globals(code: str) -> PythonGlobalsAnalysis:
if not code or not code.strip():
return PythonGlobalsAnalysis(used=[], exported_with_types=[])
try:
tree = ast.parse(to_plain_python(code))
# Deep nesting in user code exhausts the parser's stack; that cell just has no analysis.
except (SyntaxError, RecursionError):
return PythonGlobalsAnalysis(used=[], exported_with_types=[])
module_locals = collect_scope_locals(tree.body)
builtins_names = set(dir(builtins))
analyzer = GlobalAnalyzer(module_locals, builtins_names)
analyzer.visit(tree)
exported_types = collect_exported_types(tree.body)
exported_with_types = [
{"name": name, "type": exported_types.get(name, "unknown")} for name in sorted(module_locals)
]
return PythonGlobalsAnalysis(
used=sorted(analyzer.used - _IPYTHON_RUNTIME_NAMES),
exported_with_types=exported_with_types,
)