Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: ask before Python deletes files through a passed function (#17541)

Nilay2b62ba145fb0
diff --git a/studio/backend/core/inference/tools.py b/studio/backend/core/inference/tools.py
index 9d360b5cdcb..ae96d3d13da 100644
--- a/studio/backend/core/inference/tools.py
+++ b/studio/backend/core/inference/tools.py
@@ -8479,7 +8479,9 @@ def _python_is_high_risk(code: str) -> bool:
if isinstance(tgt, ast.Name):
destructive_fs_aliases.add(tgt.id)
elif isinstance(node, ast.Assign) and isinstance(node.value, ast.Attribute):
- if _is_destructive_attr(node.value.attr, node.value.value):
+ if _is_destructive_attr(node.value.attr, node.value.value) or _is_process_kill(
+ node.value
+ ):
for tgt in node.targets:
if isinstance(tgt, ast.Name):
destructive_fs_aliases.add(tgt.id)
@@ -8489,7 +8491,9 @@ def _python_is_high_risk(code: str) -> bool:
and isinstance(node.target, ast.Name)
):
# An annotated binding (f: object = os.remove) is the same alias.
- if _is_destructive_attr(node.value.attr, node.value.value):
+ if _is_destructive_attr(node.value.attr, node.value.value) or _is_process_kill(
+ node.value
+ ):
destructive_fs_aliases.add(node.target.id)
for node in _tree_nodes(tree):
if not isinstance(node, ast.Call):
@@ -8527,6 +8531,40 @@ def _python_is_high_risk(code: str) -> bool:
return True
elif _is_destructive_attr(attr_name, func.args[0]):
return True
+ # Handed to map, partial or a thread, the function runs with no call node naming it.
+ bound = {
+ id(node.value)
+ for node in _tree_nodes(tree)
+ if (isinstance(node, ast.Assign) and all(isinstance(tgt, ast.Name) for tgt in node.targets))
+ or (
+ isinstance(node, ast.AnnAssign)
+ and isinstance(node.target, ast.Name)
+ and isinstance(node.value, ast.Attribute)
+ )
+ }
+ for node in _tree_nodes(tree):
+ if isinstance(node, ast.Name):
+ if isinstance(node.ctx, ast.Load) and node.id in destructive_fs_aliases:
+ return True
+ elif id(node) in bound:
+ continue
+ elif isinstance(node, ast.Attribute):
+ if isinstance(node.ctx, ast.Load) and (
+ _is_destructive_attr(node.attr, node.value) or _is_process_kill(node)
+ ):
+ return True
+ elif isinstance(node, ast.Subscript):
+ if _folded_str_literal(node.slice) is not None and _is_module_dict_lookup(node):
+ return True
+ elif (
+ isinstance(node, ast.Call)
+ and isinstance(node.func, ast.Name)
+ and node.func.id == "getattr"
+ and len(node.args) >= 2
+ ):
+ attr_name = _folded_str_literal(node.args[1])
+ if attr_name is not None and _is_destructive_attr(attr_name, node.args[0]):
+ return True
# Fold split paths through variables; unresolved fragments use a sentinel so partial folds
# never false-positive.
str_vars: "dict[str, str]" = {}
diff --git a/studio/backend/tests/test_permission_mode.py b/studio/backend/tests/test_permission_mode.py
index 40488f4a252..224d7ab4512 100644
--- a/studio/backend/tests/test_permission_mode.py
+++ b/studio/backend/tests/test_permission_mode.py
@@ -1559,6 +1559,69 @@ def test_shell_function_passed_by_reference_is_high_risk(code, refused):
assert (_check_code_safety(code) is not None) is refused
+ "code, high_risk",
+ [
+ ("import os\nlist(map(os.remove, ['/srv/data/x']))", True),
+ ("import shutil\nlist(map(shutil.rmtree, ['build']))", True),
+ ("import functools, os\nfunctools.partial(os.remove, 'x')()", True),
+ (
+ "import shutil, threading\nthreading.Thread(target=shutil.rmtree, args=('build',)).start()",
+ True,
+ ),
+ (
+ "import os\nfrom concurrent.futures import ThreadPoolExecutor\n"
+ "ThreadPoolExecutor().submit(os.remove, 'x')",
+ True,
+ ),
+ ("import atexit, shutil\natexit.register(shutil.rmtree, 'build')", True),
+ ("from pathlib import Path\nlist(map(Path.unlink, Path('.').glob('*.bin')))", True),
+ ("import os\ndef go(f=os.remove):\n f('x')\ngo()", True),
+ ("import os\nfor f in [os.unlink]:\n f('x')", True),
+ ("import os\nrm, n = os.remove, 1\nrm('x')", True),
+ ("import os\nrms = {'rm': os.remove}\nrms['rm']('x')", True),
+ ("import os\ndef pick():\n return os.remove\npick()('x')", True),
+ ("import os\nclass Box: ...\nrm = Box.rm = os.remove\nBox.rm('x')", True),
+ ("import os\nrm = os.remove\nlist(map(rm, ['x']))", True),
+ ("import os\nrm = os.remove\ngo = rm\ngo('x')", True),
+ ("from shutil import rmtree\nlist(map(rmtree, ['build']))", True),
+ ("from os import remove as rm\nlist(map(rm, ['x']))", True),
+ ("import os as fs\nlist(map(fs.remove, ['x']))", True),
+ ("import os\nlist(map(getattr(os, 'remove'), ['x']))", True),
+ ("import os\nlist(map(vars(os)['remove'], ['x']))", True),
+ ("import os\nlist(map(os.kill, [1234], [9]))", True),
+ ("import psutil\nlist(map(psutil.Process.kill, psutil.process_iter()))", True),
+ ("import psutil\nk = psutil.Process.kill\nlist(map(k, psutil.process_iter()))", True),
+ ("import psutil\nk = psutil.Process(5).kill\nk()", True),
+ ("import os\nprint(hasattr(os, 'remove'))", False),
+ ("import os\nprint('use os.remove or shutil.rmtree to delete')", False),
+ ("import os\nrm = os.remove\nprint('ready')", False),
+ ("from shutil import rmtree, copy\ncopy('a', 'b')", False),
+ ("import os\nlist(map(os.path.basename, ['a/b']))", False),
+ ("import os\nprint(getattr(os, 'sep'))", False),
+ ("import shutil\nfor name in dir(shutil):\n print(name, getattr(shutil, name))", False),
+ ("import os\nfor name in vars(os):\n print(name, vars(os)[name])", False),
+ ("items = [3, 1]\nlist(map(items.remove, [1]))", False),
+ ("def remove(p): ...\nlist(map(remove, ['x']))", False),
+ ],
+)
+def test_file_delete_function_passed_by_reference_is_high_risk(code, high_risk):
+ assert is_high_risk_tool_call("python", {"code": code}) is high_risk
+
+
+ "command, high_risk",
+ [
+ ("python3 -c 'import shutil; list(map(shutil.rmtree, [\"build\"]))'", True),
+ ("python3 -c 'import os, functools; functools.partial(os.remove, \"x\")()'", True),
+ ("python3 - <<'PY'\nimport os\nlist(map(os.remove, ['x']))\nPY", True),
+ ("python3 -c 'import os; print(hasattr(os, \"remove\"))'", False),
+ ],
+)
+def test_terminal_python_file_delete_by_reference_is_high_risk(command, high_risk):
+ assert is_high_risk_tool_call("terminal", {"command": command}) is high_risk
+
+
@pytest.mark.parametrize("code", ["value = " + "+".join(["1"] * 400), "value = data" + "[0]" * 400])
def test_long_expression_chain_is_classified(code):
import threading