Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(skills): verify installs by loadability and repair missing deps

wizardchenfaefceec1503
diff --git a/internal/application/service/tenant_skill_install.go b/internal/application/service/tenant_skill_install.go
index 0d0bcfb03..420685fc5 100644
--- a/internal/application/service/tenant_skill_install.go
+++ b/internal/application/service/tenant_skill_install.go
@@ -44,6 +44,17 @@ const (
// files. Writing each file with MakeDir+WriteFile is two round trips per
// entry, which is why a 50-file skill crawled through "seeding 12/56".
skillSeedArchivePath = sandbox.SkillsImageRoot + "/.weknora-seed.tar"
+
+ // skillInstallVerifyRounds bounds the installer conversation.
+ //
+ // The first round works from SKILL.md. A second is driven by the gate's own
+ // findings, and exists because those are two different descriptions of what
+ // a skill needs: the official office toolkit imports defusedxml and lxml at
+ // module level while its SKILL.md names neither, so an installer working
+ // from prose alone cannot know to install them. One round of reconciliation
+ // closes that gap; more would only be a retry loop over an answer that is
+ // not going to change.
+ skillInstallVerifyRounds = 2
)
// InstallSkill validates an uploaded archive, records it, and kicks off the
@@ -367,25 +378,16 @@ func (s *TenantSkillService) runInstall(
}
s.publishProgress(ctx, tenantID, configID, skillID, SkillProgress{Percent: 35, Stage: "seeded"})
- // 3. Let the installer agent install dependencies.
- if err := s.driveInstallerAgent(ctx, tenantID, skillID, sess, transcript, prompt); err != nil {
- return err
- }
- s.publishProgress(ctx, tenantID, configID, skillID, SkillProgress{Percent: 80, Stage: "agent_done"})
-
- // 4. Hand the tree to the execution user BEFORE verifying it. The agent
- // created these files as root, and the language passes below
- // deliberately run as the ordinary user: verifying first would test
- // permissions that never reach the image, and a restrictive root umask
- // would fail a perfectly good install because the .venv interpreter was
- // unreadable.
- if err := s.normalizeSkillPermissions(ctx, mgr, sess.ID, skillDir); err != nil {
- return err
- }
-
- // 5. Verify ourselves. "The agent said it worked" is a sentence, not
- // evidence, and this is the last gate before the image is switched.
- if err := s.verifySkill(ctx, mgr, sess.ID, skillDir, bundle); err != nil {
+ // 3-5. Install dependencies and verify the result. "The agent said it
+ // worked" is a sentence, not evidence, and this is the last gate
+ // before the image is switched — but the gate and the agent must not
+ // answer "what does this skill need" separately, so a failure it can
+ // still fix goes back to the same agent before the run is failed.
+ if err := s.installDependenciesAndVerify(ctx, installerJob{
+ tenantID: tenantID, configID: configID, skillID: skillID,
+ sess: sess, mgr: mgr, transcript: transcript,
+ prompt: prompt, skillDir: skillDir, bundle: bundle,
+ }); err != nil {
return err
}
if err := s.writeManifestEntry(ctx, mgr, sess.ID, skillID, bundle); err != nil {
@@ -690,19 +692,114 @@ func (s *TenantSkillService) beginInstallTranscript(
return transcript, prompt
}
-// driveInstallerAgent runs one installer conversation. It calls the engine
-// directly instead of going through sessionService.AgentQA, because AgentQA
-// swallows engine failures (it emits an error event and returns nil) and we
-// need a reliable signal before switching the image.
-func (s *TenantSkillService) driveInstallerAgent(
- ctx context.Context, tenantID uint64, skillID string, sess *types.Session,
- transcript *installTranscript, prompt string,
-) error {
+// installerJob is everything the install-and-verify step needs. It is a struct
+// because the step takes the union of what the agent side and the image side
+// each need, and a nine-parameter call tells the reader nothing.
+type installerJob struct {
+ tenantID uint64
+ configID string
+ skillID string
+ sess *types.Session
+ mgr sandbox.Manager
+ transcript *installTranscript
+ prompt string
+ skillDir string
+ bundle *SkillBundle
+}
+
+// installDependenciesAndVerify runs the installer conversation and the gate as
+// one decision.
+//
+// They used to be two, and that is what let a working skill be rejected: the
+// installer derived what to install from SKILL.md prose while the gate derived
+// what must resolve from the imports every file executes. Those disagree on any
+// skill whose library modules import something its documentation never names,
+// and the install died with the answer already in hand. The gate is the only
+// authority here, so its own findings become the next round's instruction, and
+// nothing in this path derives that list a second time.
+//
+// A failure the gate marked unrepairable — a syntax error, a file the execution
+// user cannot read — ends the run immediately. Another round cannot change it,
+// and the bundle has to change instead.
+func (s *TenantSkillService) installDependenciesAndVerify(
+ ctx context.Context, job installerJob,
+) (err error) {
+ run, err := s.openInstallerRun(ctx, job.tenantID, job.sess, job.transcript)
+ if err != nil {
+ job.transcript.Finish(context.WithoutCancel(ctx), err)
+ return err
+ }
+ // Detached from cancellation: when the install lock's renewal fails the
+ // context dies, and the transcript of the run that just died is precisely
+ // what someone will want to read.
+ defer func() { job.transcript.Finish(context.WithoutCancel(ctx), err) }()
+
+ prompt := job.prompt
+ for round := 1; ; round++ {
+ if err = run.round(ctx, prompt); err != nil {
+ return err
+ }
+ s.publishProgress(ctx, job.tenantID, job.configID, job.skillID,
+ SkillProgress{Percent: 80, Stage: "agent_done"})
+
+ // The tree is handed to the execution user BEFORE it is verified. The
+ // agent created these files as root, and the language passes
+ // deliberately run as the ordinary user: verifying first would test
+ // permissions that never reach the image, and a restrictive root umask
+ // would fail a perfectly good install because the .venv interpreter
+ // was unreadable.
+ if err = s.normalizeSkillPermissions(ctx, job.mgr, job.sess.ID, job.skillDir); err != nil {
+ return err
+ }
+
+ notes, verifyErr := s.verifySkill(ctx, job.mgr, job.sess.ID, job.skillDir, job.bundle)
+ s.reportVerificationNotes(ctx, job, notes)
+ if verifyErr == nil {
+ return nil
+ }
+
+ var gate *skillVerificationError
+ if round >= skillInstallVerifyRounds || !errors.As(verifyErr, &gate) || !gate.Repairable {
+ err = verifyErr
+ return err
+ }
+
+ logger.Infof(ctx, "[skill] %s failed %s verification with %d fixable finding(s); "+
+ "handing them back to the installer", job.skillID, gate.Language, len(gate.Problems))
+ s.publishProgress(ctx, job.tenantID, job.configID, job.skillID, SkillProgress{
+ Percent: 82, Stage: "repairing",
+ Log: fmt.Sprintf("%s verification found %d missing dependency/dependencies; "+
+ "asking the installer to add them", gate.Language, len(gate.Problems)),
+ })
+ if err = s.reopenSkillDirForRepair(ctx, job.mgr, job.sess.ID, job.skillDir); err != nil {
+ return err
+ }
+ prompt = buildRepairPrompt(job.skillDir, gate)
+ job.transcript.RecordPrompt(prompt)
+ }
+}
+
+// installerRun is one installer conversation, held open across rounds. A repair
+// has to reach the same root shell, in the same sandbox, and be readable in the
+// same transcript as the install it is repairing.
+type installerRun struct {
+ engine interfaces.AgentEngine
+ transcript *installTranscript
+ sessionID string
+}
+
+// openInstallerRun builds the engine the conversation runs on. It calls the
+// engine directly instead of going through sessionService.AgentQA, because
+// AgentQA swallows engine failures (it emits an error event and returns nil)
+// and we need a reliable signal before switching the image.
+func (s *TenantSkillService) openInstallerRun(
+ ctx context.Context, tenantID uint64, sess *types.Session, transcript *installTranscript,
+) (*installerRun, error) {
if s.installerAgents == nil {
- return errors.New("custom agent service is not configured")
+ return nil, errors.New("custom agent service is not configured")
}
if transcript == nil {
- return errors.New("install transcript was not seeded")
+ return nil, errors.New("install transcript was not seeded")
}
// The record is tenant-writable: updateBuiltinAgent lets a tenant persist a
// Config for any built-in ID, this one included. It is therefore read for
@@ -710,42 +807,129 @@ func (s *TenantSkillService) driveInstallerAgent(
// platform's own registry entry.
record, err := s.installerAgents.GetAgentByID(ctx, types.BuiltinSkillInstallerID)
if err != nil {
- return fmt.Errorf("load installer agent: %w", err)
+ return nil, fmt.Errorf("load installer agent: %w", err)
}
agentConfig := installerAgentConfig(installerAgentDefaults(ctx, tenantID), sess.SandboxConfigID)
chatModel, err := s.resolveInstallerModel(ctx, tenantID, record)
if err != nil {
- return err
+ return nil, err
}
engine, err := s.agents.CreateAgentEngine(
ctx, agentConfig, chatModel, nil, transcript.bus, sess.ID, transcript.assistantMessageID,
)
if err != nil {
- runErr := fmt.Errorf("create installer engine: %w", err)
- transcript.Finish(ctx, runErr)
- return runErr
+ return nil, fmt.Errorf("create installer engine: %w", err)
}
+ return &installerRun{
+ engine: engine, transcript: transcript, sessionID: sess.ID,
+ }, nil
+}
- state, err := engine.Execute(ctx, sess.ID, transcript.assistantMessageID, prompt, nil)
- runErr := err
- if runErr == nil && (state == nil || !state.IsComplete) {
- runErr = errors.New("installer agent stopped without completing")
+// round runs one installer turn.
+//
+// No conversation history is replayed. The engine is stateless across turns, so
+// a repair round could be handed the first round's transcript — but what the
+// gate reported is both shorter and more exact than anything that could be
+// inferred from it, and re-reading the round that already missed a dependency
+// is not what makes the next one find it.
+func (r *installerRun) round(ctx context.Context, prompt string) error {
+ state, err := r.engine.Execute(ctx, r.sessionID, r.transcript.assistantMessageID, prompt, nil)
+ if err != nil {
+ return fmt.Errorf("installer agent failed: %w", err)
}
- // Detached from cancellation: when the install lock's renewal fails the
- // context dies, and the transcript of the run that just died is precisely
- // what someone will want to read.
- transcript.Finish(context.WithoutCancel(ctx), runErr)
- if runErr != nil {
- if err != nil {
- return fmt.Errorf("installer agent failed: %w", err)
- }
- return runErr
+ if state == nil || !state.IsComplete {
+ return errors.New("installer agent stopped without completing")
+ }
+ return nil
+}
+
+// reportVerificationNotes surfaces what the gate noticed but did not refuse: an
+// auxiliary file that cannot load, a requirement whose environment marker
+// cannot be evaluated here, an import that resolves only from a directory the
+// skill ships no script in. None is evidence of a broken install, and all of
+// them are what someone debugging this skill later will wish they had been
+// told, so they are logged and streamed rather than discarded.
+func (s *TenantSkillService) reportVerificationNotes(
+ ctx context.Context, job installerJob, notes []string,
+) {
+ if len(notes) == 0 {
+ return
+ }
+ for _, note := range notes {
+ logger.Infof(ctx, "[skill] %s verification note: %s", job.skillID, note)
+ }
+ // One event, not one per note. Progress keeps only the latest value, so
+ // publishing them separately would leave a late subscriber holding whichever
+ // note happened to be last.
+ s.publishProgress(ctx, job.tenantID, job.configID, job.skillID, SkillProgress{
+ Percent: 80, Stage: "verify_note", Log: strings.Join(notes, "\n"),
+ })
+}
+
+// reopenSkillDirForRepair undoes normalizeSkillPermissions for another round.
+//
+// Verification runs as the ordinary user, so the tree is locked to 555 and root
+// before it; a repair then has to install into .venv again. Install commands
+// run as root and would get away with writing through those modes, but relying
+// on that makes the next reader work out whether it still holds — restoring the
+// state the seed left is the same two commands and needs no such argument.
+func (s *TenantSkillService) reopenSkillDirForRepair(
+ ctx context.Context, mgr sandbox.Manager, sessionID, skillDir string,
+) error {
+ if err := guardSkillDir(skillDir); err != nil {
+ return err
+ }
+ dir := sandbox.ShellQuote(skillDir)
+ user := sandbox.DefaultSandboxExecUser
+ cmd := fmt.Sprintf("chown -R %s:%s %s && chmod -R u+rwX,go+rX %s", user, user, dir, dir)
+ if _, err := s.execInstall(ctx, mgr, sessionID, cmd); err != nil {
+ return fmt.Errorf("reopen skill directory %s for a repair round: %w", skillDir, err)
}
return nil
}
+// buildRepairPrompt turns the gate's own findings into the next round's brief.
+//
+// It carries no analysis of its own, deliberately. The reason a first round can
+// fail on a working skill is that SKILL.md and the imports its files execute are
+// two different descriptions of what the skill needs; a third description
+// written here would reintroduce exactly that gap. The only thing added is the
+// import-name-to-distribution mapping, which is a property of PyPI rather than
+// of this skill.
+func buildRepairPrompt(skillDir string, gate *skillVerificationError) string {
+ var findings strings.Builder
+ for _, problem := range gate.Problems {
+ findings.WriteString("- ")
+ findings.WriteString(problem)
+ findings.WriteString("\n")
+ }
+ return fmt.Sprintf(`Verification of the skill you just installed failed. Fix only this and stop.
+
+The %s check reported:
+%s
+Every line above is a dependency this image cannot resolve. The names were read
+from the imports the skill's own files execute when they load, which is why some
+of them appear nowhere in SKILL.md — install them anyway.
+
+- Python packages go into %s/.venv (`+"`uv pip install`"+`, or
+ %s/.venv/bin/python -m pip install). Node packages go under %s/node_modules.
+- The name in each line is an IMPORT name; install the distribution that
+ provides it. Common cases where they differ: PIL -> pillow, yaml -> pyyaml,
+ docx -> python-docx, pptx -> python-pptx, cv2 -> opencv-python-headless,
+ bs4 -> beautifulsoup4, sklearn -> scikit-learn, fitz -> pymupdf.
+- Do NOT edit, move or delete any of the skill's own files, and do NOT edit
+ SKILL.md or requirements.txt to make the check pass. The uploaded archive is
+ what read_skill serves, so a source edit here makes the installed skill differ
+ from what everyone else sees.
+- If a package genuinely cannot be installed in this image, say so plainly in
+ your summary rather than working around it.
+
+The same verification runs again as soon as you finish.
+`, gate.Language, findings.String(), skillDir, skillDir, skillDir)
+}
+
// normalizeSkillPermissions makes the skill tree readable and executable by
// the non-root execution user, and writable by nobody. Installs run as root,
// so without the mode change the user that actually runs skills could not read
@@ -1526,6 +1710,13 @@ venv. It never runs the skill's code, so nothing is expected to answer --help.
Lazy imports and install_deps.py extras are invisible to that check — you still
have to install them.
+SKILL.md is not the dependency list. What the check resolves is the set of
+top-level imports the skill's files actually execute, and a library module
+routinely imports something the documentation never mentions. Read every .py in
+the tree (grep '^import\|^from' over it) and install what those lines need, not
+only what the Dependencies section names. If verification does fail on a missing
+package, you get its exact findings back and one round to install them.
+
SKILL.md:
%s
`, skillDir, uvAvailable, skillDir, skillDir, skillDir,
diff --git a/internal/application/service/tenant_skill_install_test.go b/internal/application/service/tenant_skill_install_test.go
index 62b33b7e0..f5f470979 100644
--- a/internal/application/service/tenant_skill_install_test.go
+++ b/internal/application/service/tenant_skill_install_test.go
@@ -1043,9 +1043,9 @@ func TestRunInstallRequiresVenvWhenRequirementsExist(t *testing.T) {
require.Nil(t, fx.configRepo.saved)
}
-// Verification used to run one guessed "entry script" with --help. Nothing in
-// the runtime designates an entry script — the model names whichever path it
-// likes in execute_skill_script — so the guess left every other file unchecked
+// Verification used to run one guessed "entry script" with --help. A skill is
+// not one entry point: it is every file the tree ships, some run as scripts
+// and some imported as packages. The guess left every other file unchecked
// while failing installs over the one it happened to pick.
func TestRunInstallVerifiesEveryScriptOfEveryLanguage(t *testing.T) {
fx := newInstallFixture(t)
@@ -1058,7 +1058,7 @@ func TestRunInstallVerifiesEveryScriptOfEveryLanguage(t *testing.T) {
pythonPass := skillPythonVerifyCommand(installSkillDir, []string{
"scripts/__init__.py", "scripts/extract.py", "scripts/helper.py",
- })
+ }, nil)
require.Contains(t, fx.commands, pythonPass,
"every python file must be checked, not the first one in sort order")
require.Contains(t, fx.commands,
@@ -1083,7 +1083,7 @@ func TestRunInstallDoesNotExecuteAnyScriptToVerifyIt(t *testing.T) {
}
func TestSkillPythonVerifyCommandPrefersTheSkillVenv(t *testing.T) {
- command := skillPythonVerifyCommand("/opt/skills/demo", []string{"scripts/a.py"})
+ command := skillPythonVerifyCommand("/opt/skills/demo", []string{"scripts/a.py"}, nil)
require.Contains(t, command, "if [ -x /opt/skills/demo/.venv/bin/python ]; "+
"then py=/opt/skills/demo/.venv/bin/python; else py=python3; fi",
@@ -1095,11 +1095,68 @@ func TestSkillPythonVerifyCommandPrefersTheSkillVenv(t *testing.T) {
// A skill may ship a file whose name needs quoting; the command is assembled
// by hand, so the shell must never see it as more than one word.
func TestSkillVerifyCommandsQuoteAwkwardPaths(t *testing.T) {
- python := skillPythonVerifyCommand("/opt/skills/demo", []string{"scripts/a b'c.py"})
+ python := skillPythonVerifyCommand("/opt/skills/demo", []string{"scripts/a b'c.py"}, nil)
require.Contains(t, python, `'scripts/a b'\''c.py'`)
shell := skillShellVerifyCommand("/opt/skills/demo", []string{"a b.sh"})
- require.Equal(t, `for f in '/opt/skills/demo/a b.sh'; do sh -n "$f" || exit 1; done`, shell)
+ require.Equal(t,
+ `if command -v bash >/dev/null 2>&1; then parser=bash; else parser=sh; fi; `+
+ `for f in '/opt/skills/demo/a b.sh'; do "$parser" -n "$f" || exit 1; done`,
+ shell)
+}
+
+// A skill's shell scripts carry a bash shebang almost exclusively, and
+// SkillInterpreterCommand runs them with bash. Checking them with sh — dash on
+// Debian — refused installs over array literals, `function f()`, C-style for
+// loops and process substitution, none of which dash can parse.
+func TestSkillShellVerifyCommandParsesWithBash(t *testing.T) {
+ command := skillShellVerifyCommand("/opt/skills/demo", []string{"bin/setup.sh"})
+
+ require.Contains(t, command, "parser=bash")
+ require.Contains(t, command, `"$parser" -n "$f"`)
+ require.Contains(t, command, "else parser=sh",
+ "an image without bash must still get a syntax check")
+ require.NotContains(t, command, `sh -n "$f"`,
+ "the check must not hard-code the shell that cannot parse these files")
+}
+
+// The Python pass is the only one whose verdict depends on what the image
+// carries, so it is the only one that can fail an install over a file nothing
+// the skill offers ever loads. Those files are named separately.
+func TestSkillPythonVerifyCommandSeparatesAuxiliaryFiles(t *testing.T) {
+ command := skillPythonVerifyCommand("/opt/skills/demo",
+ []string{"scripts/run.py"}, []string{"tests/test_run.py"})
+
+ require.True(t, strings.HasSuffix(command,
+ `- /opt/skills/demo scripts/run.py --optional tests/test_run.py`),
+ "got %q", command)
+}
+
+// The split follows the conventions the language ecosystems already share. A
+// bundled tests/ directory imports pytest and an examples/ script imports
+// whatever it illustrates; neither is reachable from anything the skill
+// exposes, and neither may decide whether the skill installs.
+func TestSkillAuxiliaryScriptFollowsEcosystemConventions(t *testing.T) {
+ for _, rel := range []string{
+ "tests/test_run.py", "test/helpers.py", "examples/demo.py",
+ "scripts/__tests__/a.py", "benchmarks/bench.py", "docs/conf.py",
+ "conftest.py", "setup.py", "scripts/extract_test.py",
+ "scripts/test_extract.py",
+ } {
+ require.True(t, skillAuxiliaryScript(rel), "%s must be auxiliary", rel)
+ }
+ for _, rel := range []string{
+ "scripts/extract.py", "scripts/office/validators/base.py",
+ "recalc.py", "scripts/latest.py", "scripts/contest.py",
+ } {
+ require.False(t, skillAuxiliaryScript(rel), "%s is part of what the skill offers", rel)
+ }
+
+ entry, auxiliary := splitAuxiliaryScripts([]string{
+ "scripts/a.py", "tests/test_a.py", "scripts/b.py",
+ })
+ require.Equal(t, []string{"scripts/a.py", "scripts/b.py"}, entry)
+ require.Equal(t, []string{"tests/test_a.py"}, auxiliary)
}
func TestSkillNodeVerifyCommandChecksDeclaredDependencies(t *testing.T) {
@@ -1307,7 +1364,8 @@ func TestRunInstallKeepsOldImageWhenVerificationFails(t *testing.T) {
fx.configRepo.entity.Config.SkillImage = &types.SkillImageConfig{
SnapshotID: "snap-old", Generation: 3, OwnerFingerprint: fx.fingerprint,
}
- fx.loadCheckExitCode = 1
+ fx.loadCheckExitCodes = []int{1}
+ fx.loadCheckStderr = "scripts/extract.py has a syntax error on line 1: invalid syntax"
err := fx.svc.runInstall(context.Background(), 7, "cfg-1", "sk-1", fx.bundle)
@@ -1322,6 +1380,98 @@ func TestRunInstallKeepsOldImageWhenVerificationFails(t *testing.T) {
require.NotEmpty(t, skill.Error)
}
+// The failure this closes: the installer derives what to install from SKILL.md
+// prose, and the gate derives what must resolve from the imports every file
+// executes. Those disagree on any skill whose library modules import something
+// its documentation never names — the official office toolkit imports
+// defusedxml and lxml at module level and mentions neither — so the install
+// died with the answer already in hand, and a retry replayed the same prompt to
+// the same effect. The gate's own lines are the repair round's brief.
+func TestRunInstallHandsVerificationFindingsBackToTheInstaller(t *testing.T) {
+ fx := newInstallFixture(t)
+ fx.loadCheckExitCodes = []int{skillVerifyRepairableExit, 0}
+ fx.loadCheckStderr = "scripts/office/validators/base.py imports defusedxml, " +
+ "which is not available in this image\n" +
+ "scripts/recalc.py imports openpyxl, which is not available in this image"
+
+ require.NoError(t, fx.svc.runInstall(context.Background(), 7, "cfg-1", "sk-1", fx.bundle))
+
+ require.Len(t, fx.agentPrompts, 2, "a fixable failure must reach the installer again")
+ repair := fx.agentPrompts[1]
+ require.Contains(t, repair, "imports defusedxml",
+ "the repair round is driven by the gate's own findings, not by a second guess")
+ require.Contains(t, repair, "imports openpyxl")
+ require.Contains(t, repair, installSkillDir+"/.venv",
+ "the round has to be told where the packages belong")
+ require.Contains(t, repair, "Do NOT edit",
+ "a repair must not be allowed to edit the tree into passing")
+
+ require.Equal(t, 2, fx.loadCheckPasses, "the repair has to be verified, not trusted")
+ require.Contains(t, fx.events, "create-snapshot")
+ require.NotNil(t, fx.configRepo.saved, "a repaired install must reach the image pointer")
+
+ skill, _ := fx.skillRepo.GetSkill(context.Background(), 7, "cfg-1", "sk-1")
+ require.Equal(t, types.SkillStatusReady, skill.Status)
+ require.Empty(t, skill.Error)
+}
+
+// A repair round has to be able to write into .venv again, which the
+// verification pass just locked down: the checks run as the ordinary user, so
+// the tree is handed over at 555 and root-owned before every one of them.
+func TestRunInstallReopensTheTreeBeforeARepairRound(t *testing.T) {
+ fx := newInstallFixture(t)
+ fx.loadCheckExitCodes = []int{skillVerifyRepairableExit, 0}
+
+ require.NoError(t, fx.svc.runInstall(context.Background(), 7, "cfg-1", "sk-1", fx.bundle))
+
+ reopen := indexOfCommandContaining(fx.commands, "chmod -R u+rwX,go+rX "+installSkillDir)
+ require.GreaterOrEqual(t, reopen, 0, "the tree must be writable again for the repair")
+
+ firstLock := indexOfCommandContaining(fx.commands, "chmod -R 555 "+installSkillDir)
+ require.GreaterOrEqual(t, firstLock, 0)
+ require.Greater(t, reopen, firstLock,
+ "the tree is reopened after the pass that locked it, not before")
+
+ // And locked again for the pass that checks the repair, so the final image
+ // carries the modes every verification ran against.
+ require.Greater(t, lastIndexOfCommandContaining(fx.commands, "chmod -R 555 "+installSkillDir),
+ reopen, "the repaired tree must be locked down again before it is verified")
+}
+
+// Installing a package cannot fix a file that does not parse, and the bundle
+// has to change instead. Spending another agent round on it would only delay
+// the same failure by minutes.
+func TestRunInstallDoesNotRetryAFailureInstallingCannotFix(t *testing.T) {
+ fx := newInstallFixture(t)
+ fx.loadCheckExitCodes = []int{1}
+ fx.loadCheckStderr = "scripts/extract.py has a syntax error on line 1: invalid syntax"
+
+ err := fx.svc.runInstall(context.Background(), 7, "cfg-1", "sk-1", fx.bundle)
+
+ require.Error(t, err)
+ require.ErrorContains(t, err, "syntax error")
+ require.Len(t, fx.agentPrompts, 1, "an unfixable finding must not buy another round")
+ require.Equal(t, 1, fx.loadCheckPasses)
+}
+
+// The loop is bounded. An installer that cannot satisfy the gate in one repair
+// is not going to satisfy it in ten, and a skill install must not become an
+// open-ended retry against a billed sandbox.
+func TestRunInstallStopsAfterOneRepairRound(t *testing.T) {
+ fx := newInstallFixture(t)
+ fx.loadCheckExitCodes = []int{skillVerifyRepairableExit}
+
+ err := fx.svc.runInstall(context.Background(), 7, "cfg-1", "sk-1", fx.bundle)
+
+ require.Error(t, err)
+ require.ErrorContains(t, err, "imports pandas",
+ "the failure must name what the gate found, not just that it failed")
+ require.Len(t, fx.agentPrompts, skillInstallVerifyRounds)
+ require.Equal(t, skillInstallVerifyRounds, fx.loadCheckPasses)
+ require.NotContains(t, fx.events, "create-snapshot")
+ require.Nil(t, fx.configRepo.saved)
+}
+
func TestRunInstallDeletesTheSnapshotWhenSwitchFails(t *testing.T) {
fx := newInstallFixture(t)
fx.configRepo.updateErr = errUpdateBoom
@@ -1549,7 +1699,7 @@ const (
// reason; the properties worth stating verbatim are asserted separately in
// TestSkillPythonVerifyCommand*.
var installPythonVerifyCommand = skillPythonVerifyCommand(
- installSkillDir, []string{"scripts/extract.py"},
+ installSkillDir, []string{"scripts/extract.py"}, nil,
)
func indexOfEvent(events []string, needle string) int {
@@ -1561,6 +1711,27 @@ func indexOfEvent(events []string, needle string) int {
return -1
}
+// indexOfCommandContaining and its Last variant locate one shell command in the
+// ordered transcript. A repair round issues the same commands twice, so the
+// tests that care about ordering need both ends.
+func indexOfCommandContaining(commands []string, needle string) int {
+ for i, command := range commands {
+ if strings.Contains(command, needle) {
+ return i
+ }
+ }
+ return -1
+}
+
+func lastIndexOfCommandContaining(commands []string, needle string) int {
+ for i := len(commands) - 1; i >= 0; i-- {
+ if strings.Contains(commands[i], needle) {
+ return i
+ }
+ }
+ return -1
+}
+
// staleMark is one request to mark a config's bound sandboxes stale. The
// tenant is part of it because marking the right config of the wrong workspace
// would rebuild sandboxes that never carried this image.
@@ -1584,10 +1755,17 @@ type installFixture struct {
events []string
commands []string
fingerprint string
- // loadCheck* drive the per-language script verification pass, which is
- // the last gate before the snapshot.
- loadCheckExitCode int
- loadCheckResult *sandbox.ExecuteResult
+ // loadCheck* drive the per-language script verification pass, which is the
+ // last gate before the snapshot. exitCodes is consumed one entry per python
+ // pass and its last entry repeats, so a test about a single round writes one
+ // value and a test about the repair round writes two. Exit 2 is the
+ // checker's "everything I found is a missing dependency", which is what
+ // earns another installer round.
+ loadCheckExitCodes []int
+ loadCheckStdout string
+ loadCheckStderr string
+ loadCheckPasses int
+ loadCheckResult *sandbox.ExecuteResult
// depsExitCode fails the declared-dependency check (venv / node_modules).
depsExitCode int
// execResult is scoped to execResultCommand: an unscoped stub result
@@ -1597,6 +1775,10 @@ type installFixture struct {
execResult *sandbox.ExecuteResult
loadCheckRanAsRoot bool
agentErr error
+ // agentPrompts is every prompt the installer engine was handed, in order.
+ // A repair round is a second entry, and what it says is the whole point of
+ // having one.
+ agentPrompts []string
// beforeExecute runs at the moment the engine would start, so a test can
// observe the state an attaching console would see mid-install.
beforeExecute func()
@@ -1657,6 +1839,10 @@ func newInstallFixture(t *testing.T) *installFixture {
fx := &installFixture{t: t}
fx.fingerprint = sandbox.SkillImageFingerprint("e2b", "key-1", "https://e2b.example")
+ // The checker's own wording for a missing dependency, so a test reading the
+ // repair prompt sees what a real run would put in it.
+ fx.loadCheckStderr = "scripts/extract.py imports pandas, " +
+ "which is not available in this image"
fx.bundle = &SkillBundle{
Name: "pdf-tools",
Version: "1.0.0",
@@ -1721,6 +1907,19 @@ func (f *installFixture) record(event string) {
f.events = append(f.events, event)
}
+// nextLoadCheckExit returns the code for this verification pass, repeating the
+// last configured entry once they run out.
+func (f *installFixture) nextLoadCheckExit() int {
+ f.loadCheckPasses++
+ if len(f.loadCheckExitCodes) == 0 {
+ return 0
+ }
+ if f.loadCheckPasses > len(f.loadCheckExitCodes) {
+ return f.loadCheckExitCodes[len(f.loadCheckExitCodes)-1]
+ }
+ return f.loadCheckExitCodes[f.loadCheckPasses-1]
+}
+
// now is the fixture's clock, so a test can express "one heartbeat ago"
// against the same instant the service reads.
func (f *installFixture) now() time.Time { return f.svc.now() }
@@ -2558,7 +2757,9 @@ func (m *installSandboxManager) ExecShellCommandWithOptions(
return m.fx.loadCheckResult, nil
}
return &sandbox.ExecuteResult{
- ExitCode: m.fx.loadCheckExitCode, Stderr: "scripts/extract.py imports pandas",
+ ExitCode: m.fx.nextLoadCheckExit(),
+ Stdout: m.fx.loadCheckStdout,
+ Stderr: m.fx.loadCheckStderr,
}, nil
case command == removeSkillDirCommand:
m.fx.record("remove-skill-dir")
@@ -2741,16 +2942,17 @@ type installAgentEngine struct {
}
func (e *installAgentEngine) Execute(
- context.Context,
- string,
- string,
- string,
- []chat.Message,
- ...[]string,
+ _ context.Context,
+ _ string,
+ _ string,
+ prompt string,
+ _ []chat.Message,
+ _ ...[]string,
) (*types.AgentState, error) {
if e.fx.beforeExecute != nil {
e.fx.beforeExecute()
}
+ e.fx.agentPrompts = append(e.fx.agentPrompts, prompt)
e.fx.record("agent-execute")
if e.fx.agentDelay > 0 {
time.Sleep(e.fx.agentDelay)
diff --git a/internal/application/service/tenant_skill_transcript.go b/internal/application/service/tenant_skill_transcript.go
index 883a60630..ed34d02ae 100644
--- a/internal/application/service/tenant_skill_transcript.go
+++ b/internal/application/service/tenant_skill_transcript.go
@@ -38,11 +38,17 @@ type installTranscript struct {
sessionID string
assistantMessageID string
- mu sync.Mutex
- message *types.Message
- answers []*installAnswerSegment
- starts map[string]time.Time
- finished bool
+ mu sync.Mutex
+ message *types.Message
+ answers []*installAnswerSegment
+ starts map[string]time.Time
+ // closed guards the terminal event and the final write, both of which say
+ // "this install is over" and must be said once.
+ closed bool
+ // Totals accumulated across the run's engine turns, reported on the one
+ // terminal event Finish emits.
+ totalSteps int
+ totalDurationMs int64
}
// installAnswerSegment accumulates the prose streamed under one final-answer
@@ -142,13 +148,20 @@ func (tr *installTranscript) Subscribe() {
tr.bus.On(event.EventAgentComplete, tr.onComplete)
}
-// Finish closes the record. runErr is the engine's verdict: the engine emits
-// no complete event when it fails, and a failed install is the one people
-// actually come to read, so the failure is written here rather than hoped for.
+// Finish closes the record. runErr is the install's verdict, not just the
+// engine's: verification runs after the agent stops, and a failure there is the
+// one people actually come to read, so it is written here rather than hoped for.
func (tr *installTranscript) Finish(ctx context.Context, runErr error) {
if tr == nil {
return
}
+ tr.mu.Lock()
+ alreadyClosed := tr.closed
+ tr.closed = true
+ tr.mu.Unlock()
+ if alreadyClosed {
+ return
+ }
if runErr != nil {
tr.append(interfaces.StreamEvent{
ID: uuid.NewString(),
@@ -173,20 +186,39 @@ func (tr *installTranscript) Finish(ctx context.Context, runErr error) {
}
tr.mu.Lock()
- alreadyComplete := tr.finished
- tr.finished = true
+ steps, duration := tr.totalSteps, tr.totalDurationMs
tr.mu.Unlock()
+ tr.append(interfaces.StreamEvent{
+ ID: uuid.NewString(),
+ Type: types.ResponseTypeComplete,
+ Done: true,
+ Timestamp: time.Now(),
+ Data: map[string]interface{}{
+ "total_steps": steps,
+ "total_duration_ms": duration,
+ },
+ })
+ tr.save(ctx)
+}
- if !alreadyComplete {
- tr.append(interfaces.StreamEvent{
- ID: uuid.NewString(),
- Type: types.ResponseTypeComplete,
- Done: true,
- Timestamp: time.Now(),
- Data: map[string]interface{}{},
- })
+// RecordPrompt logs a follow-up instruction the installer was given mid-run.
+//
+// The transcript exists so one replay of the event log is the whole
+// conversation. A repair round whose instruction is missing reads as the agent
+// spontaneously deciding to install more packages, which is precisely the
+// moment someone is reading this to find out why.
+func (tr *installTranscript) RecordPrompt(prompt string) {
+ if tr == nil {
+ return
}
- tr.save(ctx)
+ tr.append(interfaces.StreamEvent{
+ ID: uuid.NewString(),
+ Type: types.ResponseTypeInstallPrompt,
+ Content: prompt,
+ Done: true,
+ Timestamp: time.Now(),
+ Data: map[string]interface{}{},
+ })
}
func (tr *installTranscript) onThought(_ context.Context, evt event.Event) error {
@@ -326,13 +358,16 @@ func (tr *installTranscript) onError(_ context.Context, evt event.Event) error {
return nil
}
+// onComplete records what the round finished with. It deliberately emits no
+// terminal event: an install may run another installer round after
+// verification, and a console that saw "complete" would stop following before
+// that round began. Only Finish closes the stream.
func (tr *installTranscript) onComplete(_ context.Context, evt event.Event) error {
data, ok := evt.Data.(event.AgentCompleteData)
if !ok {
return nil
}
tr.mu.Lock()
- tr.finished = true
if data.MessageID == tr.assistantMessageID {
msg := tr.ensureMessageLocked()
msg.IsCompleted = true
@@ -347,18 +382,11 @@ func (tr *installTranscript) onComplete(_ context.Context, evt event.Event) erro
if tr.composeAnswerLocked() == "" && data.FinalAnswer != "" {
tr.segmentLocked(evt.ID).content = data.FinalAnswer
}
+ // Summed rather than overwritten: an install that needed a repair round ran
+ // two engine turns, and its cost is both of them.
+ tr.totalSteps += data.TotalSteps
+ tr.totalDurationMs += data.TotalDurationMs
tr.mu.Unlock()
-
- tr.append(interfaces.StreamEvent{
- ID: evt.ID,
- Type: types.ResponseTypeComplete,
- Done: true,
- Timestamp: time.Now(),
- Data: map[string]interface{}{
- "total_steps": data.TotalSteps,
- "total_duration_ms": data.TotalDurationMs,
- },
- })
return nil
}
diff --git a/internal/application/service/tenant_skill_transcript_test.go b/internal/application/service/tenant_skill_transcript_test.go
index 327960012..ddce68585 100644
--- a/internal/application/service/tenant_skill_transcript_test.go
+++ b/internal/application/service/tenant_skill_transcript_test.go
@@ -265,3 +265,56 @@ func TestInstallTranscriptCreateOpensTheLogWithThePrompt(t *testing.T) {
require.Equal(t, []types.ResponseType{types.ResponseTypeInstallPrompt}, streams.types())
require.Equal(t, "install pdf-tools", streams.events[0].Content)
}
+
+// An install may run a second installer round, because verification happens
+// after the agent stops and a dependency it names is something the agent can
+// still fix. The engine reports "complete" at the end of every turn, so the
+// transcript must not treat the first one as the end of the install — a console
+// that saw it would stop following before the repair round began.
+func TestInstallTranscriptStaysOpenAcrossInstallerRounds(t *testing.T) {
+ tr, bus, streams, messages := newTranscriptForTest(t)
+ ctx := context.Background()
+
+ finishRound := func(steps int, ms int64) {
+ require.NoError(t, bus.Emit(ctx, event.Event{
+ ID: "done", Type: event.EventAgentComplete,
+ Data: event.AgentCompleteData{
+ MessageID: "msg-1", TotalSteps: steps, TotalDurationMs: ms,
+ },
+ }))
+ }
+
+ finishRound(3, 1000)
+ tr.RecordPrompt("Verification failed. Install defusedxml into the venv.")
+ finishRound(2, 500)
+ tr.Finish(ctx, nil)
+
+ kinds := streams.types()
+ require.Equal(t, []types.ResponseType{
+ types.ResponseTypeInstallPrompt,
+ types.ResponseTypeComplete,
+ }, kinds, "the repair instruction has to be in the log, and the install ends once")
+ require.Contains(t, streams.events[0].Content, "Install defusedxml")
+
+ terminal := streams.events[len(streams.events)-1]
+ require.EqualValues(t, 5, terminal.Data["total_steps"],
+ "an install that needed a repair round cost both turns")
+ require.EqualValues(t, 1500, terminal.Data["total_duration_ms"])
+ require.Len(t, messages.updated, 1)
+}
+
+// Finish is deferred by the install path and also called on the error path that
+// gives up before that defer is armed. Saying "this install is over" twice would
+// duplicate the verdict in the record people read.
+func TestInstallTranscriptFinishIsIdempotent(t *testing.T) {
+ tr, _, streams, messages := newTranscriptForTest(t)
+
+ tr.Finish(context.Background(), errors.New("python verification failed: boom"))
+ tr.Finish(context.Background(), errors.New("python verification failed: boom"))
+
+ require.Equal(t, []types.ResponseType{
+ types.ResponseTypeError,
+ types.ResponseTypeComplete,
+ }, streams.types())
+ require.Len(t, messages.updated, 1)
+}
diff --git a/internal/application/service/tenant_skill_verify.go b/internal/application/service/tenant_skill_verify.go
index 89df23ef0..1e63645ff 100644
--- a/internal/application/service/tenant_skill_verify.go
+++ b/internal/application/service/tenant_skill_verify.go
@@ -20,28 +20,66 @@ import (
//go:embed tenant_skill_verify.py
var skillPythonVerifier string
+// skillVerifyRepairableExit is the exit code a verification pass uses when
+// every problem it found is a dependency missing from this image. It separates
+// "another installer round can fix this" from "the bundle has to change", which
+// is the only distinction that decides what the install flow does next.
+const skillVerifyRepairableExit = 2
+
+// skillVerifyNotePrefix marks a line the checker reports without refusing the
+// install. Notes travel on stdout so a non-zero exit stays unambiguous.
+const skillVerifyNotePrefix = "note: "
+
+// skillVerificationError is what the gate said, kept structured because it is
+// also the brief for a repair round. The gate is the only authority on what has
+// to resolve in this image, so handing back its own lines is what keeps the
+// installer from deriving "what this skill needs" a second time.
+type skillVerificationError struct {
+ // Language names the pass that failed, as the operator sees it.
+ Language string
+ // Repairable is true when installing a package would satisfy every line.
+ Repairable bool
+ // Problems are the checker's own lines, one per finding.
+ Problems []string
+ // Summary describes the command result itself, and is the only thing worth
+ // printing when the pass died without producing lines at all.
+ Summary string
+}
+
+func (e *skillVerificationError) Error() string {
+ if len(e.Problems) == 0 {
+ return fmt.Sprintf("%s verification failed (%s)", e.Language, e.Summary)
+ }
+ return fmt.Sprintf("%s verification failed: %s", e.Language, strings.Join(e.Problems, "; "))
+}
+
// verifySkill is the server's own check, and the last gate before the image
// pointer moves: a broken install must leave the previous snapshot serving.
//
-// It deliberately proves loadability rather than behaviour. Nothing in the
-// runtime designates an entry point — the model names any script it likes in
-// execute_skill_script — so there is no single command whose success would
-// stand in for the skill working. What every one of those calls does need is
-// that the interpreter can parse the file and resolve its imports inside this
-// image, and that is checkable for all scripts at once without executing a
-// line of the skill's own code.
+// It proves loadability, not behaviour, and not "every file can be used as
+// python this_file.py". A skill ships scripts and library packages together;
+// execute_skill_script names files, but those files are also imported by each
+// other. The Python pass resolves each file's imports against the prefixes
+// Python itself would use — the file's own directory and every ancestor up to
+// the skill root — plus the venv. Guessing a single entry script, or treating
+// every .py as an isolated __main__, both fail real skills.
+//
+// Findings are graded rather than uniformly fatal. Refusing an install costs
+// the minutes of dependency work that already succeeded, so only evidence that
+// the install itself is broken may do it: a file nothing the skill offers ever
+// loads, or a requirement pip would have skipped, is returned as a note.
//
// Not executing is the point, not a limitation. The previous implementation
// ran one guessed script with --help; skills that do not parse arguments
// simply ran their whole main path inside the tree about to be snapshotted.
func (s *TenantSkillService) verifySkill(
ctx context.Context, mgr sandbox.Manager, sessionID, skillDir string, bundle *SkillBundle,
-) error {
+) ([]string, error) {
if err := s.verifySkillTree(ctx, mgr, sessionID, skillDir, bundle); err != nil {
- return err
+ return nil, err
}
if err := s.verifyDeclaredDependencies(ctx, mgr, sessionID, skillDir, bundle); err != nil {
- return err
+ return nil, err
}
return s.verifyScriptsLoad(ctx, mgr, sessionID, skillDir, bundle)
}
@@ -93,30 +131,43 @@ func (s *TenantSkillService) verifyDeclaredDependencies(
}
// verifyScriptsLoad runs one pass per language present in the bundle. Each
-// pass covers every script of that language, because every one of them is a
-// script the model may name at runtime.
+// pass covers every file of that language: syntax and imports are properties
+// of the file, not of a guessed entry point.
+//
+// Only the Python pass takes the auxiliary split. Its findings depend on what
+// the image carries, so a bundled tests/ directory can fail an install that
+// works; `node --check` and `bash -n` only state whether a file parses, which
+// no amount of installing changes either way.
func (s *TenantSkillService) verifyScriptsLoad(
ctx context.Context, mgr sandbox.Manager, sessionID, skillDir string, bundle *SkillBundle,
-) error {
+) ([]string, error) {
+ var notes []string
if scripts := sortedScriptPaths(bundle, ".py"); len(scripts) > 0 {
- if err := s.execVerify(ctx, mgr, sessionID, skillDir, "python",
- skillPythonVerifyCommand(skillDir, scripts)); err != nil {
- return err
+ entry, auxiliary := splitAuxiliaryScripts(scripts)
+ found, err := s.execVerify(ctx, mgr, sessionID, skillDir, "python",
+ skillPythonVerifyCommand(skillDir, entry, auxiliary))
+ notes = append(notes, found...)
+ if err != nil {
+ return notes, err
}
}
if scripts := sortedScriptPaths(bundle, ".js", ".mjs", ".cjs"); len(scripts) > 0 {
- if err := s.execVerify(ctx, mgr, sessionID, skillDir, "node",
- skillNodeVerifyCommand(skillDir, scripts, nodeDependencyNames(bundle))); err != nil {
- return err
+ found, err := s.execVerify(ctx, mgr, sessionID, skillDir, "node",
+ skillNodeVerifyCommand(skillDir, scripts, nodeDependencyNames(bundle)))
+ notes = append(notes, found...)
+ if err != nil {
+ return notes, err
}
}
if scripts := sortedScriptPaths(bundle, ".sh"); len(scripts) > 0 {
- if err := s.execVerify(ctx, mgr, sessionID, skillDir, "shell",
- skillShellVerifyCommand(skillDir, scripts)); err != nil {
- return err
+ found, err := s.execVerify(ctx, mgr, sessionID, skillDir, "shell",
+ skillShellVerifyCommand(skillDir, scripts))
+ notes = append(notes, found...)
+ if err != nil {
+ return notes, err
}
}
- return nil
+ return notes, nil
}
// execVerify runs one verification pass as the ordinary sandbox user, with the
@@ -124,10 +175,10 @@ func (s *TenantSkillService) verifyScriptsLoad(
// install-mode root would test permissions that never reach a session.
func (s *TenantSkillService) execVerify(
ctx context.Context, mgr sandbox.Manager, sessionID, skillDir, label, command string,
-) error {
+) ([]string, error) {
executor, err := installExecutor(mgr)
if err != nil {
- return err
+ return nil, err
}
res, err := executor.ExecShellCommandWithOptions(ctx, sessionID, command,
sandbox.ShellExecOptions{
@@ -139,12 +190,46 @@ func (s *TenantSkillService) execVerify(
},
})
if err != nil {
- return fmt.Errorf("%s verification: %w", label, err)
+ return nil, fmt.Errorf("%s verification: %w", label, err)
}
+ // Notes are read even from a failed pass: an install refused for a missing
+ // package may also have noticed something about a file it did not refuse
+ // over, and that is exactly the context whoever reads the failure needs.
+ notes := verificationNotes(res.Stdout)
if res.ExitCode != 0 {
- return fmt.Errorf("%s verification failed (%s)", label, describeExecFailure(res))
+ return notes, &skillVerificationError{
+ Language: label,
+ Repairable: res.ExitCode == skillVerifyRepairableExit,
+ Problems: verificationProblems(res.Stderr),
+ Summary: describeExecFailure(res),
+ }
}
- return nil
+ return notes, nil
+}
+
+// verificationNotes pulls the reported-but-not-refused findings out of stdout.
+func verificationNotes(stdout string) []string {
+ var notes []string
+ for _, line := range strings.Split(stdout, "\n") {
+ line = strings.TrimSpace(line)
+ if after, ok := strings.CutPrefix(line, skillVerifyNotePrefix); ok {
+ notes = append(notes, after)
+ }
+ }
+ return notes
+}
+
+// verificationProblems splits a failed pass's stderr into its findings. Blank
+// lines are dropped; everything else is the checker's own wording, which is
+// what a repair round is given.
+func verificationProblems(stderr string) []string {
+ var problems []string
+ for _, line := range strings.Split(stderr, "\n") {
+ if line = strings.TrimSpace(line); line != "" {
+ problems = append(problems, line)
+ }
+ }
+ return problems
}
// skillPythonVerifyCommand pipes the verifier into the same interpreter a
@@ -152,13 +237,23 @@ func (s *TenantSkillService) execVerify(
// line. The list is explicit rather than a directory walk so the pass covers
// exactly the uploaded sources — never .venv, node_modules or anything else
// the agent created while installing.
-func skillPythonVerifyCommand(skillDir string, scripts []string) string {
+//
+// Auxiliary files are named after --optional. They are checked the same way,
+// and what is found in them is reported rather than allowed to refuse the
+// install.
+func skillPythonVerifyCommand(skillDir string, entry, auxiliary []string) string {
venv := path.Join(skillDir, ".venv", "bin", "python")
quotedVenv := sandbox.ShellQuote(venv)
args := []string{sandbox.ShellQuote(skillDir)}
- for _, rel := range scripts {
+ for _, rel := range entry {
args = append(args, sandbox.ShellQuote(rel))
}
+ if len(auxiliary) > 0 {
+ args = append(args, skillVerifyOptionalFlag)
+ for _, rel := range auxiliary {
+ args = append(args, sandbox.ShellQuote(rel))
+ }
+ }
// The script arrives base64-encoded on stdin: it contains quotes, newlines
// and backslashes, and a heredoc would still be at the mercy of whatever
// the remote exec layer does with multi-line command strings.
@@ -173,6 +268,9 @@ func skillPythonVerifyCommand(skillDir string, scripts []string) string {
// skillNodeVerifyCommand parses every JavaScript file and, when the bundle
// declares runtime dependencies, checks each one resolves. `node --check` is
// parse-only: it never runs the module body.
+//
+// A declared dependency that is missing exits with the repairable code: it is
+// the one thing here another installer round can still put in place.
func skillNodeVerifyCommand(skillDir string, scripts, deps []string) string {
parts := make([]string, 0, 2)
if len(deps) > 0 {
@@ -182,18 +280,75 @@ func skillNodeVerifyCommand(skillDir string, scripts, deps []string) string {
}
parts = append(parts, fmt.Sprintf(
"for d in %s; do [ -e %s/\"$d\" ] || "+
- "{ echo \"package.json declares $d but node_modules/$d is missing\" >&2; exit 1; }; done",
+ "{ echo \"package.json declares $d but node_modules/$d is missing\" >&2; exit %d; }; done",
strings.Join(quoted, " "), sandbox.ShellQuote(path.Join(skillDir, "node_modules")),
+ skillVerifyRepairableExit,
))
}
parts = append(parts, forEachScript(skillDir, scripts, "node --check \"$f\""))
return strings.Join(parts, "; ")
}
-// skillShellVerifyCommand parses every shell script. `sh -n` reads the file
-// and exits without executing any of it.
+// skillShellVerifyCommand parses every shell script with the shell that would
+// run it. `-n` reads the file and exits without executing any of it.
+//
+// bash, not sh: skills ship `#!/bin/bash` almost exclusively, and
+// SkillInterpreterCommand runs them with bash for that reason. /bin/sh is dash
+// on Debian, where an array literal, `function f()`, a C-style for loop and
+// process substitution are all outright syntax errors — so checking with sh
+// would refuse installs over scripts that run perfectly.
func skillShellVerifyCommand(skillDir string, scripts []string) string {
- return forEachScript(skillDir, scripts, "sh -n \"$f\"")
+ return "if command -v bash >/dev/null 2>&1; then parser=bash; else parser=sh; fi; " +
+ forEachScript(skillDir, scripts, `"$parser" -n "$f"`)
+}
+
+// skillVerifyOptionalFlag separates the files whose failure refuses the install
+// from the ones whose failure is only reported. It is the checker's own argv
+// contract; both sides must keep using this spelling.
+const skillVerifyOptionalFlag = "--optional"
+
+// auxiliaryScriptDirs are the directory names Python and JavaScript projects
+// universally use for files that ship with a project without being part of
+// what it offers. A skill's tests import pytest; its examples import whatever
+// they illustrate. Nothing the skill exposes loads either, so neither may
+// decide whether the skill installs.
+var auxiliaryScriptDirs = map[string]struct{}{
+ "test": {}, "tests": {}, "testing": {}, "__tests__": {},
+ "example": {}, "examples": {}, "sample": {}, "samples": {},
+ "benchmark": {}, "benchmarks": {}, "fixtures": {},
+ "doc": {}, "docs": {},
+}
+
+// skillAuxiliaryScript reports whether a bundled path is one of those files.
+//
+// The rule is the naming convention the language communities already share,
+// not a list grown one skill at a time: a directory the ecosystem reserves for
+// tests, examples or docs, or a file name pytest and setuptools already treat
+// as theirs.
+func skillAuxiliaryScript(rel string) bool {
+ segments := strings.Split(path.Clean(rel), "/")
+ for _, segment := range segments[:len(segments)-1] {
+ if _, ok := auxiliaryScriptDirs[strings.ToLower(segment)]; ok {
+ return true
+ }
+ }
+ base := strings.ToLower(segments[len(segments)-1])
+ stem := strings.TrimSuffix(base, path.Ext(base))
+ return stem == "conftest" || stem == "setup" ||
+ strings.HasPrefix(stem, "test_") || strings.HasSuffix(stem, "_test")
+}
+
+// splitAuxiliaryScripts separates the files whose failure is a failed install
+// from the ones whose failure is a note, preserving the caller's order.
+func splitAuxiliaryScripts(scripts []string) (entry, auxiliary []string) {
+ for _, rel := range scripts {
+ if skillAuxiliaryScript(rel) {
+ auxiliary = append(auxiliary, rel)
+ continue
+ }
+ entry = append(entry, rel)
+ }
+ return entry, auxiliary
}
func forEachScript(skillDir string, scripts []string, check string) string {
diff --git a/internal/application/service/tenant_skill_verify.py b/internal/application/service/tenant_skill_verify.py
index 0ae0ae7b0..78d6264dd 100644
--- a/internal/application/service/tenant_skill_verify.py
+++ b/internal/application/service/tenant_skill_verify.py
@@ -3,14 +3,36 @@
Run inside the sandbox, by the interpreter a real skill call would use, as the
ordinary execution user, after the tree has been given its final permissions:
- python3 - <skill-dir> <relative-script> [<relative-script> ...]
+ python3 - <skill-dir> <script> [<script> ...] [--optional <script> ...]
Nothing here executes the skill's own code. Everything it reports is decided
from the parse tree and from what the environment can resolve, so a skill that
writes files or calls the network on import cannot do either while being
checked, and a script that never learned --help cannot be failed for it.
-Exits 1 and writes one line per problem to stderr; exits 0 otherwise.
+Loadability follows Python's import rules, not "this file as isolated
+__main__". A file's import prefixes are its own directory plus every ancestor
+up to the skill root, because any of those can be sys.path[0]: either the
+bundle ships a script there (the runtime hands `python <file>` a directory),
+or the file put it there itself. A sibling subtree is never an ancestor, so
+vendor/requests.py still cannot make `import requests` look satisfied to a
+script under scripts/.
+
+Findings are graded, because rejecting an install is expensive: it throws away
+minutes of dependency work and leaves the previous image serving.
+
+ stdout `note: ...` lines - reported, image kept
+ stderr problem lines - the install is refused
+ exit 0 the image may be kept
+ exit 1 a problem no installer round can fix: a syntax error, a file the
+ execution user cannot read, a relative import that can never
+ resolve
+ exit 2 every problem is a dependency missing from this image, so handing
+ these lines back to the installer is worth a round
+
+Files named after --optional are checked identically, but their findings are
+notes: nothing the skill offers loads a test or an example, and a bundled
+tests/ directory must not decide whether the skill installs.
"""
import ast
@@ -19,54 +41,72 @@ import os
import re
import sys
+OPTIONAL_FLAG = "--optional"
+
+EXIT_UNREPAIRABLE = 1
+EXIT_MISSING_DEPENDENCY = 2
+
root = os.path.abspath(sys.argv[1])
-relative_scripts = sys.argv[2:]
+_argv_scripts = sys.argv[2:]
+if OPTIONAL_FLAG in _argv_scripts:
+ _cut = _argv_scripts.index(OPTIONAL_FLAG)
+ entry_scripts = _argv_scripts[:_cut]
+ optional_scripts = _argv_scripts[_cut + 1 :]
+else:
+ entry_scripts = _argv_scripts
+ optional_scripts = []
+all_scripts = entry_scripts + optional_scripts
+optional_set = set(optional_scripts)
+
+# problems refuse the install; notes are reported and the image is kept.
problems = []
+notes = []
-# Directories that belong to the installed environment rather than the skill's
-# own sources, and would drown the scan below in third-party module names.
-PRUNED = {".venv", "venv", "node_modules", "__pycache__", ".git"}
+# Whether any problem is something installing a package cannot fix. It decides
+# the exit code, which is how the caller knows if another installer round could
+# help or if the bundle itself has to change.
+unrepairable = False
+# The directories the bundle actually ships a script in. A prefix that is one
+# of these is reachable on its own, because the runtime can be asked to execute
+# a file there; any other prefix is only on sys.path if the skill puts it there.
+script_dirs = {os.path.dirname(os.path.join(root, rel)) for rel in all_scripts}
+
+
+def add_problem(message, repairable=False):
+ global unrepairable
+ if message not in problems:
+ problems.append(message)
+ if not repairable:
+ unrepairable = True
-def under_root(candidate):
- return candidate == root or candidate.startswith(root + os.sep)
+def add_note(message):
+ if message not in notes:
+ notes.append(message)
-def own_top_level_names():
- """Top-level names that refer to something the skill itself ships.
- Only the skill root is consulted. A nested file such as vendor/requests.py
- must not make `import requests` look first-party — that name is a
- dependency the installer was asked to put in the venv, and skipping it
- would snapshot a broken install.
+def note_instead_of_problem(message, repairable=False):
+ """Reporter for an auxiliary file: the finding is real, the verdict is not.
- An import of one of these is the skill reaching for its own code, and
- whether it resolves depends on how the script arranges sys.path at run
- time — commonly by inserting its parent before importing `scripts.x`.
- That is a runtime decision this checker cannot evaluate without executing
- the file, so such imports are left to the skill. Sibling modules still
- resolve through find_spec, which puts the script's own directory first.
+ repairable is accepted and ignored so this can stand in for add_problem;
+ a note never influences the exit code.
"""
- names = set()
- try:
- entries = os.listdir(root)
- except OSError:
- return names
- for name in entries:
- if name in PRUNED or name.startswith("."):
- continue
- full = os.path.join(root, name)
- if os.path.isdir(full):
- names.add(name)
- elif name.endswith(".py"):
- names.add(name[:-3])
- return names
+ add_note("%s (auxiliary file; this does not fail the install)" % message)
-own_modules = own_top_level_names()
+def under_root(candidate):
+ return candidate == root or candidate.startswith(root + os.sep)
def is_package(directory):
+ """Whether directory is a regular Python package.
+
+ Only __init__.py counts, and only the relative-import check needs it: a
+ `from . import x` in a directory without one can never resolve when the
+ file is run directly. Absolute imports do not consult this, because a
+ PEP 420 namespace package has no __init__.py and is still importable.
+ """
return os.path.isfile(os.path.join(directory, "__init__.py"))
@@ -78,19 +118,45 @@ def module_exists(base, dotted):
)
-def is_importable(name, script_dir):
- """Whether a top-level name is available to this script at run time.
+def import_search_dirs(script_path):
+ """Every directory that could be sys.path[0] when this file is loaded.
+
+ Its own directory comes first: execute_skill_script runs `python <file>`.
+ Each ancestor up to the skill root follows, because that is what makes a
+ neighbouring entry script's imports resolve - Python puts the directory of
+ the script it was handed on the path, and a package below that directory,
+ regular or namespace, is importable from there. It is also where the
+ `sys.path.insert(0, parent)` idiom lands, which cannot be seen without
+ executing the file.
+
+ Over-approximating here rejects nothing that works. Under-approximating
+ rejected the official office toolkit, whose library modules import their
+ sibling packages by short name.
+ """
+ current = os.path.dirname(os.path.abspath(script_path))
+ dirs = []
+ while under_root(current):
+ if current not in dirs:
+ dirs.append(current)
+ parent = os.path.dirname(current)
+ if parent == current:
+ break
+ current = parent
+ return dirs
+
+
+def find_spec_with(name, prefixes):
+ """Whether find_spec resolves name with prefixes ahead of sys.path.
find_spec consults the finders without importing, so a missing dependency
- is reported without the side effects of loading a present one. script_dir
- leads the path because that is where the interpreter puts the directory of
- the script it was handed, which is how sibling modules resolve at runtime.
+ is reported without the side effects of loading a present one. Only
+ top-level names are ever queried: find_spec on a dotted name imports the
+ parent package, which would run the skill's own module body.
"""
- if name in own_modules:
- return True
saved = sys.path[:]
- sys.path.insert(0, script_dir)
try:
+ for directory in reversed(prefixes):
+ sys.path.insert(0, directory)
return importlib.util.find_spec(name) is not None
except Exception:
return False
@@ -98,39 +164,68 @@ def is_importable(name, script_dir):
sys.path[:] = saved
-def check_imports(rel, tree, script_dir):
- """Check the imports that running this file is guaranteed to execute.
+def resolve_top_level(name, script_path):
+ """How a top-level name resolves for this file.
+
+ "image" the venv, the standard library, or a directory the bundle
+ ships a script in - something the runtime reaches on its own.
+ "unattested" only a directory that ships no script of its own can provide
+ it, so it is on sys.path only if the file puts it there. That
+ is unprovable without executing, so it is reported rather
+ than trusted or refused.
+ "" nothing in this image can provide it.
+ """
+ dirs = import_search_dirs(script_path)
+ attested = [directory for directory in dirs if directory in script_dirs]
+ if find_spec_with(name, attested):
+ return "image"
+ if len(attested) != len(dirs) and find_spec_with(name, dirs):
+ return "unattested"
+ return ""
+
+
+def check_imports(rel, tree, script_path, report):
+ """Check the imports that loading this file is guaranteed to execute.
Only statements at module level are checked, and only unconditionally: an
import nested in try/except, in an `if`, or inside a function is how a
skill declares an optional or lazily loaded dependency, and failing an
install over one would reject a working skill.
"""
+ script_dir = os.path.dirname(script_path)
for node in tree.body:
if isinstance(node, ast.Import):
for alias in node.names:
- top = alias.name.split(".")[0]
- if not is_importable(top, script_dir):
- problems.append(
- "%s imports %s, which is not available in this image"
- % (rel, top)
- )
+ check_absolute_import(rel, alias.name, script_path, report)
elif isinstance(node, ast.ImportFrom):
if node.level:
- check_relative_import(rel, node, script_dir)
+ check_relative_import(rel, node, script_dir, report)
elif node.module:
- top = node.module.split(".")[0]
- if not is_importable(top, script_dir):
- problems.append(
- "%s imports %s, which is not available in this image"
- % (rel, top)
- )
+ check_absolute_import(rel, node.module, script_path, report)
+
+
+def check_absolute_import(rel, dotted, script_path, report):
+ name = dotted.split(".")[0]
+ origin = resolve_top_level(name, script_path)
+ if origin == "image":
+ return
+ if origin == "unattested":
+ add_note(
+ "%s imports %s, which the skill ships but no directory it executes "
+ "scripts from can reach; it resolves only if the file puts that "
+ "directory on sys.path itself" % (rel, name)
+ )
+ return
+ report(
+ "%s imports %s, which is not available in this image" % (rel, name),
+ repairable=True,
+ )
-def check_relative_import(rel, node, script_dir):
+def check_relative_import(rel, node, script_dir, report):
spelling = "." * node.level + (node.module or "")
if not is_package(script_dir):
- problems.append(
+ report(
"%s uses the relative import '%s' but its directory has no "
"__init__.py, so the import can never resolve when the script is "
"run directly" % (rel, spelling)
@@ -140,7 +235,7 @@ def check_relative_import(rel, node, script_dir):
for _ in range(node.level - 1):
base = os.path.dirname(base)
if not under_root(base):
- problems.append(
+ report(
"%s uses the relative import '%s', which reaches outside the "
"skill directory" % (rel, spelling)
)
@@ -149,9 +244,7 @@ def check_relative_import(rel, node, script_dir):
# __init__ defines rather than a submodule, so only the package itself is
# checked in that case.
if node.module and not module_exists(base, node.module):
- problems.append(
- "%s imports '%s', which does not exist in the skill" % (rel, spelling)
- )
+ report("%s imports '%s', which does not exist in the skill" % (rel, spelling))
def distribution_name(raw):
@@ -170,6 +263,35 @@ def distribution_name(raw):
return name
+def marker_of(raw):
+ """The PEP 508 environment marker on a requirement line, or ''."""
+ _, _, marker = raw.split("#")[0].partition(";")
+ return marker.strip()
+
+
+def marker_applies(marker):
+ """Whether pip would install a line carrying this marker in this image.
+
+ None means the answer cannot be established, and callers must read that as
+ "not enforceable" rather than "absent". Markers compare versions with
+ version semantics, so they are evaluated by `packaging` or not at all: a
+ hand-rolled string comparison reads "3.10" as lower than "3.9" and would
+ fail installs whose requirements pip resolved correctly. A bare venv has
+ no `packaging`, so None is the common answer.
+ """
+ if re.search(r"\bextra\b", marker):
+ # Gated on an extra, which pip does not install unless it is requested.
+ return False
+ try:
+ from packaging.markers import Marker
+ except Exception:
+ return None
+ try:
+ return bool(Marker(marker).evaluate())
+ except Exception:
+ return None
+
+
def load_pyproject(path):
try:
import tomllib
@@ -200,13 +322,29 @@ def declared_requirement_lines():
yield "pyproject.toml", item
poetry = ((data.get("tool") or {}).get("poetry") or {}).get("dependencies") or {}
if isinstance(poetry, dict):
- for name in poetry:
+ for name, spec in poetry.items():
if str(name).strip().lower() == "python":
continue
+ # A table-valued dependency carrying optional, markers or a python
+ # constraint is conditional, and poetry would not have installed it
+ # here unconditionally.
+ if isinstance(spec, dict) and (
+ spec.get("optional") or spec.get("markers") or spec.get("python")
+ ):
+ continue
yield "pyproject.toml", str(name)
def check_declared_requirements():
+ """Check the manifests name nothing the venv is missing.
+
+ This is the installer's literal instruction - "install requirements.txt" -
+ so a distribution it names that pip did not land is a failed install. A
+ line pip would have skipped is not: an environment marker that is false
+ here, or an extras-gated dependency, is reported instead. Refusing an
+ install over `pywin32; sys_platform == "win32"` on Linux rejects a skill
+ whose requirements are all present.
+ """
try:
from importlib import metadata
except ImportError:
@@ -217,39 +355,56 @@ def check_declared_requirements():
continue
try:
metadata.distribution(name)
+ continue
except Exception:
- problems.append(
- "%s declares %s but it is not installed in %s"
- % (source, name, sys.prefix)
+ pass
+ missing = "%s declares %s but it is not installed in %s" % (
+ source,
+ name,
+ sys.prefix,
+ )
+ marker = marker_of(raw)
+ if not marker:
+ add_problem(missing, repairable=True)
+ continue
+ applies = marker_applies(marker)
+ if applies:
+ add_problem(missing, repairable=True)
+ elif applies is None:
+ add_note(
+ "%s, and its environment marker '%s' cannot be evaluated here, "
+ "so it is not enforced" % (missing, marker)
)
-for relative in relative_scripts:
+for relative in all_scripts:
+ report = note_instead_of_problem if relative in optional_set else add_problem
script = os.path.join(root, relative)
try:
with open(script, "rb") as source:
code = source.read()
except OSError as exc:
- problems.append(
- "%s cannot be read by the skill execution user (%s)" % (relative, exc)
- )
+ report("%s cannot be read by the skill execution user (%s)" % (relative, exc))
continue
try:
parsed = ast.parse(code, filename=relative)
except SyntaxError as exc:
- problems.append(
+ report(
"%s has a syntax error on line %s: %s" % (relative, exc.lineno, exc.msg)
)
continue
- check_imports(relative, parsed, os.path.dirname(script))
+ check_imports(relative, parsed, script, report)
check_declared_requirements()
+for note in notes:
+ sys.stdout.write("note: %s\n" % note)
+
if problems:
sys.stderr.write("\n".join(problems) + "\n")
- sys.exit(1)
+ sys.exit(EXIT_UNREPAIRABLE if unrepairable else EXIT_MISSING_DEPENDENCY)
print(
"verified %d python file(s) against %s"
- % (len(relative_scripts), sys.executable or "python3")
+ % (len(all_scripts), sys.executable or "python3")
)
diff --git a/internal/application/service/tenant_skill_verify_python_test.go b/internal/application/service/tenant_skill_verify_python_test.go
index 898cdff0a..51193d935 100644
--- a/internal/application/service/tenant_skill_verify_python_test.go
+++ b/internal/application/service/tenant_skill_verify_python_test.go
@@ -21,9 +21,21 @@ func TestSkillPythonVerifier(t *testing.T) {
cases := []struct {
name string
files map[string]string
+ // optional names the files whose findings must be reported instead of
+ // refusing the install. The install path fills this from the naming
+ // conventions the ecosystems share; here it is explicit.
+ optional []string
// wantProblem is a substring of the expected stderr. Empty means the
// tree must verify cleanly.
wantProblem string
+ // wantExit is the code a failing tree must exit with: 2 when installing
+ // a package would satisfy everything found, 1 when it would not. The
+ // install flow reads it to decide whether another installer round is
+ // worth its minutes.
+ wantExit int
+ // wantNote is a substring of a stdout note - something the checker
+ // reported without refusing the install.
+ wantNote string
}{{
name: "a package whose __init__ imports its own submodules",
files: map[string]string{
@@ -34,7 +46,8 @@ func TestSkillPythonVerifier(t *testing.T) {
// The shape that failed a real install: sys.path is rearranged at run
// time, so the absolute import of the skill's own package resolves
// only once the file executes. Deciding it statically is not possible,
- // and guessing wrong rejects a working skill.
+ // and guessing wrong rejects a working skill. It is reported, because
+ // the only directory that can satisfy it ships no script of its own.
name: "a script that puts the skill root on sys.path and imports its own package",
files: map[string]string{
"scripts/__init__.py": "",
@@ -44,6 +57,8 @@ func TestSkillPythonVerifier(t *testing.T) {
"sys.path.insert(0, str(Path(__file__).resolve().parent.parent))\n" +
"from scripts.helper import go\n",
},
+ wantNote: "imports scripts, which the skill ships but no directory it " +
+ "executes scripts from can reach",
}, {
name: "a script importing a sibling module directly",
files: map[string]string{
@@ -51,6 +66,17 @@ func TestSkillPythonVerifier(t *testing.T) {
"scripts/helper.py": "x = 1\n",
"scripts/README.txt": "not python\n",
},
+ }, {
+ // scripts/ is both a package (relative imports in __init__) and a
+ // directory of files the runtime executes with python file.py.
+ // Loadability is the union of those prefixes, so a sibling import
+ // still resolves when the package marker is present.
+ name: "a packaged scripts directory whose files also import siblings",
+ files: map[string]string{
+ "scripts/__init__.py": "",
+ "scripts/run.py": "import helper\n",
+ "scripts/helper.py": "x = 1\n",
+ },
}, {
name: "an optional dependency behind try/except",
files: map[string]string{
@@ -69,12 +95,25 @@ func TestSkillPythonVerifier(t *testing.T) {
},
wantProblem: "scripts/run.py imports totally_absent_package, " +
"which is not available in this image",
+ wantExit: 2,
}, {
name: "a syntax error",
files: map[string]string{
"scripts/run.py": "def broken(:\n pass\n",
},
wantProblem: "scripts/run.py has a syntax error on line 1",
+ wantExit: 1,
+ }, {
+ // The exit code is the whole verdict, so one unfixable finding among
+ // fixable ones has to sink the batch: sending the installer back for a
+ // package it can install would only delay a failure it cannot.
+ name: "a syntax error alongside a missing dependency",
+ files: map[string]string{
+ "scripts/bad.py": "def broken(:\n",
+ "scripts/run.py": "import totally_absent_package\n",
+ },
+ wantProblem: "scripts/bad.py has a syntax error",
+ wantExit: 1,
}, {
name: "a relative import in a directory that is not a package",
files: map[string]string{
@@ -83,6 +122,7 @@ func TestSkillPythonVerifier(t *testing.T) {
"scripts/notinit.py": "",
},
wantProblem: "has no __init__.py",
+ wantExit: 1,
}, {
name: "a relative import of a module the skill does not ship",
files: map[string]string{
@@ -91,14 +131,45 @@ func TestSkillPythonVerifier(t *testing.T) {
"pkg/sub/run.py": "from ..missing import go\n",
},
wantProblem: "pkg/sub/run.py imports '..missing', which does not exist in the skill",
+ wantExit: 1,
}, {
name: "a requirement the venv does not carry",
files: map[string]string{
- "requirements.txt": "# pinned\npandas==3.0.1\n-r other.txt\n" +
- "totally_absent_package>=1.0 ; python_version>=\"3.9\"\n",
+ "requirements.txt": "# pinned\npandas==3.0.1\n-r other.txt\n",
+ "scripts/run.py": "x = 1\n",
+ },
+ wantProblem: "requirements.txt declares pandas but it is not installed",
+ wantExit: 2,
+ }, {
+ // pip skips a line whose marker is false here, so refusing the install
+ // over it rejects a skill whose requirements are all present. Markers
+ // compare versions with version semantics, so they are evaluated by
+ // `packaging` or not at all - and a bare venv has no `packaging`, which
+ // is why "cannot evaluate" has to mean "report", not "enforce".
+ name: "a requirement gated by an environment marker",
+ files: map[string]string{
+ "requirements.txt": "pywin32; sys_platform == \"win32\"\n" +
+ "totally_absent_package; extra == \"dev\"\n",
+ "scripts/run.py": "x = 1\n",
+ },
+ wantNote: "requirements.txt declares pywin32 but it is not installed",
+ }, {
+ // An extras-gated dependency is never installed unless the extra is
+ // requested, so it is not even worth a note.
+ name: "an extras-gated requirement is not reported at all",
+ files: map[string]string{
+ "requirements.txt": "totally_absent_package; extra == \"dev\"\n",
+ "scripts/run.py": "x = 1\n",
+ },
+ }, {
+ // poetry tables carrying optional, markers or a python constraint are
+ // conditional, and poetry would not have installed them here either.
+ name: "a poetry dependency marked optional",
+ files: map[string]string{
+ "pyproject.toml": "[tool.poetry.dependencies]\npython = \"^3.11\"\n" +
+ "totally_absent_package = { version = \"^1.0\", optional = true }\n",
"scripts/run.py": "x = 1\n",
},
- wantProblem: "requirements.txt declares totally_absent_package but it is not installed",
}, {
name: "a nested file name is not treated as a first-party import",
files: map[string]string{
@@ -107,6 +178,49 @@ func TestSkillPythonVerifier(t *testing.T) {
},
wantProblem: "scripts/run.py imports totally_absent_package, " +
"which is not available in this image",
+ wantExit: 2,
+ }, {
+ // The official Anthropic office toolkit (xlsx/docx/pptx): a non-package
+ // directory holds entry scripts plus sibling packages. Library modules
+ // import those siblings by short name (`from helpers import ...`).
+ // Treating every .py as isolated __main__ rejects this layout; Python's
+ // package path root is scripts/office/, which is where helpers lives.
+ name: "a library package importing a sibling package by short name",
+ files: map[string]string{
+ "scripts/recalc.py": "from office.soffice import run_soffice\n",
+ "scripts/office/soffice.py": "def run_soffice():\n pass\n",
+ "scripts/office/validate.py": "from helpers import safe_extract\n" +
+ "from validators import BaseSchemaValidator\n",
+ "scripts/office/helpers/__init__.py": "def safe_extract():\n pass\n",
+ "scripts/office/validators/__init__.py": "from .base import BaseSchemaValidator\n",
+ "scripts/office/validators/base.py": "from helpers import safe_extract\n" +
+ "class BaseSchemaValidator:\n pass\n",
+ "scripts/office/validators/docx.py": "from helpers import safe_extract\n",
+ },
+ }, {
+ name: "a library package importing a dependency the installer never installed",
+ files: map[string]string{
+ "scripts/office/validate.py": "x = 1\n",
+ "scripts/office/helpers/__init__.py": "x = 1\n",
+ "scripts/office/validators/__init__.py": "",
+ "scripts/office/validators/base.py": "import totally_absent_package\n",
+ },
+ wantProblem: "scripts/office/validators/base.py imports totally_absent_package, " +
+ "which is not available in this image",
+ wantExit: 2,
+ }, {
+ // The same layout without __init__.py anywhere. PEP 420 makes those
+ // directories namespace packages, so a neighbouring entry script still
+ // resolves the short-name import - which is why prefixes are every
+ // ancestor and not only the ones a package marker chains together.
+ name: "a namespace package importing a sibling by short name",
+ files: map[string]string{
+ "scripts/office/validate.py": "from helpers import safe_extract\n",
+ "scripts/office/helpers/__init__.py": "def safe_extract():\n pass\n",
+ "scripts/office/validators/base.py": "from helpers import safe_extract\n",
+ "scripts/pkg/sub/mod.py": "from util import go\n",
+ "scripts/pkg/util.py": "def go():\n pass\n",
+ },
}, {
name: "a pyproject.toml dependency the venv does not carry",
files: map[string]string{
@@ -115,6 +229,23 @@ func TestSkillPythonVerifier(t *testing.T) {
"scripts/run.py": "x = 1\n",
},
wantProblem: "pyproject.toml declares totally_absent_package but it is not installed",
+ wantExit: 2,
+ }, {
+ // Skills ship their tests. Nothing the skill offers loads them, so a
+ // bundled tests/ directory must not decide whether the skill installs -
+ // and refusing over one throws away the dependency work that succeeded.
+ name: "a bundled test file importing a package the image does not carry",
+ files: map[string]string{
+ "scripts/run.py": "x = 1\n",
+ "tests/test_run.py": "import pytest\nimport totally_absent_package\n",
+ "examples/demo.py": "import totally_absent_package\n",
+ "tests/conftest.py": "def broken(:\n",
+ "scripts/helper.py": "x = 1\n",
+ },
+ optional: []string{
+ "examples/demo.py", "tests/conftest.py", "tests/test_run.py",
+ },
+ wantNote: "auxiliary file; this does not fail the install",
}, {
// Lines that name a distribution only indirectly cannot be checked by
// name, and inventing one from the URL would fail installs whose
@@ -132,19 +263,52 @@ func TestSkillPythonVerifier(t *testing.T) {
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
root := writeSkillTree(t, tc.files)
- stdout, stderr, err := runSkillPythonVerifier(t, root, pythonFiles(tc.files))
+ entry := pythonFiles(tc.files)
+ if len(tc.optional) > 0 {
+ entry = withoutPaths(entry, tc.optional)
+ }
+ stdout, stderr, err := runSkillPythonVerifier(t, root, entry, tc.optional)
if tc.wantProblem == "" {
require.NoError(t, err, "this skill must install; stderr: %s", stderr)
require.Contains(t, stdout, "verified")
- return
+ } else {
+ require.Error(t, err, "this skill is broken and must not reach a snapshot")
+ require.Contains(t, stderr, tc.wantProblem)
+ require.Equal(t, tc.wantExit, verifierExitCode(t, err),
+ "the exit code is what decides whether an installer round can fix this; "+
+ "stderr: %s", stderr)
+ }
+ if tc.wantNote != "" {
+ require.Contains(t, stdout, "note: ",
+ "a finding that does not refuse the install must still be reported")
+ require.Contains(t, stdout, tc.wantNote)
}
- require.Error(t, err, "this skill is broken and must not reach a snapshot")
- require.Contains(t, stderr, tc.wantProblem)
})
}
}
+func verifierExitCode(t *testing.T, err error) int {
+ t.Helper()
+ var exit *exec.ExitError
+ require.ErrorAs(t, err, &exit, "the checker must fail by exiting, not by crashing")
+ return exit.ExitCode()
+}
+
+func withoutPaths(all, drop []string) []string {
+ dropped := make(map[string]struct{}, len(drop))
+ for _, rel := range drop {
+ dropped[rel] = struct{}{}
+ }
+ kept := make([]string, 0, len(all))
+ for _, rel := range all {
+ if _, ok := dropped[rel]; !ok {
+ kept = append(kept, rel)
+ }
+ }
+ return kept
+}
+
// Verification must be able to read a skill that writes files or opens sockets
// the moment it is imported, without doing either.
func TestSkillPythonVerifierNeverExecutesTheSkill(t *testing.T) {
@@ -153,7 +317,7 @@ func TestSkillPythonVerifierNeverExecutesTheSkill(t *testing.T) {
"open(os.path.join(os.path.dirname(__file__), 'SIDE_EFFECT'), 'w').close()\n",
})
- _, stderr, err := runSkillPythonVerifier(t, root, []string{"scripts/run.py"})
+ _, stderr, err := runSkillPythonVerifier(t, root, []string{"scripts/run.py"}, nil)
require.NoError(t, err, stderr)
_, statErr := os.Stat(filepath.Join(root, "scripts", "SIDE_EFFECT"))
@@ -170,10 +334,50 @@ func TestSkillPythonVerifierReportsAnUnreadableScript(t *testing.T) {
root := writeSkillTree(t, map[string]string{"scripts/run.py": "x = 1\n"})
require.NoError(t, os.Chmod(filepath.Join(root, "scripts", "run.py"), 0o000))
- _, stderr, err := runSkillPythonVerifier(t, root, []string{"scripts/run.py"})
+ _, stderr, err := runSkillPythonVerifier(t, root, []string{"scripts/run.py"}, nil)
require.Error(t, err)
require.Contains(t, stderr, "cannot be read by the skill execution user")
+ require.Equal(t, 1, verifierExitCode(t, err),
+ "a file the execution user cannot read is not something installing a package fixes")
+}
+
+// The layout that started this: the official office toolkit ships entry scripts
+// beside sibling packages, and its library modules import those siblings by
+// short name. Treating every .py as an isolated __main__ rejected it outright.
+// The imports it cannot resolve here are the third-party ones it genuinely
+// needs — including two its SKILL.md never mentions — and that is the list the
+// installer gets handed back.
+func TestSkillPythonVerifierAcceptsTheOfficeToolkitLayout(t *testing.T) {
+ files := map[string]string{
+ "SKILL.md": "# xlsx\n",
+ "scripts/recalc.py": "import json\nimport sys\nfrom pathlib import Path\n" +
+ "from office.soffice import run_soffice\n",
+ "scripts/office/soffice.py": "import subprocess\nimport tempfile\n" +
+ "def run_soffice():\n pass\n",
+ "scripts/office/validate.py": "import argparse\n" +
+ "from helpers import safe_extract\n" +
+ "from validators import DOCXSchemaValidator\n",
+ "scripts/office/helpers/__init__.py": "import zipfile\n" +
+ "def safe_extract():\n pass\n",
+ "scripts/office/validators/__init__.py": "from .docx import DOCXSchemaValidator\n",
+ "scripts/office/validators/base.py": "import re\n" +
+ "from helpers import safe_extract\n" +
+ "class BaseSchemaValidator:\n pass\n",
+ "scripts/office/validators/docx.py": "from helpers import safe_extract\n" +
+ "from .base import BaseSchemaValidator\n" +
+ "class DOCXSchemaValidator(BaseSchemaValidator):\n pass\n",
+ "scripts/office/helpers/pptx_chart.py": "from __future__ import annotations\n" +
+ "import re\nfrom . import part_text\n",
+ }
+ root := writeSkillTree(t, files)
+
+ stdout, stderr, err := runSkillPythonVerifier(t, root, pythonFiles(files), nil)
+
+ require.NoError(t, err, "the toolkit's own layout must not be a failed install; stderr: %s", stderr)
+ require.Contains(t, stdout, "verified")
+ require.NotContains(t, stderr, "helpers",
+ "helpers is a sibling package of validators/, reachable from scripts/office/")
}
func writeSkillTree(t *testing.T, files map[string]string) string {
@@ -200,14 +404,20 @@ func pythonFiles(files map[string]string) []string {
// runSkillPythonVerifier feeds the embedded checker to a real interpreter the
// same way the sandbox command does: on stdin, with the tree and the files to
-// check as argv.
-func runSkillPythonVerifier(t *testing.T, root string, scripts []string) (string, string, error) {
+// check as argv, auxiliary files last behind the separator.
+func runSkillPythonVerifier(
+ t *testing.T, root string, scripts, optional []string,
+) (string, string, error) {
t.Helper()
python, err := exec.LookPath("python3")
if err != nil {
t.Skip("python3 is not on PATH")
}
- cmd := exec.Command(python, append([]string{"-", root}, scripts...)...)
+ argv := append([]string{"-", root}, scripts...)
+ if len(optional) > 0 {
+ argv = append(append(argv, skillVerifyOptionalFlag), optional...)
+ }
+ cmd := exec.Command(python, argv...)
cmd.Stdin = strings.NewReader(skillPythonVerifier)
var stdout, stderr bytes.Buffer
cmd.Stdout = &stdout
diff --git a/internal/sandbox/skill_paths.go b/internal/sandbox/skill_paths.go
index fa7c8f00d..0102e66e4 100644
--- a/internal/sandbox/skill_paths.go
+++ b/internal/sandbox/skill_paths.go
@@ -189,7 +189,17 @@ func SkillInterpreterCommand(skillDir, scriptPath string) (string, []string) {
case ".js", ".mjs", ".cjs":
return "node", []string{scriptPath}
case ".sh":
- return "/bin/sh", []string{scriptPath}
+ // bash, with sh only as a fallback. Skill shell scripts carry a
+ // `#!/bin/bash` shebang almost exclusively, and /bin/sh is dash on
+ // Debian: an array literal, `function f()`, a C-style for loop and
+ // process substitution are all syntax errors there, so running these
+ // files with sh breaks scripts that are perfectly valid. The
+ // install-time check parses them with the same shell.
+ script := ShellQuote(scriptPath)
+ return "/bin/sh", []string{"-c", fmt.Sprintf(
+ `if command -v bash >/dev/null 2>&1; then exec bash %s "$@"; else exec sh %s "$@"; fi`,
+ script, script,
+ ), skillShellArgv0}
default:
return "/bin/sh", []string{scriptPath}
}
diff --git a/internal/sandbox/skill_paths_test.go b/internal/sandbox/skill_paths_test.go
index 97ae6a4b5..efb37e508 100644
--- a/internal/sandbox/skill_paths_test.go
+++ b/internal/sandbox/skill_paths_test.go
@@ -161,10 +161,34 @@ func TestSkillInterpreterCommand(t *testing.T) {
}
})
- t.Run("shell scripts run with sh", func(t *testing.T) {
+ // Skills ship `#!/bin/bash` almost exclusively, and on Debian /bin/sh is
+ // dash: an array literal, `function f()`, a C-style for loop and process
+ // substitution are syntax errors there. Running these files with sh broke
+ // scripts that are perfectly valid, and made the install-time `sh -n` check
+ // refuse them on the way in.
+ t.Run("shell scripts prefer bash", func(t *testing.T) {
cmd, args := SkillInterpreterCommand(dir, dir+"/scripts/run.sh")
require.Equal(t, "/bin/sh", cmd)
- require.Equal(t, []string{dir + "/scripts/run.sh"}, args)
+ require.Len(t, args, 3)
+ require.Equal(t, "-c", args[0])
+ require.Contains(t, args[1], "exec bash "+dir+"/scripts/run.sh")
+ require.Contains(t, args[1], "else", "there must be a fallback when bash is absent")
+ require.Equal(t, "weknora-skill", args[2])
+ })
+
+ t.Run("a shell script receives the caller's arguments", func(t *testing.T) {
+ if _, err := os.Stat("/bin/sh"); err != nil {
+ t.Skipf("shell is not available: %v", err)
+ }
+ scriptDir := t.TempDir()
+ script := filepath.Join(scriptDir, "echo-args.sh")
+ require.NoError(t, os.WriteFile(script, []byte("printf '%s\\n' \"$@\"\n"), 0o755))
+
+ cmd, baseArgs := SkillInterpreterCommand(scriptDir, script)
+ out, err := exec.Command(cmd, append(append([]string{}, baseArgs...),
+ "--first", "value")...).CombinedOutput()
+ require.NoError(t, err, string(out))
+ require.Equal(t, "--first\nvalue\n", string(out))
})
t.Run("unknown extension falls back to sh", func(t *testing.T) {