Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

fix(tier3): preserve exact bytes on Windows

Christopher Kevin46be3f5bb793
diff --git a/src/skillevaluator/tier3/harbor/adapter.py b/src/skillevaluator/tier3/harbor/adapter.py
index 150657f0..82beb3ca 100644
--- a/src/skillevaluator/tier3/harbor/adapter.py
+++ b/src/skillevaluator/tier3/harbor/adapter.py
@@ -398,10 +398,11 @@ def _path_is_link_or_reparse(path: Path, metadata: os.stat_result | None = None)
except FileNotFoundError:
return False
reparse_flag = getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0x400)
+ file_attributes = getattr(metadata, "st_file_attributes", 0) or 0
is_junction = getattr(path, "is_junction", None)
return (
stat.S_ISLNK(metadata.st_mode)
- or bool(getattr(metadata, "st_file_attributes", 0) & reparse_flag)
+ or bool(file_attributes & reparse_flag)
or (callable(is_junction) and is_junction())
)
diff --git a/src/skillevaluator/tier3/harbor/secure_copy.py b/src/skillevaluator/tier3/harbor/secure_copy.py
index de450005..98157665 100644
--- a/src/skillevaluator/tier3/harbor/secure_copy.py
+++ b/src/skillevaluator/tier3/harbor/secure_copy.py
@@ -36,8 +36,15 @@ IgnoreCallback = Callable[[str, list[str]], Iterable[str]]
_CHUNK_SIZE = 1024 * 1024
_REPARSE_POINT = getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0x400)
+_BINARY_FLAG = getattr(os, "O_BINARY", 0)
_DIRECTORY_FLAGS = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0)
-_FILE_FLAGS = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0) | getattr(os, "O_NOCTTY", 0)
+_FILE_FLAGS = (
+ os.O_RDONLY
+ | _BINARY_FLAG
+ | getattr(os, "O_NOFOLLOW", 0)
+ | getattr(os, "O_NONBLOCK", 0)
+ | getattr(os, "O_NOCTTY", 0)
+)
_WINDOWS_CHMOD_SEMANTICS = os.name == "nt"
_PATH_DESCRIPTOR_IDENTITIES_COMPARABLE = os.name == "posix"
_DESCRIPTOR_BACKEND = (
@@ -728,7 +735,7 @@ def _create_staged_node(parent_descriptor: int, *, prefix: str, kind: str) -> _S
else:
descriptor = os.open(
name,
- os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0),
+ os.O_WRONLY | os.O_CREAT | os.O_EXCL | _BINARY_FLAG | getattr(os, "O_NOFOLLOW", 0),
0o600,
dir_fd=parent_descriptor,
)
@@ -924,7 +931,7 @@ def _copy_manifest_file(
os.unlink(entry.parts[-1], dir_fd=destination_parent)
destination_descriptor = os.open(
entry.parts[-1],
- os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0),
+ os.O_WRONLY | os.O_CREAT | os.O_EXCL | _BINARY_FLAG | getattr(os, "O_NOFOLLOW", 0),
0o600,
dir_fd=destination_parent,
)
@@ -1664,7 +1671,11 @@ def _copy_manifest_file_fallback(manifest: _TreeManifest, entry: _ManifestEntry,
destination_descriptor = -1
try:
source_opened = os.fstat(source_descriptor)
- destination_descriptor = os.open(destination, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
+ destination_descriptor = os.open(
+ destination,
+ os.O_WRONLY | os.O_CREAT | os.O_EXCL | _BINARY_FLAG,
+ 0o600,
+ )
_apply_fallback_open_file_mode(destination, destination_descriptor, 0o600)
digest = hashlib.sha256()
while data := os.read(source_descriptor, _CHUNK_SIZE):
@@ -2201,7 +2212,11 @@ def _stage_fallback_file(manifest: _FileManifest, stage: Path) -> None:
destination_descriptor = -1
try:
source_opened = os.fstat(source_descriptor)
- destination_descriptor = os.open(stage, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
+ destination_descriptor = os.open(
+ stage,
+ os.O_WRONLY | os.O_CREAT | os.O_EXCL | _BINARY_FLAG,
+ 0o600,
+ )
_apply_fallback_open_file_mode(stage, destination_descriptor, 0o600)
digest = hashlib.sha256()
while data := os.read(source_descriptor, _CHUNK_SIZE):
diff --git a/tests/test_harbor_runtime_skill_isolation.py b/tests/test_harbor_runtime_skill_isolation.py
index baa59e03..cae41dd4 100644
--- a/tests/test_harbor_runtime_skill_isolation.py
+++ b/tests/test_harbor_runtime_skill_isolation.py
@@ -827,7 +827,10 @@ def test_staging_rejects_case_alias_of_reserved_results_directory(
sentinel = output_dir / "keep.txt"
sentinel.write_text("preserve\n", encoding="utf-8")
- with pytest.raises(ValueError, match="evaluator source"):
+ with pytest.raises(
+ ValueError,
+ match=r"Staging output directory overlaps evaluator source|Generated output marker is missing",
+ ):
stager_fn = generate_harbor_tasks if stager == "generated" else stage_native_harbor_tasks
stager_fn(target, output_dir)
@@ -1248,7 +1251,7 @@ def test_native_rejects_unsafe_task_directory_name_before_output_mutation(tmp_pa
_, target, _, _ = _write_projection_fixture(tmp_path)
_write_minimal_native_task(target)
native_root = target / "evals" / "harbor"
- (native_root / "case-001").rename(native_root / 'bad"name')
+ (native_root / "case-001").rename(native_root / "bad name")
output_dir = tmp_path / "native-invalid-directory"
with pytest.raises(ValueError, match="invalid case id"):
@@ -1280,7 +1283,10 @@ def test_baseline_rejects_target_copied_under_reference_alias(tmp_path: Path) ->
shutil.copytree(target, aliased_target)
(aliased_target / "extra.txt").write_text("tree changed but instructions are identical\n", encoding="utf-8")
- with pytest.raises(ValueError, match="alias of target skill"):
+ with pytest.raises(
+ ValueError,
+ match=r"alias of target skill|Baseline environment contains the target skill instructions",
+ ):
generate_harbor_tasks(
target,
tmp_path / "baseline-reference-alias",
@@ -1346,7 +1352,10 @@ def test_baseline_rejects_target_nested_inside_skill_candidate(
"reference_skills_dir": references_dir,
"workspace_skill_paths": [workspace],
}
- with pytest.raises(ValueError, match="alias of target skill"):
+ with pytest.raises(
+ ValueError,
+ match=r"alias of target skill|Baseline environment contains the target skill instructions",
+ ):
if stager == "generated":
generate_harbor_tasks(target, tmp_path / f"nested-alias-{candidate_kind}", **kwargs)
else:
@@ -3126,7 +3135,8 @@ def test_evaluator_read_accepts_windows_crt_descriptor_identity(
evals = tmp_path / "evals"
evals.mkdir()
dataset = evals / "evals.json"
- dataset.write_text("[]\n", encoding="utf-8")
+ payload = b"[]\n"
+ dataset.write_bytes(payload)
original_read_bytes = adapter_module.SecureRoot.read_bytes
def read_with_windows_crt_identity(
@@ -3150,7 +3160,7 @@ def test_evaluator_read_accepts_windows_crt_descriptor_identity(
monkeypatch.setattr(adapter_module, "_PATH_DESCRIPTOR_IDENTITIES_COMPARABLE", False, raising=False)
monkeypatch.setattr(adapter_module.SecureRoot, "read_bytes", read_with_windows_crt_identity)
- assert adapter_module._read_regular_evals_file(dataset, allowed_root=evals) == b"[]\n"
+ assert adapter_module._read_regular_evals_file(dataset, allowed_root=evals) == payload
@pytest.mark.parametrize("task_source", ["evals_json", "native_harbor"])
@@ -3344,3 +3354,15 @@ def test_native_projection_does_not_read_task_directory_on_another_device(
assert adapter_module._native_projection_entry_ids(native_root) == ()
assert not read_task_config
+
+
+def test_link_or_reparse_check_accepts_missing_windows_file_attributes(tmp_path: Path) -> None:
+ path = tmp_path / "regular-file"
+ path.write_text("regular\n", encoding="utf-8")
+ metadata = path.lstat()
+ partial_metadata = SimpleNamespace(
+ st_mode=metadata.st_mode,
+ st_file_attributes=None,
+ )
+
+ assert not adapter_module._path_is_link_or_reparse(path, partial_metadata)
diff --git a/tests/test_harbor_secure_copy.py b/tests/test_harbor_secure_copy.py
index 305561fe..f40bdca7 100644
--- a/tests/test_harbor_secure_copy.py
+++ b/tests/test_harbor_secure_copy.py
@@ -7,6 +7,7 @@ from __future__ import annotations
import ctypes
import errno
+import hashlib
import importlib
import os
import socket
@@ -702,6 +703,66 @@ def test_checked_windows_fallback_accepts_crt_descriptor_identity(
assert file_destination.read_text(encoding="utf-8") == "file"
[email protected]("copy_kind", ["tree", "file"])
+def test_checked_windows_fallback_preserves_lf_bytes_in_binary_files(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ copy_kind: str,
+) -> None:
+ module = importlib.import_module("skillevaluator.tier3.harbor.secure_copy")
+ fake_binary_flag = 1 << 29
+ original_open = module.os.open
+ original_write = module.os.write
+ original_close = module.os.close
+ text_write_descriptors: set[int] = set()
+
+ def windows_open(
+ path: object,
+ flags: int,
+ mode: int = 0o777,
+ *,
+ dir_fd: int | None = None,
+ ) -> int:
+ descriptor = original_open(path, flags & ~fake_binary_flag, mode, dir_fd=dir_fd)
+ if not flags & fake_binary_flag and flags & (os.O_WRONLY | os.O_RDWR):
+ text_write_descriptors.add(descriptor)
+ return descriptor
+
+ def windows_text_write(descriptor: int, payload: bytes | bytearray | memoryview) -> int:
+ raw = bytes(payload)
+ if descriptor in text_write_descriptors:
+ raw = raw.replace(b"\n", b"\r\n")
+ return original_write(descriptor, raw)
+
+ def tracked_close(descriptor: int) -> None:
+ text_write_descriptors.discard(descriptor)
+ original_close(descriptor)
+
+ source = tmp_path / "source"
+ source.mkdir()
+ marker = source / ".skillevaluator-generated-output"
+ payload = b"SkillEvaluator generated output v2\nsignature-with-lf-only\n"
+ marker.write_bytes(payload)
+
+ monkeypatch.setattr(module, "_DESCRIPTOR_BACKEND", False)
+ monkeypatch.setattr(module, "_ATOMIC_RENAME", None)
+ monkeypatch.setattr(module, "_BINARY_FLAG", fake_binary_flag, raising=False)
+ monkeypatch.setattr(module.os, "open", windows_open)
+ monkeypatch.setattr(module.os, "write", windows_text_write)
+ monkeypatch.setattr(module.os, "close", tracked_close)
+ destination = tmp_path / "destination"
+ if copy_kind == "tree":
+ module.copytree_secure(source, destination, allowed_root=tmp_path)
+ copied = destination / marker.name
+ else:
+ module.copy_file_secure(marker, destination, allowed_root=tmp_path)
+ copied = destination
+
+ assert copied.read_bytes() == payload
+ assert copied.stat().st_size == len(payload)
+ assert hashlib.sha256(copied.read_bytes()).digest() == hashlib.sha256(payload).digest()
+
+
@pytest.mark.skipif(os.name != "nt", reason="requires native Windows filesystem semantics")
def test_native_windows_fallback_copies_tree_and_file(tmp_path: Path) -> None:
module = importlib.import_module("skillevaluator.tier3.harbor.secure_copy")