Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
Make S3 publishing replace stale index files instead of accumulating (#3129)
r33drichardse7caf6f03811
diff --git a/docs/scripts/modal_app.py b/docs/scripts/modal_app.pyindex 0c7d1b2c6d..68f49f0769 100644--- a/docs/scripts/modal_app.py+++ b/docs/scripts/modal_app.py@@ -469,6 +469,48 @@ async def crawl_docs():return summary+def _publish_dir(s3, bucket: str, local_dir, prefix: str) -> int:+ """Publish local_dir to s3://bucket/prefix, removing anything stale.++ Uploads every file, then deletes keys under the prefix that the current+ build did not write. Plain upload-only publishing is what broke docs+ vector search: the indexer rebuilds docs.lance from scratch, but old+ files were never removed from S3, so manifests from a previous Lance+ format lingered next to the new ones. Lance then refused to open the+ table ("Found multiple manifest naming schemes in the same directory:+ V1 and V2") and the server reported it as a missing database.++ Deletes run after the uploads, not before, so the prefix is never empty+ and the S3->PVC sync job cannot mirror a half-published table.+ """+ uploaded_keys = set()+ count = 0+ for fpath in sorted(local_dir.rglob("*")):+ if fpath.is_file():+ key = f"{prefix}{fpath.relative_to(local_dir).as_posix()}"+ s3.upload_file(str(fpath), bucket, key)+ uploaded_keys.add(key)+ count += 1++ stale = []+ for page in s3.get_paginator("list_objects_v2").paginate(+ Bucket=bucket, Prefix=prefix+ ):+ for obj in page.get("Contents", []):+ if obj["Key"] not in uploaded_keys:+ stale.append({"Key": obj["Key"]})++ for batch_start in range(0, len(stale), 1000):+ s3.delete_objects(+ Bucket=bucket,+ Delete={"Objects": stale[batch_start : batch_start + 1000]},+ )+ if stale:+ print(f" Removed {len(stale)} stale object(s) under {prefix}")++ return count++@app.function(image=image,volumes={VOLUME_PATH: docs_volume, CODE_VOLUME_PATH: code_volume},@@ -527,11 +569,7 @@ def sync_to_s3(bucket: str = S3_BUCKET_NAME):# LanceDB directorylance_dir = docs_db_dir / "docs.lance"if lance_dir.exists():- for fpath in lance_dir.rglob("*"):- if fpath.is_file():- key = f"docs_db/docs.lance/{fpath.relative_to(lance_dir)}"- s3.upload_file(str(fpath), bucket, key)- uploaded += 1+ uploaded += _publish_dir(s3, bucket, lance_dir, "docs_db/docs.lance/")print(" Uploaded docs LanceDB directory")# --- code databases ---@@ -548,11 +586,9 @@ def sync_to_s3(bucket: str = S3_BUCKET_NAME):# Aggregated LanceDB directorycode_lance = code_db_dir / "code_index.lancedb"if code_lance.exists():- for fpath in code_lance.rglob("*"):- if fpath.is_file():- key = f"code_db/code_index.lancedb/{fpath.relative_to(code_lance)}"- s3.upload_file(str(fpath), bucket, key)- uploaded += 1+ uploaded += _publish_dir(+ s3, bucket, code_lance, "code_db/code_index.lancedb/"+ )print(" Uploaded code LanceDB directory")print(f"S3 sync complete: {uploaded} files uploaded to s3://{bucket}/")