Skip to content

The open-source app everyone uses to manage agents at work

Fix company zip imports

dottadcead97650a8
diff --git a/cli/src/__tests__/company-import-export-e2e.test.ts b/cli/src/__tests__/company-import-export-e2e.test.ts
index 9c141a13c0..56171c71ca 100644
--- a/cli/src/__tests__/company-import-export-e2e.test.ts
+++ b/cli/src/__tests__/company-import-export-e2e.test.ts
@@ -1,5 +1,5 @@
import { execFile, spawn } from "node:child_process";
-import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs";
import net from "node:net";
import os from "node:os";
import path from "node:path";
@@ -182,6 +182,107 @@ function createCliEnv() {
return env;
}
+function writeUint16(target: Uint8Array, offset: number, value: number) {
+ target[offset] = value & 0xff;
+ target[offset + 1] = (value >>> 8) & 0xff;
+}
+
+function writeUint32(target: Uint8Array, offset: number, value: number) {
+ target[offset] = value & 0xff;
+ target[offset + 1] = (value >>> 8) & 0xff;
+ target[offset + 2] = (value >>> 16) & 0xff;
+ target[offset + 3] = (value >>> 24) & 0xff;
+}
+
+function crc32(bytes: Uint8Array) {
+ let crc = 0xffffffff;
+ for (const byte of bytes) {
+ crc ^= byte;
+ for (let bit = 0; bit < 8; bit += 1) {
+ crc = (crc & 1) === 1 ? (crc >>> 1) ^ 0xedb88320 : crc >>> 1;
+ }
+ }
+ return (crc ^ 0xffffffff) >>> 0;
+}
+
+function collectTextFiles(root: string, current: string, files: Record<string, string>) {
+ for (const entry of readdirSync(current, { withFileTypes: true })) {
+ const absolutePath = path.join(current, entry.name);
+ if (entry.isDirectory()) {
+ collectTextFiles(root, absolutePath, files);
+ continue;
+ }
+ if (!entry.isFile()) continue;
+ const relativePath = path.relative(root, absolutePath).replace(/\\/g, "/");
+ files[relativePath] = readFileSync(absolutePath, "utf8");
+ }
+}
+
+function createStoredZipArchive(files: Record<string, string>, rootPath: string) {
+ const encoder = new TextEncoder();
+ const localChunks: Uint8Array[] = [];
+ const centralChunks: Uint8Array[] = [];
+ let localOffset = 0;
+ let entryCount = 0;
+
+ for (const [relativePath, content] of Object.entries(files).sort(([left], [right]) => left.localeCompare(right))) {
+ const fileName = encoder.encode(`${rootPath}/${relativePath}`);
+ const body = encoder.encode(content);
+ const checksum = crc32(body);
+
+ const localHeader = new Uint8Array(30 + fileName.length);
+ writeUint32(localHeader, 0, 0x04034b50);
+ writeUint16(localHeader, 4, 20);
+ writeUint16(localHeader, 6, 0x0800);
+ writeUint16(localHeader, 8, 0);
+ writeUint32(localHeader, 14, checksum);
+ writeUint32(localHeader, 18, body.length);
+ writeUint32(localHeader, 22, body.length);
+ writeUint16(localHeader, 26, fileName.length);
+ localHeader.set(fileName, 30);
+
+ const centralHeader = new Uint8Array(46 + fileName.length);
+ writeUint32(centralHeader, 0, 0x02014b50);
+ writeUint16(centralHeader, 4, 20);
+ writeUint16(centralHeader, 6, 20);
+ writeUint16(centralHeader, 8, 0x0800);
+ writeUint16(centralHeader, 10, 0);
+ writeUint32(centralHeader, 16, checksum);
+ writeUint32(centralHeader, 20, body.length);
+ writeUint32(centralHeader, 24, body.length);
+ writeUint16(centralHeader, 28, fileName.length);
+ writeUint32(centralHeader, 42, localOffset);
+ centralHeader.set(fileName, 46);
+
+ localChunks.push(localHeader, body);
+ centralChunks.push(centralHeader);
+ localOffset += localHeader.length + body.length;
+ entryCount += 1;
+ }
+
+ const centralDirectoryLength = centralChunks.reduce((sum, chunk) => sum + chunk.length, 0);
+ const archive = new Uint8Array(
+ localChunks.reduce((sum, chunk) => sum + chunk.length, 0) + centralDirectoryLength + 22,
+ );
+ let offset = 0;
+ for (const chunk of localChunks) {
+ archive.set(chunk, offset);
+ offset += chunk.length;
+ }
+ const centralDirectoryOffset = offset;
+ for (const chunk of centralChunks) {
+ archive.set(chunk, offset);
+ offset += chunk.length;
+ }
+ writeUint32(archive, offset, 0x06054b50);
+ writeUint16(archive, offset + 8, entryCount);
+ writeUint16(archive, offset + 10, entryCount);
+ writeUint32(archive, offset + 12, centralDirectoryLength);
+ writeUint32(archive, offset + 16, centralDirectoryOffset);
+
+ return archive;
+}
+
async function stopServerProcess(child: ServerProcess | null) {
if (!child || child.exitCode !== null) return;
child.kill("SIGTERM");
@@ -345,6 +446,8 @@ describe("paperclipai company import/export e2e", () => {
},
);
+ const largeIssueDescription = `Round-trip the company package through the CLI.\n\n${"portable-data ".repeat(12_000)}`;
+
const sourceIssue = await api<{ id: string; title: string; identifier: string }>(
apiBase,
`/api/companies/${sourceCompany.id}/issues`,
@@ -353,7 +456,7 @@ describe("paperclipai company import/export e2e", () => {
headers: { "content-type": "application/json" },
body: JSON.stringify({
title: "Validate company import/export",
- description: "Round-trip the company package through the CLI.",
+ description: largeIssueDescription,
status: "todo",
projectId: sourceProject.id,
assigneeAgentId: sourceAgent.id,
@@ -397,6 +500,7 @@ describe("paperclipai company import/export e2e", () => {
`Imported ${sourceCompany.name}`,
"--include",
"company,agents,projects,issues",
+ "--yes",
],
{ apiBase, configPath },
);
@@ -470,6 +574,7 @@ describe("paperclipai company import/export e2e", () => {
"company,agents,projects,issues",
"--collision",
"rename",
+ "--yes",
],
{ apiBase, configPath },
);
@@ -494,5 +599,32 @@ describe("paperclipai company import/export e2e", () => {
expect(new Set(twiceImportedAgents.map((agent) => agent.name)).size).toBe(2);
expect(twiceImportedProjects).toHaveLength(2);
expect(twiceImportedIssues).toHaveLength(2);
+
+ const zipPath = path.join(tempRoot, "exported-company.zip");
+ const portableFiles: Record<string, string> = {};
+ collectTextFiles(exportDir, exportDir, portableFiles);
+ writeFileSync(zipPath, createStoredZipArchive(portableFiles, "paperclip-demo"));
+
+ const importedFromZip = await runCliJson<{
+ company: { id: string; name: string; action: string };
+ agents: Array<{ id: string | null; action: string; name: string }>;
+ }>(
+ [
+ "company",
+ "import",
+ zipPath,
+ "--target",
+ "new",
+ "--new-company-name",
+ `Zip Imported ${sourceCompany.name}`,
+ "--include",
+ "company,agents,projects,issues",
+ "--yes",
+ ],
+ { apiBase, configPath },
+ );
+
+ expect(importedFromZip.company.action).toBe("created");
+ expect(importedFromZip.agents.some((agent) => agent.action === "created")).toBe(true);
}, 60_000);
});
diff --git a/cli/src/__tests__/company-import-zip.test.ts b/cli/src/__tests__/company-import-zip.test.ts
new file mode 100644
index 0000000000..0c48a24a6e
--- /dev/null
+++ b/cli/src/__tests__/company-import-zip.test.ts
@@ -0,0 +1,133 @@
+import { mkdtemp, rm, writeFile } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { afterEach, describe, expect, it } from "vitest";
+import { resolveInlineSourceFromPath } from "../commands/client/company.js";
+
+function writeUint16(target: Uint8Array, offset: number, value: number) {
+ target[offset] = value & 0xff;
+ target[offset + 1] = (value >>> 8) & 0xff;
+}
+
+function writeUint32(target: Uint8Array, offset: number, value: number) {
+ target[offset] = value & 0xff;
+ target[offset + 1] = (value >>> 8) & 0xff;
+ target[offset + 2] = (value >>> 16) & 0xff;
+ target[offset + 3] = (value >>> 24) & 0xff;
+}
+
+function crc32(bytes: Uint8Array) {
+ let crc = 0xffffffff;
+ for (const byte of bytes) {
+ crc ^= byte;
+ for (let bit = 0; bit < 8; bit += 1) {
+ crc = (crc & 1) === 1 ? (crc >>> 1) ^ 0xedb88320 : crc >>> 1;
+ }
+ }
+ return (crc ^ 0xffffffff) >>> 0;
+}
+
+function createStoredZipArchive(files: Record<string, string>, rootPath: string) {
+ const encoder = new TextEncoder();
+ const localChunks: Uint8Array[] = [];
+ const centralChunks: Uint8Array[] = [];
+ let localOffset = 0;
+ let entryCount = 0;
+
+ for (const [relativePath, content] of Object.entries(files).sort(([left], [right]) => left.localeCompare(right))) {
+ const fileName = encoder.encode(`${rootPath}/${relativePath}`);
+ const body = encoder.encode(content);
+ const checksum = crc32(body);
+
+ const localHeader = new Uint8Array(30 + fileName.length);
+ writeUint32(localHeader, 0, 0x04034b50);
+ writeUint16(localHeader, 4, 20);
+ writeUint16(localHeader, 6, 0x0800);
+ writeUint16(localHeader, 8, 0);
+ writeUint32(localHeader, 14, checksum);
+ writeUint32(localHeader, 18, body.length);
+ writeUint32(localHeader, 22, body.length);
+ writeUint16(localHeader, 26, fileName.length);
+ localHeader.set(fileName, 30);
+
+ const centralHeader = new Uint8Array(46 + fileName.length);
+ writeUint32(centralHeader, 0, 0x02014b50);
+ writeUint16(centralHeader, 4, 20);
+ writeUint16(centralHeader, 6, 20);
+ writeUint16(centralHeader, 8, 0x0800);
+ writeUint16(centralHeader, 10, 0);
+ writeUint32(centralHeader, 16, checksum);
+ writeUint32(centralHeader, 20, body.length);
+ writeUint32(centralHeader, 24, body.length);
+ writeUint16(centralHeader, 28, fileName.length);
+ writeUint32(centralHeader, 42, localOffset);
+ centralHeader.set(fileName, 46);
+
+ localChunks.push(localHeader, body);
+ centralChunks.push(centralHeader);
+ localOffset += localHeader.length + body.length;
+ entryCount += 1;
+ }
+
+ const centralDirectoryLength = centralChunks.reduce((sum, chunk) => sum + chunk.length, 0);
+ const archive = new Uint8Array(
+ localChunks.reduce((sum, chunk) => sum + chunk.length, 0) + centralDirectoryLength + 22,
+ );
+
+ let offset = 0;
+ for (const chunk of localChunks) {
+ archive.set(chunk, offset);
+ offset += chunk.length;
+ }
+ const centralDirectoryOffset = offset;
+ for (const chunk of centralChunks) {
+ archive.set(chunk, offset);
+ offset += chunk.length;
+ }
+
+ writeUint32(archive, offset, 0x06054b50);
+ writeUint16(archive, offset + 8, entryCount);
+ writeUint16(archive, offset + 10, entryCount);
+ writeUint32(archive, offset + 12, centralDirectoryLength);
+ writeUint32(archive, offset + 16, centralDirectoryOffset);
+
+ return archive;
+}
+
+const tempDirs: string[] = [];
+
+afterEach(async () => {
+ for (const dir of tempDirs.splice(0)) {
+ await rm(dir, { recursive: true, force: true });
+ }
+});
+
+describe("resolveInlineSourceFromPath", () => {
+ it("imports portable files from a zip archive instead of scanning the parent directory", async () => {
+ const tempDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-company-import-zip-"));
+ tempDirs.push(tempDir);
+
+ const archivePath = path.join(tempDir, "paperclip-demo.zip");
+ const archive = createStoredZipArchive(
+ {
+ "COMPANY.md": "# Company\n",
+ ".paperclip.yaml": "schema: paperclip/v1\n",
+ "agents/ceo/AGENT.md": "# CEO\n",
+ "notes/todo.txt": "ignore me\n",
+ },
+ "paperclip-demo",
+ );
+ await writeFile(archivePath, archive);
+
+ const resolved = await resolveInlineSourceFromPath(archivePath);
+
+ expect(resolved).toEqual({
+ rootPath: "paperclip-demo",
+ files: {
+ "COMPANY.md": "# Company\n",
+ ".paperclip.yaml": "schema: paperclip/v1\n",
+ "agents/ceo/AGENT.md": "# CEO\n",
+ },
+ });
+ });
+});
diff --git a/cli/src/commands/client/company.ts b/cli/src/commands/client/company.ts
index dbf52890af..013f8e565a 100644
--- a/cli/src/commands/client/company.ts
+++ b/cli/src/commands/client/company.ts
@@ -12,6 +12,7 @@ import type {
CompanyPortabilityImportResult,
} from "@paperclipai/shared";
import { ApiRequestError } from "../../client/http.js";
+import { readZipArchive } from "./zip.js";
import {
addCommonClientOptions,
formatInlineRecord,
@@ -184,6 +185,14 @@ function normalizePortablePath(filePath: string): string {
return filePath.replace(/\\/g, "/");
}
+function shouldIncludePortableFile(filePath: string): boolean {
+ const baseName = path.basename(filePath);
+ const isMarkdown = baseName.endsWith(".md");
+ const isPaperclipYaml = baseName === ".paperclip.yaml" || baseName === ".paperclip.yml";
+ const contentType = binaryContentTypeByExtension[path.extname(baseName).toLowerCase()];
+ return isMarkdown || isPaperclipYaml || Boolean(contentType);
+}
+
function findPortableExtensionPath(files: Record<string, CompanyPortabilityFileEntry>): string | null {
if (files[".paperclip.yaml"] !== undefined) return ".paperclip.yaml";
if (files[".paperclip.yml"] !== undefined) return ".paperclip.yml";
@@ -853,21 +862,29 @@ async function collectPackageFiles(
continue;
}
if (!entry.isFile()) continue;
- const isMarkdown = entry.name.endsWith(".md");
- const isPaperclipYaml = entry.name === ".paperclip.yaml" || entry.name === ".paperclip.yml";
- const contentType = binaryContentTypeByExtension[path.extname(entry.name).toLowerCase()];
- if (!isMarkdown && !isPaperclipYaml && !contentType) continue;
const relativePath = path.relative(root, absolutePath).replace(/\\/g, "/");
+ if (!shouldIncludePortableFile(relativePath)) continue;
files[relativePath] = readPortableFileEntry(relativePath, await readFile(absolutePath));
}
}
-async function resolveInlineSourceFromPath(inputPath: string): Promise<{
+export async function resolveInlineSourceFromPath(inputPath: string): Promise<{
rootPath: string;
files: Record<string, CompanyPortabilityFileEntry>;
}> {
const resolved = path.resolve(inputPath);
const resolvedStat = await stat(resolved);
+ if (resolvedStat.isFile() && path.extname(resolved).toLowerCase() === ".zip") {
+ const archive = await readZipArchive(await readFile(resolved));
+ const filteredFiles = Object.fromEntries(
+ Object.entries(archive.files).filter(([relativePath]) => shouldIncludePortableFile(relativePath)),
+ );
+ return {
+ rootPath: archive.rootPath ?? path.basename(resolved, ".zip"),
+ files: filteredFiles,
+ };
+ }
+
const rootDir = resolvedStat.isDirectory() ? resolved : path.dirname(resolved);
const files: Record<string, CompanyPortabilityFileEntry> = {};
await collectPackageFiles(rootDir, rootDir, files);
diff --git a/cli/src/commands/client/zip.ts b/cli/src/commands/client/zip.ts
new file mode 100644
index 0000000000..ff1eb66992
--- /dev/null
+++ b/cli/src/commands/client/zip.ts
@@ -0,0 +1,129 @@
+import { inflateRawSync } from "node:zlib";
+import path from "node:path";
+import type { CompanyPortabilityFileEntry } from "@paperclipai/shared";
+
+const textDecoder = new TextDecoder();
+
+const binaryContentTypeByExtension: Record<string, string> = {
+ ".gif": "image/gif",
+ ".jpeg": "image/jpeg",
+ ".jpg": "image/jpeg",
+ ".png": "image/png",
+ ".svg": "image/svg+xml",
+ ".webp": "image/webp",
+};
+
+function normalizeArchivePath(pathValue: string) {
+ return pathValue
+ .replace(/\\/g, "/")
+ .split("/")
+ .filter(Boolean)
+ .join("/");
+}
+
+function readUint16(source: Uint8Array, offset: number) {
+ return source[offset]! | (source[offset + 1]! << 8);
+}
+
+function readUint32(source: Uint8Array, offset: number) {
+ return (
+ source[offset]! |
+ (source[offset + 1]! << 8) |
+ (source[offset + 2]! << 16) |
+ (source[offset + 3]! << 24)
+ ) >>> 0;
+}
+
+function sharedArchiveRoot(paths: string[]) {
+ if (paths.length === 0) return null;
+ const firstSegments = paths
+ .map((entry) => normalizeArchivePath(entry).split("/").filter(Boolean))
+ .filter((parts) => parts.length > 0);
+ if (firstSegments.length === 0) return null;
+ const candidate = firstSegments[0]![0]!;
+ return firstSegments.every((parts) => parts.length > 1 && parts[0] === candidate)
+ ? candidate
+ : null;
+}
+
+function bytesToPortableFileEntry(pathValue: string, bytes: Uint8Array): CompanyPortabilityFileEntry {
+ const contentType = binaryContentTypeByExtension[path.extname(pathValue).toLowerCase()];
+ if (!contentType) return textDecoder.decode(bytes);
+ return {
+ encoding: "base64",
+ data: Buffer.from(bytes).toString("base64"),
+ contentType,
+ };
+}
+
+async function inflateZipEntry(compressionMethod: number, bytes: Uint8Array) {
+ if (compressionMethod === 0) return bytes;
+ if (compressionMethod !== 8) {
+ throw new Error("Unsupported zip archive: only STORE and DEFLATE entries are supported.");
+ }
+ return new Uint8Array(inflateRawSync(bytes));
+}
+
+export async function readZipArchive(source: ArrayBuffer | Uint8Array): Promise<{
+ rootPath: string | null;
+ files: Record<string, CompanyPortabilityFileEntry>;
+}> {
+ const bytes = source instanceof Uint8Array ? source : new Uint8Array(source);
+ const entries: Array<{ path: string; body: CompanyPortabilityFileEntry }> = [];
+ let offset = 0;
+
+ while (offset + 4 <= bytes.length) {
+ const signature = readUint32(bytes, offset);
+ if (signature === 0x02014b50 || signature === 0x06054b50) break;
+ if (signature !== 0x04034b50) {
+ throw new Error("Invalid zip archive: unsupported local file header.");
+ }
+
+ if (offset + 30 > bytes.length) {
+ throw new Error("Invalid zip archive: truncated local file header.");
+ }
+
+ const generalPurposeFlag = readUint16(bytes, offset + 6);
+ const compressionMethod = readUint16(bytes, offset + 8);
+ const compressedSize = readUint32(bytes, offset + 18);
+ const fileNameLength = readUint16(bytes, offset + 26);
+ const extraFieldLength = readUint16(bytes, offset + 28);
+
+ if ((generalPurposeFlag & 0x0008) !== 0) {
+ throw new Error("Unsupported zip archive: data descriptors are not supported.");
+ }
+
+ const nameOffset = offset + 30;
+ const bodyOffset = nameOffset + fileNameLength + extraFieldLength;
+ const bodyEnd = bodyOffset + compressedSize;
+ if (bodyEnd > bytes.length) {
+ throw new Error("Invalid zip archive: truncated file contents.");
+ }
+
+ const rawArchivePath = textDecoder.decode(bytes.slice(nameOffset, nameOffset + fileNameLength));
+ const archivePath = normalizeArchivePath(rawArchivePath);
+ const isDirectoryEntry = /\/$/.test(rawArchivePath.replace(/\\/g, "/"));
+ if (archivePath && !isDirectoryEntry) {
+ const entryBytes = await inflateZipEntry(compressionMethod, bytes.slice(bodyOffset, bodyEnd));
+ entries.push({
+ path: archivePath,
+ body: bytesToPortableFileEntry(archivePath, entryBytes),
+ });
+ }
+
+ offset = bodyEnd;
+ }
+
+ const rootPath = sharedArchiveRoot(entries.map((entry) => entry.path));
+ const files: Record<string, CompanyPortabilityFileEntry> = {};
+ for (const entry of entries) {
+ const normalizedPath =
+ rootPath && entry.path.startsWith(`${rootPath}/`)
+ ? entry.path.slice(rootPath.length + 1)
+ : entry.path;
+ if (!normalizedPath) continue;
+ files[normalizedPath] = entry.body;
+ }
+
+ return { rootPath, files };
+}
diff --git a/server/src/app.ts b/server/src/app.ts
index 55a4e53b59..5535ab3df2 100644
--- a/server/src/app.ts
+++ b/server/src/app.ts
@@ -79,6 +79,8 @@ export async function createApp(
const app = express();
app.use(express.json({
+ // Company import/export payloads can inline full portable packages.
+ limit: "10mb",
verify: (req, _res, buf) => {
(req as unknown as { rawBody: Buffer }).rawBody = buf;
},