Skip to content

armgit / hypit

Public Imported from GitHub
0

Clone any viral video with AI agents. Not just a script, the whole workflow: swap the face, the words, the B-roll, ship 100 variants in one command, and get your 100M views.

feat: give the S3 ArtifactStore its streaming and retention facets

kashorinf38e649f87cc
diff --git a/packages/artifact-store-s3/src/activation.ts b/packages/artifact-store-s3/src/activation.ts
index f30552b8..f9ec180e 100644
--- a/packages/artifact-store-s3/src/activation.ts
+++ b/packages/artifact-store-s3/src/activation.ts
@@ -3,9 +3,11 @@ import {
runtimeConfigBoolean,
runtimeConfigExact,
runtimeConfigObject,
+ runtimeConfigPositiveInteger,
runtimeConfigString,
} from "@narratage/runtime-adapter";
+import { AwsS3ObjectClient } from "./client.js";
import { createS3ArtifactStorePackage } from "./store.js";
const s3ArtifactStoreRuntimeAdapter = createRuntimeServiceAdapterFacet({
@@ -13,7 +15,7 @@ const s3ArtifactStoreRuntimeAdapter = createRuntimeServiceAdapterFacet({
create(context) {
const config = runtimeConfigObject(context.config, "S3 ArtifactStore");
runtimeConfigExact(config, [
- "bucket", "prefix", "expectedBucketOwner", "region", "endpoint", "forcePathStyle",
+ "bucket", "prefix", "expectedBucketOwner", "region", "endpoint", "forcePathStyle", "partSizeBytes",
], "S3 ArtifactStore");
const bucket = runtimeConfigString(config.bucket, "S3 bucket");
if (bucket === undefined) throw new Error("S3 bucket is required");
@@ -30,8 +32,46 @@ const s3ArtifactStoreRuntimeAdapter = createRuntimeServiceAdapterFacet({
? {} : { endpoint: config.endpoint as string }),
...(runtimeConfigBoolean(config.forcePathStyle, "S3 forcePathStyle") === undefined
? {} : { forcePathStyle: config.forcePathStyle as boolean }),
+ ...(runtimeConfigPositiveInteger(config.partSizeBytes, "S3 partSizeBytes") === undefined
+ ? {} : { partSizeBytes: config.partSizeBytes as number }),
});
},
+ /**
+ * A bucket that cannot be reached, or is owned by another account, is a
+ * deployment fact a Build cannot discover for itself without first writing
+ * something. Say it here instead.
+ */
+ async doctor(context) {
+ const config = runtimeConfigObject(context.config, "S3 ArtifactStore");
+ const bucket = runtimeConfigString(config.bucket, "S3 bucket");
+ if (bucket === undefined) return [];
+ const client = new AwsS3ObjectClient({
+ ...(runtimeConfigString(config.region, "S3 region") === undefined
+ ? {} : { region: config.region as string }),
+ ...(runtimeConfigString(config.endpoint, "S3 endpoint") === undefined
+ ? {} : { endpoint: config.endpoint as string }),
+ ...(runtimeConfigBoolean(config.forcePathStyle, "S3 forcePathStyle") === undefined
+ ? {} : { forcePathStyle: config.forcePathStyle as boolean }),
+ });
+ const owner = runtimeConfigString(config.expectedBucketOwner, "S3 expectedBucketOwner");
+ try {
+ // A key that cannot exist: this answers "can I reach and read this
+ // bucket as the expected owner" without depending on any object.
+ await client.head({
+ Bucket: bucket,
+ Key: `${runtimeConfigString(config.prefix, "S3 prefix") ?? ""}/.svml-doctor-probe`.replace(/^\/+/u, ""),
+ ...(owner === undefined ? {} : { ExpectedBucketOwner: owner }),
+ });
+ return [];
+ } catch (error) {
+ return [{
+ severity: "error" as const,
+ code: "ARTIFACT_STORE_UNREACHABLE",
+ message: `S3 bucket ${bucket} cannot be read: ${error instanceof Error ? error.message : String(error)}`,
+ subject: context.instance,
+ }];
+ }
+ },
});
export const svmlPackage = {
diff --git a/packages/artifact-store-s3/src/client.ts b/packages/artifact-store-s3/src/client.ts
index 65a970c1..3db52cfe 100644
--- a/packages/artifact-store-s3/src/client.ts
+++ b/packages/artifact-store-s3/src/client.ts
@@ -1,17 +1,57 @@
import {
+ AbortMultipartUploadCommand,
+ CompleteMultipartUploadCommand,
+ CopyObjectCommand,
+ CreateMultipartUploadCommand,
+ DeleteObjectCommand,
GetObjectCommand,
+ HeadObjectCommand,
+ ListObjectsV2Command,
PutObjectCommand,
S3Client,
+ UploadPartCommand,
} from "@aws-sdk/client-s3";
import type {
+ AbortMultipartUploadCommandInput,
+ CompleteMultipartUploadCommandInput,
+ CopyObjectCommandInput,
+ CreateMultipartUploadCommandInput,
+ DeleteObjectCommandInput,
GetObjectCommandInput,
+ ListObjectsV2CommandInput,
PutObjectCommandInput,
S3ClientConfig,
+ UploadPartCommandInput,
} from "@aws-sdk/client-s3";
+export type S3ListPage = {
+ readonly keys: readonly string[];
+ readonly continuationToken?: string;
+};
+
+/**
+ * `put` and `get` are the whole port. The rest are optional: a client that
+ * omits them leaves the store implementing only the three-method ArtifactStore,
+ * which is exactly what the optional Streaming and Managed facets mean.
+ */
export type S3ObjectClient = {
put(input: PutObjectCommandInput): Promise<void>;
get(input: GetObjectCommandInput): Promise<Uint8Array | undefined>;
+ /**
+ * The object's bytes as they arrive. Separate from `get` because an Artifact
+ * may be a whole programme: a caller that asked to stream must not have the
+ * object assembled in memory on its behalf.
+ */
+ open?(input: GetObjectCommandInput): Promise<AsyncIterable<Uint8Array> | undefined>;
+ /** Undefined when the key is absent. */
+ head?(input: GetObjectCommandInput): Promise<{ readonly size: number } | undefined>;
+ createMultipart?(input: CreateMultipartUploadCommandInput): Promise<string>;
+ uploadPart?(input: UploadPartCommandInput): Promise<{ readonly etag: string }>;
+ completeMultipart?(input: CompleteMultipartUploadCommandInput): Promise<void>;
+ abortMultipart?(input: AbortMultipartUploadCommandInput): Promise<void>;
+ copy?(input: CopyObjectCommandInput): Promise<void>;
+ delete?(input: DeleteObjectCommandInput): Promise<void>;
+ list?(input: ListObjectsV2CommandInput): Promise<S3ListPage>;
};
function statusCode(error: unknown): number | undefined {
@@ -21,10 +61,19 @@ function statusCode(error: unknown): number | undefined {
return typeof metadata.httpStatusCode === "number" ? metadata.httpStatusCode : undefined;
}
+function absent(error: unknown): boolean {
+ return statusCode(error) === 404
+ || (error instanceof Error && (error.name === "NoSuchKey" || error.name === "NotFound"));
+}
+
export class AwsS3ObjectClient implements S3ObjectClient {
readonly #client: S3Client;
constructor(config: S3ClientConfig = {}) {
+ // Credentials come from the SDK default chain and are never rebuilt here
+ // from AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY. A managed runtime also
+ // injects AWS_SESSION_TOKEN, and a hand-assembled two-field credential
+ // object silently drops it — every request then fails InvalidAccessKeyId.
this.#client = new S3Client(config);
}
@@ -38,8 +87,78 @@ export class AwsS3ObjectClient implements S3ObjectClient {
if (response.Body === undefined) throw new Error(`S3 object ${input.Key ?? "<unknown>"} has no body`);
return Uint8Array.from(await response.Body.transformToByteArray());
} catch (error) {
- if (statusCode(error) === 404 || (error instanceof Error && error.name === "NoSuchKey")) return undefined;
+ if (absent(error)) return undefined;
+ throw error;
+ }
+ }
+
+ async open(input: GetObjectCommandInput): Promise<AsyncIterable<Uint8Array> | undefined> {
+ let body: AsyncIterable<Uint8Array>;
+ try {
+ const response = await this.#client.send(new GetObjectCommand(input));
+ if (response.Body === undefined) throw new Error(`S3 object ${input.Key ?? "<unknown>"} has no body`);
+ body = response.Body as unknown as AsyncIterable<Uint8Array>;
+ } catch (error) {
+ if (absent(error)) return undefined;
+ throw error;
+ }
+ return (async function* () {
+ for await (const chunk of body) yield Uint8Array.from(chunk);
+ })();
+ }
+
+ async head(input: GetObjectCommandInput): Promise<{ readonly size: number } | undefined> {
+ try {
+ const response = await this.#client.send(new HeadObjectCommand({
+ Bucket: input.Bucket,
+ Key: input.Key,
+ ...(input.ExpectedBucketOwner === undefined ? {} : { ExpectedBucketOwner: input.ExpectedBucketOwner }),
+ }));
+ return { size: response.ContentLength ?? 0 };
+ } catch (error) {
+ if (absent(error)) return undefined;
throw error;
}
}
+
+ async createMultipart(input: CreateMultipartUploadCommandInput): Promise<string> {
+ const response = await this.#client.send(new CreateMultipartUploadCommand(input));
+ if (response.UploadId === undefined) throw new Error("S3 did not return a multipart upload id");
+ return response.UploadId;
+ }
+
+ async uploadPart(input: UploadPartCommandInput): Promise<{ readonly etag: string }> {
+ const response = await this.#client.send(new UploadPartCommand(input));
+ if (response.ETag === undefined) throw new Error("S3 did not return a part ETag");
+ return { etag: response.ETag };
+ }
+
+ async completeMultipart(input: CompleteMultipartUploadCommandInput): Promise<void> {
+ await this.#client.send(new CompleteMultipartUploadCommand(input));
+ }
+
+ async abortMultipart(input: AbortMultipartUploadCommandInput): Promise<void> {
+ await this.#client.send(new AbortMultipartUploadCommand(input));
+ }
+
+ async copy(input: CopyObjectCommandInput): Promise<void> {
+ await this.#client.send(new CopyObjectCommand(input));
+ }
+
+ async delete(input: DeleteObjectCommandInput): Promise<void> {
+ await this.#client.send(new DeleteObjectCommand(input));
+ }
+
+ async list(input: ListObjectsV2CommandInput): Promise<S3ListPage> {
+ const response = await this.#client.send(new ListObjectsV2Command(input));
+ const keys = (response.Contents ?? [])
+ .map((item) => item.Key)
+ .filter((key): key is string => key !== undefined);
+ return {
+ keys,
+ ...(response.IsTruncated === true && response.NextContinuationToken !== undefined
+ ? { continuationToken: response.NextContinuationToken }
+ : {}),
+ };
+ }
}
diff --git a/packages/artifact-store-s3/src/store.ts b/packages/artifact-store-s3/src/store.ts
index 427e711c..414d007b 100644
--- a/packages/artifact-store-s3/src/store.ts
+++ b/packages/artifact-store-s3/src/store.ts
@@ -1,4 +1,4 @@
-import { createHash } from "node:crypto";
+import { createHash, randomUUID } from "node:crypto";
import { digestOf, isDigest } from "@narratage/protocol";
import type { BlobRef, Digest } from "@narratage/protocol";
@@ -6,7 +6,7 @@ import { defineRuntimeServicePackage } from "@narratage/runtime";
import type { ArtifactStore, RuntimeServicePackage } from "@narratage/runtime";
import { AwsS3ObjectClient } from "./client.js";
-import type { S3ObjectClient } from "./client.js";
+import type { S3ListPage, S3ObjectClient } from "./client.js";
export const s3ArtifactStoreModuleRef = {
name: "@narratage/artifact-store-s3",
@@ -26,10 +26,13 @@ type S3Location = {
export type S3ArtifactStoreOptions = S3Location & {
readonly client: S3ObjectClient;
readonly maxConflictRetries?: number;
+ /** Bytes buffered before a part is sent. S3 requires at least 5 MiB per non-final part. */
+ readonly partSizeBytes?: number;
};
export type CreateS3ArtifactStorePackageOptions = S3Location & {
readonly instance?: string;
+ readonly partSizeBytes?: number;
readonly region?: string;
readonly endpoint?: string;
readonly forcePathStyle?: boolean;
@@ -64,13 +67,27 @@ function statusCode(error: unknown): number | undefined {
return typeof metadata.httpStatusCode === "number" ? metadata.httpStatusCode : undefined;
}
-/** Whole-object S3 implementation of the content-addressed ArtifactStore port. */
+/**
+ * Content-addressed S3 ArtifactStore.
+ *
+ * The streaming and retention facets are detected by method presence, so this
+ * store attaches them only when the injected client can back them. Claiming a
+ * facet and then throwing would be worse than not claiming it: a deployment
+ * would discover the gap during a Build rather than when it chose the client.
+ */
export class S3ArtifactStore implements ArtifactStore {
readonly #client: S3ObjectClient;
readonly #bucket: string;
readonly #prefix: string;
readonly #expectedBucketOwner: string | undefined;
readonly #maxConflictRetries: number;
+ readonly #partSizeBytes: number;
+ /** Present only when the client can stream and upload in parts. */
+ readonly open?: (digest: Digest) => Promise<AsyncIterable<Uint8Array> | undefined>;
+ readonly putStream?: (chunks: AsyncIterable<Uint8Array>, mediaType: string) => Promise<BlobRef>;
+ /** Present only when the client can enumerate and remove objects. */
+ readonly list?: () => Promise<readonly Digest[]>;
+ readonly delete?: (digest: Digest) => Promise<boolean>;
constructor(options: S3ArtifactStoreOptions) {
assert(options.bucket.trim().length > 0, "S3 Artifact bucket must not be empty");
@@ -82,6 +99,18 @@ export class S3ArtifactStore implements ArtifactStore {
this.#prefix = normalizePrefix(options.prefix);
this.#expectedBucketOwner = options.expectedBucketOwner;
this.#maxConflictRetries = positiveInteger(options.maxConflictRetries ?? 3, "maxConflictRetries");
+ this.#partSizeBytes = positiveInteger(options.partSizeBytes ?? 16 * 1024 * 1024, "partSizeBytes");
+ assert(this.#partSizeBytes >= 5 * 1024 * 1024, "S3 requires multipart parts of at least 5 MiB");
+ const client = options.client;
+ if (client.open !== undefined && client.createMultipart !== undefined && client.uploadPart !== undefined
+ && client.completeMultipart !== undefined && client.copy !== undefined) {
+ this.open = (digest) => this.#openStream(digest);
+ this.putStream = (chunks, mediaType) => this.#putStream(chunks, mediaType);
+ }
+ if (client.list !== undefined && client.delete !== undefined && client.head !== undefined) {
+ this.list = () => this.#list();
+ this.delete = (digest) => this.#delete(digest);
+ }
}
key(digest: Digest): string {
@@ -149,6 +178,166 @@ export class S3ArtifactStore implements ArtifactStore {
// Existence by key is insufficient: ArtifactStore promises content identity, so verify bytes.
return (await this.get(digest)) !== undefined;
}
+
+ /**
+ * Stream the object, hashing as it passes. A whole-object read can refuse
+ * before returning anything; a stream cannot, because its first bytes are
+ * already with the caller by the time the last ones arrive. So verification
+ * lands at the end: the iterator throws instead of completing, and a consumer
+ * that treats an incomplete stream as an error — which every consumer must —
+ * still never accepts wrong bytes as an Artifact.
+ */
+ async #openStream(digest: Digest): Promise<AsyncIterable<Uint8Array> | undefined> {
+ const chunks = await this.#client.open!({
+ Bucket: this.#bucket,
+ Key: this.key(digest),
+ ...(this.#expectedBucketOwner === undefined
+ ? {}
+ : { ExpectedBucketOwner: this.#expectedBucketOwner }),
+ });
+ if (chunks === undefined) return undefined;
+ return (async function* () {
+ const hash = createHash("sha256");
+ for await (const chunk of chunks) {
+ hash.update(chunk);
+ yield chunk;
+ }
+ const actual = `sha256:${hash.digest("hex")}`;
+ if (actual !== digest) throw new Error(`Artifact ${digest} content digest differs`);
+ })();
+ }
+
+ /**
+ * Multipart upload to a staging key, then a server-side copy onto the
+ * content-addressed key.
+ *
+ * The detour exists because S3 needs the destination key before the first
+ * part, and a content-addressed key is not known until the last byte has been
+ * hashed. The copy is server-side, so the bytes cross the wire once.
+ */
+ async #putStream(chunks: AsyncIterable<Uint8Array>, mediaType: string): Promise<BlobRef> {
+ assert(mediaType.trim().length > 0, "Artifact mediaType must not be empty");
+ const client = this.#client as Required<S3ObjectClient>;
+ const owner = this.#expectedBucketOwner === undefined
+ ? {}
+ : { ExpectedBucketOwner: this.#expectedBucketOwner };
+ const staging = `${this.#prefix.length === 0 ? "" : `${this.#prefix}/`}.incoming/${randomUUID()}`;
+ const uploadId = await client.createMultipart({
+ Bucket: this.#bucket, Key: staging, ContentType: mediaType, ...owner,
+ });
+ const hash = createHash("sha256");
+ const parts: { PartNumber: number; ETag: string }[] = [];
+ let size = 0;
+ try {
+ let pending: Uint8Array[] = [];
+ let pendingBytes = 0;
+ const flush = async (): Promise<void> => {
+ if (pendingBytes === 0) return;
+ const body = new Uint8Array(pendingBytes);
+ let offset = 0;
+ for (const piece of pending) {
+ body.set(piece, offset);
+ offset += piece.byteLength;
+ }
+ pending = [];
+ pendingBytes = 0;
+ const partNumber = parts.length + 1;
+ const { etag } = await client.uploadPart!({
+ Bucket: this.#bucket, Key: staging, UploadId: uploadId,
+ PartNumber: partNumber, Body: body, ContentLength: body.byteLength, ...owner,
+ });
+ parts.push({ PartNumber: partNumber, ETag: etag });
+ };
+ for await (const value of chunks) {
+ if (!(value instanceof Uint8Array)) throw new Error("Artifact stream yielded non-bytes");
+ const chunk = Uint8Array.from(value);
+ hash.update(chunk);
+ size += chunk.byteLength;
+ if (!Number.isSafeInteger(size)) throw new Error("Artifact stream exceeds the supported size");
+ pending.push(chunk);
+ pendingBytes += chunk.byteLength;
+ if (pendingBytes >= this.#partSizeBytes) await flush();
+ }
+ await flush();
+ // Only now can the stream be known to have been empty. S3 rejects a
+ // multipart upload with no parts, and an empty Artifact is a legitimate
+ // one, so that case still sends a single empty part.
+ if (parts.length === 0) {
+ const { etag } = await client.uploadPart({
+ Bucket: this.#bucket, Key: staging, UploadId: uploadId,
+ PartNumber: 1, Body: new Uint8Array(0), ContentLength: 0, ...owner,
+ });
+ parts.push({ PartNumber: 1, ETag: etag });
+ }
+ await client.completeMultipart({
+ Bucket: this.#bucket, Key: staging, UploadId: uploadId,
+ MultipartUpload: { Parts: parts }, ...owner,
+ });
+ } catch (error) {
+ await client.abortMultipart?.({
+ Bucket: this.#bucket, Key: staging, UploadId: uploadId, ...owner,
+ }).catch(() => undefined);
+ throw error;
+ }
+ const digest = `sha256:${hash.digest("hex")}` as Digest;
+ try {
+ await client.copy({
+ Bucket: this.#bucket,
+ Key: this.key(digest),
+ CopySource: `${this.#bucket}/${staging}`,
+ ContentType: mediaType,
+ MetadataDirective: "REPLACE",
+ Metadata: { "svml-digest": digest, "svml-size": String(size) },
+ IfNoneMatch: "*",
+ ...owner,
+ });
+ } catch (error) {
+ // 412 means another Build stored these exact bytes first. Content
+ // addressing makes that agreement, not conflict.
+ if (statusCode(error) !== 412) throw error;
+ } finally {
+ await client.delete?.({ Bucket: this.#bucket, Key: staging, ...owner }).catch(() => undefined);
+ }
+ return { kind: "blob", digest, size, mediaType };
+ }
+
+ /** Every Artifact this store holds. Staging keys are not Artifacts and are not listed. */
+ async #list(): Promise<readonly Digest[]> {
+ const root = this.#prefix.length === 0 ? "sha256/" : `${this.#prefix}/sha256/`;
+ const digests: Digest[] = [];
+ let continuationToken: string | undefined;
+ do {
+ const page: S3ListPage = await this.#client.list!({
+ Bucket: this.#bucket,
+ Prefix: root,
+ ...(continuationToken === undefined ? {} : { ContinuationToken: continuationToken }),
+ ...(this.#expectedBucketOwner === undefined
+ ? {}
+ : { ExpectedBucketOwner: this.#expectedBucketOwner }),
+ });
+ for (const key of page.keys) {
+ const hex = key.slice(root.length).split("/").at(-1);
+ if (hex === undefined) continue;
+ const digest = `sha256:${hex}`;
+ // A key this store did not write is not reported as an Artifact.
+ if (isDigest(digest) && this.key(digest as Digest) === key) digests.push(digest as Digest);
+ }
+ continuationToken = page.continuationToken;
+ } while (continuationToken !== undefined);
+ return digests;
+ }
+
+ /** True when the Artifact was there to remove. */
+ async #delete(digest: Digest): Promise<boolean> {
+ const key = this.key(digest);
+ const owner = this.#expectedBucketOwner === undefined
+ ? {}
+ : { ExpectedBucketOwner: this.#expectedBucketOwner };
+ const existing = await this.#client.head!({ Bucket: this.#bucket, Key: key, ...owner });
+ if (existing === undefined) return false;
+ await this.#client.delete!({ Bucket: this.#bucket, Key: key, ...owner });
+ return true;
+ }
}
export function createS3ArtifactStorePackage(
@@ -164,6 +353,7 @@ export function createS3ArtifactStorePackage(
...(options.region === undefined ? {} : { region: options.region }),
...(options.endpoint === undefined ? {} : { endpoint: options.endpoint }),
...(options.forcePathStyle === undefined ? {} : { forcePathStyle: options.forcePathStyle }),
+ ...(options.partSizeBytes === undefined ? {} : { partSizeBytes: options.partSizeBytes }),
};
const client = options.client ?? new AwsS3ObjectClient({
...(options.region === undefined ? {} : { region: options.region }),
@@ -186,6 +376,7 @@ export function createS3ArtifactStorePackage(
service: new S3ArtifactStore({
client,
bucket: options.bucket,
+ ...(options.partSizeBytes === undefined ? {} : { partSizeBytes: options.partSizeBytes }),
...(prefix.length === 0 ? {} : { prefix }),
...(options.expectedBucketOwner === undefined
? {}
diff --git a/packages/artifact-store-s3/test/store.test.ts b/packages/artifact-store-s3/test/store.test.ts
index 6e0f03f1..31cb2d8e 100644
--- a/packages/artifact-store-s3/test/store.test.ts
+++ b/packages/artifact-store-s3/test/store.test.ts
@@ -6,6 +6,7 @@ import {
createS3ArtifactStorePackage,
} from "@narratage/artifact-store-s3";
import type { S3ObjectClient } from "@narratage/artifact-store-s3";
+import { isManagedArtifactStore, isStreamingArtifactStore } from "@narratage/runtime";
class FakeS3 implements S3ObjectClient {
readonly values = new Map<string, Uint8Array>();
@@ -58,3 +59,141 @@ test("configured S3 service locks location without putting AWS credentials in th
assert.ok(configured.services[0]?.instance.configurationDigest);
assert.equal(JSON.stringify(configured).includes("AWS_SECRET_ACCESS_KEY"), false);
});
+
+/** A client that can do everything, backed by an in-memory bucket. */
+class FullFakeS3 extends FakeS3 {
+ readonly uploads = new Map<string, Uint8Array[]>();
+ copies = 0;
+
+ async open(input: Parameters<NonNullable<S3ObjectClient["open"]>>[0]) {
+ const value = this.values.get(input.Key!);
+ if (value === undefined) return undefined;
+ // Two chunks, so a consumer cannot assume one whole-object read.
+ const half = Math.ceil(value.byteLength / 2);
+ return (async function* () {
+ yield Uint8Array.from(value.subarray(0, half));
+ yield Uint8Array.from(value.subarray(half));
+ })();
+ }
+
+ async head(input: Parameters<NonNullable<S3ObjectClient["head"]>>[0]) {
+ const value = this.values.get(input.Key!);
+ return value === undefined ? undefined : { size: value.byteLength };
+ }
+
+ async createMultipart(input: Parameters<NonNullable<S3ObjectClient["createMultipart"]>>[0]) {
+ const id = `upload-${this.uploads.size + 1}`;
+ this.uploads.set(`${id}:${input.Key!}`, []);
+ return id;
+ }
+
+ async uploadPart(input: Parameters<NonNullable<S3ObjectClient["uploadPart"]>>[0]) {
+ const parts = this.uploads.get(`${input.UploadId!}:${input.Key!}`)!;
+ parts[input.PartNumber! - 1] = Uint8Array.from(input.Body as Uint8Array);
+ return { etag: `"etag-${input.PartNumber}"` };
+ }
+
+ async completeMultipart(input: Parameters<NonNullable<S3ObjectClient["completeMultipart"]>>[0]) {
+ const parts = this.uploads.get(`${input.UploadId!}:${input.Key!}`)!;
+ const size = parts.reduce((total, part) => total + part.byteLength, 0);
+ const joined = new Uint8Array(size);
+ let offset = 0;
+ for (const part of parts) {
+ joined.set(part, offset);
+ offset += part.byteLength;
+ }
+ this.values.set(input.Key!, joined);
+ }
+
+ async abortMultipart() {}
+
+ async copy(input: Parameters<NonNullable<S3ObjectClient["copy"]>>[0]) {
+ this.copies += 1;
+ const source = input.CopySource!.slice(input.CopySource!.indexOf("/") + 1);
+ const value = this.values.get(source);
+ if (value === undefined) throw new Error(`copy source ${source} is absent`);
+ if (input.IfNoneMatch === "*" && this.values.has(input.Key!)) {
+ throw { $metadata: { httpStatusCode: 412 } };
+ }
+ this.values.set(input.Key!, Uint8Array.from(value));
+ }
+
+ async delete(input: Parameters<NonNullable<S3ObjectClient["delete"]>>[0]) {
+ this.values.delete(input.Key!);
+ }
+
+ async list(input: Parameters<NonNullable<S3ObjectClient["list"]>>[0]) {
+ const keys = [...this.values.keys()].filter((key) => key.startsWith(input.Prefix ?? "")).sort();
+ return { keys };
+ }
+}
+
+test("a client that cannot stream or enumerate makes the store admit it, rather than throw later", () => {
+ const store = new S3ArtifactStore({ client: new FakeS3(), bucket: "fixture" });
+ assert.equal(isStreamingArtifactStore(store), false);
+ assert.equal(isManagedArtifactStore(store), false);
+ const full = new S3ArtifactStore({ client: new FullFakeS3(), bucket: "fixture" });
+ assert.equal(isStreamingArtifactStore(full), true);
+ assert.equal(isManagedArtifactStore(full), true);
+});
+
+test("a streamed Artifact reaches the content-addressed key it earned by being hashed", async () => {
+ const client = new FullFakeS3();
+ const store = new S3ArtifactStore({ client, bucket: "fixture", prefix: "svml" });
+ const parts = ["first ", "second ", "third"].map((text) => new TextEncoder().encode(text));
+ const ref = await store.putStream!((async function* () { yield* parts; })(), "video/mp4");
+
+ const whole = new TextEncoder().encode("first second third");
+ assert.equal(ref.size, whole.byteLength);
+ assert.deepEqual(await store.get(ref.digest), whole);
+ assert.match(store.key(ref.digest), /^svml\/sha256\//u);
+ // The staging key is gone, so it is not mistaken for an Artifact.
+ assert.deepEqual([...client.values.keys()].filter((key) => key.includes(".incoming")), []);
+});
+
+test("an empty Artifact is a legitimate one, and S3 will not accept a partless upload", async () => {
+ const store = new S3ArtifactStore({ client: new FullFakeS3(), bucket: "fixture" });
+ const ref = await store.putStream!((async function* () {})(), "application/octet-stream");
+ assert.equal(ref.size, 0);
+ assert.deepEqual(await store.get(ref.digest), new Uint8Array(0));
+});
+
+test("two Builds streaming identical bytes agree instead of conflicting", async () => {
+ const client = new FullFakeS3();
+ const store = new S3ArtifactStore({ client, bucket: "fixture" });
+ const bytes = () => (async function* () { yield new TextEncoder().encode("same"); })();
+ const first = await store.putStream!(bytes(), "text/plain");
+ const second = await store.putStream!(bytes(), "text/plain");
+ assert.equal(first.digest, second.digest);
+ assert.equal(client.copies, 2, "both attempted the copy; the second was refused as already present");
+});
+
+test("a streamed read hands back bytes as they arrive, and refuses tampered content at the end", async () => {
+ const client = new FullFakeS3();
+ const store = new S3ArtifactStore({ client, bucket: "fixture" });
+ const bytes = new TextEncoder().encode("streamed artifact bytes");
+ const ref = await store.put(bytes, "text/plain");
+
+ const chunks: Uint8Array[] = [];
+ for await (const chunk of (await store.open!(ref.digest))!) chunks.push(chunk);
+ assert.equal(chunks.length, 2, "the stream was not assembled on the caller's behalf");
+
+ client.values.set(store.key(ref.digest), new TextEncoder().encode("tampered artifact bytes"));
+ await assert.rejects(async () => {
+ for await (const chunk of (await store.open!(ref.digest))!) void chunk;
+ }, /content digest differs/u);
+});
+
+test("retention reports the Artifacts it stored, and whether a delete found one", async () => {
+ const client = new FullFakeS3();
+ const store = new S3ArtifactStore({ client, bucket: "fixture", prefix: "svml" });
+ const one = await store.put(new TextEncoder().encode("one"), "text/plain");
+ const two = await store.put(new TextEncoder().encode("two"), "text/plain");
+ // A key under the same prefix that this store did not write is not an Artifact.
+ client.values.set("svml/sha256/zz/not-a-digest", new Uint8Array(1));
+
+ assert.deepEqual([...await store.list!()].sort(), [one.digest, two.digest].sort());
+ assert.equal(await store.delete!(one.digest), true);
+ assert.equal(await store.delete!(one.digest), false, "deleting what is gone is not an error");
+ assert.deepEqual(await store.list!(), [two.digest]);
+});