Skip to content
Python
12.8 KB
Raw Blame
// Scheduled experiment cleanup. Scope comes only from immutable environment,
// never event input. Provider methods are injected for local fault qualification.
"use strict";
function settings(env, context) {
  const {
    OWS_DEPLOYMENT_ID: id,
    OWS_ACCOUNT: account,
    OWS_REGION: region,
    OWS_STACK_NAME: stack,
    OWS_BUCKET: bucket,
    OWS_OBJECT_PREFIX: objectPrefix,
    OWS_EXECUTION_ROLE: role,
  } = env;
  if (
    !/^ows-[a-f0-9]{32}$/.test(id || "") ||
    !/^\d{12}$/.test(account || "") ||
    !/^[a-z]{2}-[a-z]+-\d+$/.test(region || "")
  )
    throw Error("invalid_scope");
  const prefix = stack?.replace(/-infra$/, "");
  if (
    !new RegExp(
      "^ows-[a-z][a-z0-9-]{1,28}[a-z0-9]-" + id.slice(4, 16) + "$",
    ).test(prefix || "") ||
    stack !== prefix + "-infra" ||
    bucket !== prefix + "-" + account ||
    objectPrefix !== id + "/" ||
    role !== `arn:aws:iam::${account}:role/${prefix}-executor`
  )
    throw Error("invalid_scope");
  if (
    context?.invokedFunctionArn !==
    `arn:aws:lambda:${region}:${account}:function:${prefix}-cleanup`
  )
    throw Error("invalid_function_identity");
  if (env.OWS_AUTO_EXPIRE !== undefined && !["true", "false"].includes(env.OWS_AUTO_EXPIRE))
    throw Error("invalid_expiry_policy");
  const expiry = Number(env.OWS_EXPIRY);
  if (!Number.isSafeInteger(expiry) || expiry <= 0)
    throw Error("invalid_expiry");
  return {
    id,
    account,
    region,
    stack,
    bucket,
    objectPrefix,
    role,
    prefix,
    expiry,
    autoExpire: env.OWS_AUTO_EXPIRE !== "false",
    table: prefix + "-control",
  };
}
function tagged(tags, id) {
  return (
    Array.isArray(tags) &&
    tags.some((t) => t.Key === "ows:deployment-id" && t.Value === id)
  );
}
async function run(s, api, now, event = {}) {
  if (!Number.isSafeInteger(now) || now < 0) throw Error("invalid_clock");
  if (event.mode && !["health", "cleanup"].includes(event.mode))
    throw Error("invalid_mode");
  let record = await api.readControl();
  if (
    record &&
    (record.id !== s.id ||
      record.expiry !== s.expiry ||
      !Number.isSafeInteger(record.version) ||
      record.version < 1 ||
      typeof record.stackArn !== "string")
  )
    throw Error("control_identity_mismatch");
  const stack = await api.stack();
  if (stack) {
    const arnPrefix = `arn:aws:cloudformation:${s.region}:${s.account}:stack/${s.stack}/`;
    if (
      stack.StackName !== s.stack ||
      !stack.StackId?.startsWith(arnPrefix) ||
      !/^[a-zA-Z0-9-]+$/.test(stack.StackId.slice(arnPrefix.length)) ||
      !tagged(stack.Tags, s.id)
    )
      throw Error("stack_ownership_mismatch");
    if (record && record.stackArn !== stack.StackId)
      throw Error("stack_replaced");
  }
  if (event.mode === "health")
    return {
      status: "healthy",
      identityBound: !!record,
      destructiveActions: 0,
    };
  // Before the first destructive request, bind the observed immutable ARN in a
  // guard-owned CAS record. A lost control write cannot authorize deletion.
  if (stack && !record) {
    record = {
      id: s.id,
      expiry: s.expiry,
      stackArn: stack.StackId,
      version: 1,
      volumes: {},
      deleteIntent: null,
    };
    await api.writeControl(record, null);
  }
  async function checkpoint() {
    const previous = record.version;
    record.version++;
    await api.writeControl(record, previous);
  }
  if (now < s.expiry || s.autoExpire === false)
    return { status: "not_due", destructiveActions: 0 };
  if (stack) {
    if (stack.StackStatus === "DELETE_IN_PROGRESS")
      return { status: "deleting", destructiveActions: 0 };
    if (stack.StackStatus === "DELETE_FAILED")
      return {
        status: "operator_attention",
        reason: "stack_delete_failed",
        destructiveActions: 0,
      };
    if (
      ![
        "CREATE_COMPLETE",
        "UPDATE_COMPLETE",
        "UPDATE_ROLLBACK_COMPLETE",
        "ROLLBACK_COMPLETE",
        "CREATE_FAILED",
        "UPDATE_FAILED",
      ].includes(stack.StackStatus)
    )
      return { status: "waiting_for_stack", destructiveActions: 0 };
    if (record.deleteIntent)
      return {
        status: "operator_attention",
        reason: "prior_delete_outcome_unknown",
        destructiveActions: 0,
      };
    const info = await api.bucket();
    let objects = [],
      uploads = [];
    if (info) {
      if (!tagged(info.tags, s.id) || info.versioning)
        throw Error("bucket_ownership_or_versioning");
      // Inventory is deliberately bounded and complete BEFORE deletion. Refuse
      // overflow/foreign content rather than deleting a shared or partial scope.
      objects = await api.objects();
      uploads = await api.uploads();
      if (
        objects.length > 2000 ||
        uploads.length > 200 ||
        [...objects, ...uploads].some(
          (o) => typeof o.Key !== "string" || !o.Key.startsWith(s.objectPrefix),
        )
      )
        throw Error("foreign_or_oversized_bucket_inventory");
    }
    for (let n = 0; n < objects.length; n += 1000)
      await api.deleteObjects(objects.slice(n, n + 1000).map((o) => o.Key));
    for (const upload of uploads) await api.abortUpload(upload);
    record.deleteIntent = { token: s.id + "-expiry", at: now };
    await checkpoint();
    // Unknown outcome is not retried. A later invocation reads original status.
    await api.deleteStack(record.stackArn, record.deleteIntent.token);
    return { status: "cleanup_requested", destructiveActions: 1 };
  }
  // Stack absence without a durable original binding is not enough evidence to
  // destroy orphan resources. Let an operator inspect incomplete bootstrap.
  if (!record) return { status: "unbound_stack_absent", destructiveActions: 0 };
  const volumes = await api.volumes();
  if (volumes.length > 100) throw Error("volume_inventory_limit");
  let deleted = 0,
    pending = 0;
  for (const v of volumes) {
    if (!/^vol-[a-f0-9]+$/.test(v.VolumeId || "") || !tagged(v.Tags, s.id))
      throw Error("volume_ownership_mismatch");
    if (v.State !== "available") {
      pending++;
      continue;
    }
    if (record.volumes[v.VolumeId])
      return {
        status: "operator_attention",
        reason: "prior_volume_delete_unknown",
        destructiveActions: deleted,
      };
    record.volumes[v.VolumeId] = true;
    await checkpoint();
    await api.deleteVolume(v.VolumeId);
    deleted++;
  }
  return {
    status: volumes.length ? "volumes_pending" : "teardown_observed_complete",
    pending,
    destructiveActions: deleted,
  };
}
function provider(s, libraries) {
  const cf = libraries?.cf ?? require("@aws-sdk/client-cloudformation"),
    S3 = libraries?.s3 ?? require("@aws-sdk/client-s3"),
    ec = libraries?.ec ?? require("@aws-sdk/client-ec2"),
    D = libraries?.ddb ?? require("@aws-sdk/client-dynamodb");
  const options = { region: s.region, maxAttempts: 1 },
    cfn = new cf.CloudFormationClient(options),
    s3 = new S3.S3Client(options),
    ec2 = new ec.EC2Client(options),
    ddb = new D.DynamoDBClient(options);
  const key = { deploymentId: { S: s.id } };
  async function pages(send, field, tokenName, maxRows) {
    const rows = [];
    let token;
    const seen = new Set();
    for (let n = 0; n < 20; n++) {
      const r = await send(token);
      const entries =
        r[field] ??
        (field === "Contents" && r.KeyCount === 0 && !r.IsTruncated
          ? []
          : undefined);
      if (!Array.isArray(entries)) throw Error("invalid_inventory");
      rows.push(...entries);
      if (rows.length > maxRows) throw Error("inventory_limit");
      token = r[tokenName];
      if (field === "Contents" && r.IsTruncated && !token)
        throw Error("incomplete_inventory");
      if (!token) return rows;
      if (typeof token !== "string" || token.length > 8192)
        throw Error("invalid_token");
      if (seen.has(token)) throw Error("repeated_token");
      seen.add(token);
    }
    throw Error("page_limit");
  }
  return {
    async readControl() {
      const r = await ddb.send(
        new D.GetItemCommand({
          TableName: s.table,
          Key: key,
          ConsistentRead: true,
        }),
      );
      return r.Item ? JSON.parse(r.Item.body.S) : null;
    },
    async writeControl(record, expected) {
      await ddb.send(
        new D.PutItemCommand({
          TableName: s.table,
          Item: {
            ...key,
            version: { N: String(record.version) },
            body: { S: JSON.stringify(record) },
          },
          ConditionExpression:
            expected === null
              ? "attribute_not_exists(deploymentId)"
              : "#v = :expected",
          ...(expected === null
            ? {}
            : {
                ExpressionAttributeNames: { "#v": "version" },
                ExpressionAttributeValues: {
                  ":expected": { N: String(expected) },
                },
              }),
        }),
      );
    },
    async stack() {
      try {
        const r = await cfn.send(
          new cf.DescribeStacksCommand({ StackName: s.stack }),
        );
        if (r.Stacks?.length !== 1) throw Error("invalid_stack_response");
        return r.Stacks[0];
      } catch (e) {
        if (
          e.name === "ValidationError" &&
          e.message === `Stack with id ${s.stack} does not exist`
        )
          return null;
        throw e;
      }
    },
    async bucket() {
      try {
        const tags = await s3.send(
          new S3.GetBucketTaggingCommand({
            Bucket: s.bucket,
            ExpectedBucketOwner: s.account,
          }),
        );
        const v = await s3.send(
          new S3.GetBucketVersioningCommand({
            Bucket: s.bucket,
            ExpectedBucketOwner: s.account,
          }),
        );
        return { tags: tags.TagSet, versioning: v.Status };
      } catch (e) {
        if (e.name === "NoSuchBucket") return null;
        throw e;
      }
    },
    objects: () =>
      pages(
        (t) =>
          s3.send(
            new S3.ListObjectsV2Command({
              Bucket: s.bucket,
              ExpectedBucketOwner: s.account,
              MaxKeys: 500,
              ...(t ? { ContinuationToken: t } : {}),
            }),
          ),
        "Contents",
        "NextContinuationToken",
        2000,
      ),
    async uploads() {
      const rows = [];
      let keyMarker, uploadMarker;
      const seen = new Set();
      for (let n = 0; n < 20; n++) {
        const r = await s3.send(
          new S3.ListMultipartUploadsCommand({
            Bucket: s.bucket,
            ExpectedBucketOwner: s.account,
            MaxUploads: 100,
            KeyMarker: keyMarker,
            UploadIdMarker: uploadMarker,
          }),
        );
        if (r.Uploads !== undefined && !Array.isArray(r.Uploads))
          throw Error("invalid_upload_inventory");
        rows.push(...(r.Uploads || []));
        if (rows.length > 200) throw Error("upload_limit");
        if (!r.IsTruncated) return rows;
        keyMarker = r.NextKeyMarker;
        uploadMarker = r.NextUploadIdMarker;
        const id = JSON.stringify([keyMarker, uploadMarker]);
        if (!keyMarker || !uploadMarker || seen.has(id))
          throw Error("upload_pagination");
        seen.add(id);
      }
      throw Error("upload_page_limit");
    },
    async deleteObjects(keys) {
      const r = await s3.send(
        new S3.DeleteObjectsCommand({
          Bucket: s.bucket,
          ExpectedBucketOwner: s.account,
          Delete: { Objects: keys.map((Key) => ({ Key })), Quiet: true },
        }),
      );
      if (r.Errors?.length) throw Error("object_cleanup_incomplete");
    },
    abortUpload: (u) =>
      s3.send(
        new S3.AbortMultipartUploadCommand({
          Bucket: s.bucket,
          ExpectedBucketOwner: s.account,
          Key: u.Key,
          UploadId: u.UploadId,
        }),
      ),
    deleteStack: (StackName, ClientRequestToken) =>
      cfn.send(
        new cf.DeleteStackCommand({
          StackName,
          ClientRequestToken,
          RoleARN: s.role,
        }),
      ),
    volumes: () =>
      pages(
        (t) =>
          ec2.send(
            new ec.DescribeVolumesCommand({
              Filters: [{ Name: "tag:ows:deployment-id", Values: [s.id] }],
              ...(t ? { NextToken: t } : {}),
            }),
          ),
        "Volumes",
        "NextToken",
        100,
      ),
    deleteVolume: (VolumeId) =>
      ec2.send(new ec.DeleteVolumeCommand({ VolumeId })),
  };
}
exports.settings = settings;
exports.run = run;
exports.provider = provider;
exports.handler = async (event, context) => {
  try {
    const s = settings(process.env, context);
    const result = await run(
      s,
      provider(s),
      Math.floor(Date.now() / 1000),
      event,
    );
    console.log(JSON.stringify({ deploymentId: s.id, ...result }));
    return result;
  } catch {
    throw Error(
      "OpenWorker cleanup incomplete; inspect the bound deployment and guard control record",
    );
  }
};