Skip to content

Open-source Grok Bot alternative for named AI teammates. Local-first desktop, web, and mobile clients with per-bot tools, instructions, and memory. Independent fork of T3 Code.

15.9 KB
Raw Blame
import * as Predicate from "effect/Predicate";
import {
  type AkeruDelegationRecord,
  RoutineRunId,
  type OrchestrationEvent,
} from "@akeru/contracts";
import * as Clock from "effect/Clock";
import * as DateTime from "effect/DateTime";
import * as Effect from "effect/Effect";
import * as Layer from "effect/Layer";
import * as Option from "effect/Option";
import * as Schedule from "effect/Schedule";
import * as Stream from "effect/Stream";

import { OrchestrationEngineService } from "../orchestration/Services/OrchestrationEngine.ts";
import { RoutineRepository, type RoutineClaim } from "./Repository.ts";
import { RoutineRuntime, type RoutineRuntimeShape } from "./Runtime.ts";
import { latestScheduledFor, nextScheduledFor } from "./schedule.ts";
import {
  RoutineRuntimeAdapter,
  type Routine,
  type RoutineDependencyFailure,
  type RoutineRun,
  type RoutineRunTrigger,
} from "./types.ts";

const approvalFailure: RoutineDependencyFailure = {
  kind: "approval",
  reason: "The approved procedure version is not current.",
  nextAction: "Review and approve the current procedure, then resume the routine.",
};

const makeRun = (routine: Routine, claim: RoutineClaim): RoutineRun => {
  const base = {
    id: claim.runId,
    routineId: claim.routineId,
    procedureVersion: routine.procedureVersion,
    status: "queued" as const,
    threadRef: null,
    result: null,
    failure: null,
    usageRef: null,
    startedAt: null,
    completedAt: null,
    createdAt: claim.claimedAt,
    updatedAt: claim.claimedAt,
  };

  if (claim.scheduledFor === null) {
    return { ...base, trigger: claim.trigger, scheduledFor: null };
  }

  return { ...base, trigger: claim.trigger, scheduledFor: claim.scheduledFor };
};

const makeRunId = (routineId: string, scheduledFor: string) =>
  RoutineRunId.make(`routine:${routineId}:${scheduledFor}`);

const make = Effect.gen(function* () {
  const repository = yield* RoutineRepository;
  const adapter = yield* RoutineRuntimeAdapter;
  const engine = yield* Effect.serviceOption(OrchestrationEngineService);

  const block = Effect.fn("RoutineRuntime.block")(function* (
    routine: Routine,
    run: RoutineRun,
    failure: RoutineDependencyFailure,
  ) {
    const completedAt = DateTime.formatIso(yield* DateTime.now);
    yield* adapter.recordBlocked(run, failure);
    yield* adapter.openFailureIncident(routine, failure);
    yield* repository.markBlocked(run.id, failure.reason, completedAt);
  });

  const execute = Effect.fn("RoutineRuntime.execute")(function* (
    routine: Routine,
    claim: RoutineClaim,
  ) {
    const run = makeRun(routine, claim);
    yield* adapter.recordQueued(run);

    const failure =
      claim.trigger !== "dry-run" && routine.approvalVersion !== routine.procedureVersion
        ? approvalFailure
        : yield* adapter.checkDependencies(routine);

    if (failure !== null) {
      yield* block(routine, run, failure);

      return run;
    }

    if (claim.trigger === "dry-run") {
      const completedAt = DateTime.formatIso(yield* DateTime.now);
      yield* adapter.recordCompleted(run, routine.nextRunAt, "Dry run passed.", completedAt);
      yield* repository.markSettled(run.id, "completed", completedAt);

      return { ...run, status: "completed" as const, completedAt, summary: "Dry run passed." };
    }

    const dispatched = yield* adapter.dispatchTurn(routine, run);

    if ("failure" in dispatched) {
      yield* block(routine, run, dispatched.failure);

      return run;
    }

    if ("canceled" in dispatched) {
      const completedAt = DateTime.formatIso(yield* DateTime.now);
      yield* repository.markSettled(run.id, "canceled", completedAt);

      return { ...run, status: "canceled" as const, completedAt };
    }

    yield* repository.markDispatched(run.id, dispatched.threadRef);

    return { ...run, status: "running" as const, ...dispatched };
  });

  const claimAndExecute = Effect.fn("RoutineRuntime.claimAndExecute")(function* (
    routine: Routine,
    trigger: Extract<RoutineRunTrigger, "scheduled" | "missed">,
    scheduledFor: string,
    claimedAt: string,
  ) {
    if (yield* adapter.isTargetBusy(routine)) return null;

    const claim = {
      runId: makeRunId(routine.id, scheduledFor),
      routineId: routine.id,
      trigger,
      scheduledFor,
      claimedAt,
    } satisfies RoutineClaim;

    if (!(yield* repository.claim(claim))) return null;

    return yield* execute(routine, claim);
  });

  const runDue = Effect.gen(function* () {
    const nowEpochMillis = yield* Clock.currentTimeMillis;
    const now = DateTime.formatIso(yield* DateTime.now);
    const routines = yield* repository.listEnabled;

    for (const routine of routines) {
      if (routine.nextRunAt === null || Date.parse(routine.nextRunAt) > nowEpochMillis) continue;
      const scheduledFor = latestScheduledFor(routine.schedule, routine.timezone, nowEpochMillis);
      const trigger = scheduledFor === routine.nextRunAt ? "scheduled" : "missed";
      yield* claimAndExecute(routine, trigger, scheduledFor, now);
    }
  }).pipe(
    Effect.catchCause((cause) => Effect.logError("routine scheduler tick failed", { cause })),
  );

  const canRunNow: RoutineRuntimeShape["canRunNow"] = (routineId) =>
    Effect.gen(function* () {
      const routine = yield* repository.getById(routineId);

      return (
        routine !== null &&
        routine.lifecycle !== "deleted" &&
        !(yield* adapter.isTargetBusy(routine))
      );
    });

  const runNow: RoutineRuntimeShape["runNow"] = (routineId, runId, trigger) =>
    Effect.gen(function* () {
      const routine = yield* repository.getById(routineId);

      if (routine === null || routine.lifecycle === "deleted") return null;
      const now = DateTime.formatIso(yield* DateTime.now);

      const claim = {
        runId,
        routineId,
        trigger,
        scheduledFor: null,
        claimedAt: now,
      } satisfies RoutineClaim;

      if (!(yield* repository.claim(claim))) return null;

      return yield* execute(routine, claim);
    });

  // Scheduled bot work settles its run from the delegation's terminal phase.
  const settleDelegatedRun = Effect.fn("RoutineRuntime.settleDelegatedRun")(function* (
    delegation: AkeruDelegationRecord,
  ) {
    const phase = delegation.phase;

    if (
      delegation.trigger !== "scheduled" ||
      (!Predicate.isTagged(phase, "Completed") &&
        !Predicate.isTagged(phase, "Failed") &&
        !Predicate.isTagged(phase, "Canceled")) ||
      phase.childThreadId === null
    )
      return false;

    const run = (yield* repository.listAllRuns).find(
      (candidate) =>
        candidate.threadRef === phase.childThreadId &&
        (candidate.status === "queued" ||
          candidate.status === "running" ||
          candidate.status === "waiting-for-approval"),
    );

    if (run === undefined) return false;
    const routine = yield* repository.getById(run.routineId);

    if (routine === null || routine.lifecycle === "deleted") return false;
    const completedAt = phase.completedAt;

    if (Predicate.isTagged(phase, "Completed")) {
      const nextRunAt = routine.enabled
        ? nextScheduledFor(routine.schedule, routine.timezone, Date.parse(completedAt))
        : null;

      yield* adapter.recordCompleted(run, nextRunAt, phase.result.summary, completedAt);
      yield* repository.markSettled(run.id, "completed", completedAt);

      return true;
    }

    if (Predicate.isTagged(phase, "Canceled")) {
      yield* adapter.recordCanceled(run, completedAt);
      yield* repository.markSettled(run.id, "canceled", completedAt);

      return true;
    }

    const failure = {
      kind: "execution",
      reason: phase.failure.message,
      nextAction: "Review the bot work in the routine chat, then resume the routine.",
    } satisfies RoutineDependencyFailure;

    yield* adapter.recordFailed(run, failure, completedAt);
    yield* adapter.openFailureIncident(routine, failure);
    yield* repository.markBlocked(run.id, failure.reason, completedAt);

    return true;
  });

  const settleRunForEvent = Effect.fn("RoutineRuntime.settleRunForEvent")(function* (
    event: OrchestrationEvent,
  ) {
    if (event.type === "routine.deleted") {
      yield* adapter.resolveFailureIncident(event.payload.routine.id);

      return;
    }

    if (event.type === "delegation.updated") {
      yield* settleDelegatedRun(event.payload.delegation);

      return;
    }

    // Canceling a run also cancels the bot work it started. Pausing does not.
    if (event.type === "routine.run-canceled") {
      const run = event.payload.run;
      yield* repository.markSettled(run.id, "canceled", run.completedAt ?? event.occurredAt);
      yield* adapter.cancelDelegatedRun(run);

      return;
    }

    if (event.type !== "thread.turn-diff-completed" && event.type !== "thread.session-set") return;

    if (
      event.type === "thread.session-set" &&
      event.payload.session.status !== "ready" &&
      event.payload.session.status !== "error" &&
      event.payload.session.status !== "interrupted" &&
      event.payload.session.status !== "stopped"
    )
      return;
    const threadRef = event.payload.threadId;

    const run = yield* repository.getActiveRunByThreadRef(
      threadRef,
      event.type === "thread.turn-diff-completed" ? event.payload.turnId : null,
    );

    if (run === null) return;
    const routine = yield* repository.getById(run.routineId);

    if (routine === null || routine.lifecycle === "deleted") return;

    const completedAt =
      event.type === "thread.turn-diff-completed"
        ? event.payload.completedAt
        : event.payload.session.updatedAt;

    if (
      (event.type === "thread.turn-diff-completed" && event.payload.status === "ready") ||
      (event.type === "thread.session-set" && event.payload.session.status === "ready")
    ) {
      const nextRunAt = routine.enabled
        ? nextScheduledFor(routine.schedule, routine.timezone, Date.parse(completedAt))
        : null;

      yield* adapter.recordCompleted(run, nextRunAt, "Routine completed.", completedAt);
      yield* repository.markSettled(run.id, "completed", completedAt);

      return;
    }

    const failure = {
      kind: "execution",
      reason:
        event.type === "thread.turn-diff-completed"
          ? `Routine checkpoint ended with status '${event.payload.status}'.`
          : (event.payload.session.lastError ??
            `Routine session ended with status '${event.payload.session.status}'.`),
      nextAction: "Review the routine chat, then resume the routine.",
    } satisfies RoutineDependencyFailure;

    yield* adapter.recordFailed(run, failure, completedAt);
    yield* adapter.openFailureIncident(routine, failure);
    yield* repository.markBlocked(run.id, failure.reason, completedAt);
  });

  const recover = Effect.gen(function* () {
    for (const routine of yield* repository.listAll) {
      if (routine.lifecycle === "deleted") yield* adapter.resolveFailureIncident(routine.id);
    }

    for (const claim of yield* repository.listRecoverable) {
      const routine = yield* repository.getById(claim.routineId);

      if (routine === null || routine.lifecycle === "deleted") {
        yield* repository.markBlocked(
          claim.runId,
          "The routine no longer exists.",
          DateTime.formatIso(yield* DateTime.now),
        );
        continue;
      }

      const projectedRun = (yield* repository.listRuns(claim.routineId)).find(
        (candidate) => candidate.id === claim.runId,
      );

      if (projectedRun?.status === "completed") {
        yield* repository.markSettled(
          claim.runId,
          "completed",
          projectedRun.completedAt ?? projectedRun.updatedAt,
        );
        continue;
      }

      if (projectedRun?.status === "blocked") {
        yield* repository.markBlocked(
          claim.runId,
          projectedRun.failure?.message ?? "The routine run was blocked.",
          projectedRun.completedAt ?? projectedRun.updatedAt,
        );
        continue;
      }

      if (projectedRun?.status === "failed" || projectedRun?.status === "canceled") {
        yield* repository.markSettled(
          claim.runId,
          "failed",
          projectedRun.completedAt ?? projectedRun.updatedAt,
        );
        continue;
      }

      // Scheduled bot work settles from its delegation. A delegation that
      // ended while nothing watched it, such as one startup reconciliation
      // failed after a restart, still settles its run here.
      if (claim.status === "dispatched" && claim.threadRef != null) {
        const delegation = yield* adapter.findDelegatedRunDelegation(claim.threadRef);

        if (delegation !== null && (yield* settleDelegatedRun(delegation))) continue;
      }

      if (
        claim.status === "dispatched" &&
        claim.terminalState !== null &&
        claim.terminalState !== undefined
      ) {
        const run = (yield* repository.listRuns(claim.routineId)).find(
          (candidate) => candidate.id === claim.runId,
        );

        if (run !== undefined) {
          const completedAt = claim.terminalAt ?? DateTime.formatIso(yield* DateTime.now);

          if (claim.terminalState === "completed") {
            const nextRunAt = routine.enabled
              ? nextScheduledFor(routine.schedule, routine.timezone, Date.parse(completedAt))
              : null;

            yield* adapter.recordCompleted(run, nextRunAt, "Routine completed.", completedAt);
            yield* repository.markSettled(run.id, "completed", completedAt);
          } else {
            const failure = {
              kind: "execution",
              reason: `Routine chat ended with state '${claim.terminalState}'.`,
              nextAction: "Review the routine chat, then resume the routine.",
            } satisfies RoutineDependencyFailure;

            yield* adapter.recordFailed(run, failure, completedAt);
            yield* adapter.openFailureIncident(routine, failure);
            yield* repository.markBlocked(run.id, failure.reason, completedAt);
          }

          continue;
        }
      }

      if (
        claim.status === "dispatched" &&
        claim.sessionState !== undefined &&
        claim.sessionState !== null &&
        claim.sessionState !== "ready" &&
        claim.sessionState !== "starting" &&
        claim.sessionState !== "running"
      ) {
        const completedAt = claim.sessionUpdatedAt ?? DateTime.formatIso(yield* DateTime.now);

        const run = (yield* repository.listRuns(claim.routineId)).find(
          (candidate) => candidate.id === claim.runId,
        );

        if (run !== undefined) {
          const failure = {
            kind: "execution",
            reason: `Routine session ended with state '${claim.sessionState}'.`,
            nextAction: "Review the routine chat, then resume the routine.",
          } satisfies RoutineDependencyFailure;

          yield* adapter.recordFailed(run, failure, completedAt);
          yield* adapter.openFailureIncident(routine, failure);
          yield* repository.markBlocked(run.id, failure.reason, completedAt);
        }

        continue;
      }

      if (claim.status === "dispatched") continue;
      yield* execute(routine, claim);
    }
  }).pipe(
    Effect.catchCause((cause) => Effect.logError("routine scheduler recovery failed", { cause })),
  );

  const start = Effect.gen(function* () {
    if (Option.isSome(engine)) {
      yield* Effect.forkScoped(
        engine.value.streamDomainEvents.pipe(
          Stream.runForEach((event) =>
            settleRunForEvent(event).pipe(
              Effect.catchCause((cause) =>
                Effect.logError("routine run settlement failed", { cause }),
              ),
            ),
          ),
        ),
      );
    }

    yield* recover;
    yield* runDue;
    yield* Effect.forkScoped(
      recover.pipe(Effect.andThen(runDue), Effect.repeat(Schedule.spaced("1 minute"))),
    );
  });

  return { runDue, canRunNow, runNow, recover, start } satisfies RoutineRuntimeShape;
});

export const RoutineRuntimeLive = Layer.effect(RoutineRuntime, make);