Skip to content

Open-source Grok Bot alternative for named AI teammates. Local-first desktop, web, and mobile clients with per-bot tools, instructions, and memory. Independent fork of T3 Code.

22.6 KB
Raw Blame
import * as Schema from "effect/Schema";
import { isAccountLimitMessage, limitRetryAt } from "./accountLimits.ts";
import { decodeProviderHealth } from "./persistedSchemas.ts";
import * as NodeFS from "node:fs";
import type * as Path from "effect/Path";
import * as NodeCrypto from "node:crypto";
import * as Effect from "effect/Effect";
import type * as Clock from "effect/Clock";
import * as DateTime from "effect/DateTime";
import { FetchHttpClient, HttpClient, HttpClientError } from "effect/unstable/http";
import { runOAuthPromise } from "./oauthHttp.ts";
import { type SubscriptionAuthData, type SubscriptionCredentialStore } from "./credentialStore.ts";
import type { ApiKeyCredential, OAuthCredential } from "./types.ts";
import {
  anthropicApiBaseUrl,
  OPENCODE_GO_USAGE_URL,
  OPENCODE_GO_USER_AGENT,
  type SubscriptionProviderId,
  type LoginPollStatus,
  HEALTH_CHECK_TIMEOUT_MS,
  oauthHealthRequest,
  type RequestHealthStatus,
  type ImageRequestHealthStatus,
  type ProviderHealthData,
  oauthFailureKind,
  credentialKey,
  credentialAt,
  defaultInstanceByProvider,
  refreshedCredential,
  runRefresh,
} from "./serviceTypes.ts";

/** Changes on every atomic rewrite: each write renames a fresh file into place. */
export function fileVersion(path: string): string | undefined {
  try {
    const stat = NodeFS.statSync(path);

    return `${stat.ino}:${stat.mtimeMs}:${stat.size}`;
  } catch {
    return undefined;
  }
}

const decodeAccountOrder = Schema.decodeUnknownSync(Schema.Json);

export class SubscriptionHealthService {
  private readonly clock: Clock.Clock;
  private readonly reload: () => Effect.Effect<void>;
  private readonly path: Path.Path;
  private readonly store: SubscriptionCredentialStore;
  constructor(
    store: SubscriptionCredentialStore,
    clock: Clock.Clock,
    path: Path.Path,
    checkHealthOnConnect: boolean,
    reload: () => Effect.Effect<void>,
  ) {
    this.clock = clock;
    this.reload = reload;
    this.path = path;
    this.store = store;
    this.healthPath = store.path + ".health";
    this.checkHealthOnConnect = checkHealthOnConnect;
  }
  private get data() {
    return this.store.current().data;
  }
  private reloadAsync() {
    return Effect.runPromise(this.reload());
  }
  private updateCredentials(update: (data: SubscriptionAuthData) => SubscriptionAuthData) {
    return Effect.runPromise(this.store.update(update));
  }

  public readonly healthPath: string;

  public health: ProviderHealthData = {};
  public healthVersion: string | undefined;

  public readonly healthChecks = new Map<string, Promise<void>>();

  public readonly healthProbeVersions = new Map<string, number>();

  public readonly checkHealthOnConnect: boolean;

  public readAccountOrder(filePath: string) {
    return NodeFS.existsSync(filePath)
      ? decodeAccountOrder(JSON.parse(NodeFS.readFileSync(filePath, "utf-8")))
      : {};
  }

  public reloadHealth(): void {
    this.healthVersion = fileVersion(this.healthPath);

    if (!NodeFS.existsSync(this.healthPath)) {
      this.health = {};

      return;
    }

    try {
      this.health = decodeProviderHealth(NodeFS.readFileSync(this.healthPath, "utf-8"));
    } catch {
      this.health = {};
    }
  }

  public writeSecureJson<Value>(filePath: string, value: Value): void {
    const dir = this.path.dirname(filePath);

    if (!NodeFS.existsSync(dir)) {
      NodeFS.mkdirSync(dir, { recursive: true, mode: 0o700 });
    }

    const tempPath = `${filePath}.${NodeCrypto.randomUUID()}.tmp`;
    NodeFS.writeFileSync(tempPath, JSON.stringify(value, null, 2), {
      encoding: "utf-8",
      mode: 0o600,
    });
    NodeFS.renameSync(tempPath, filePath);
    NodeFS.chmodSync(filePath, 0o600);
  }

  public saveHealth(): void {
    this.writeSecureJson(this.healthPath, this.health);
    this.healthVersion = fileVersion(this.healthPath);
  }

  recordRequestSuccess(
    provider: SubscriptionProviderId,
    at = DateTime.formatIso(DateTime.makeUnsafe(this.clock.currentTimeMillisUnsafe())),
  ): void {
    this.recordHealthSuccess(provider, at);
  }

  recordAccountRequestSuccess(
    provider: SubscriptionProviderId,
    instanceId: string,
    at: string,
  ): void {
    this.recordHealthSuccess(credentialKey(provider, instanceId), at);
  }

  recordProviderInstanceSuccess(
    instanceId: string,
    at = DateTime.formatIso(DateTime.makeUnsafe(this.clock.currentTimeMillisUnsafe())),
  ): void {
    this.recordHealthSuccess(`provider:${instanceId}`, at);
  }

  public recordAccountFailure(
    key: string,
    message: string,
    at: string,
    failureKind: "request" | "revoked",
  ): void {
    this.reloadHealth();
    const previous = this.health[key];

    const revoked =
      previous?.failureKind === "revoked" &&
      previous.lastFailedRequest !== undefined &&
      (previous.lastSuccessfulRequestAt === undefined ||
        previous.lastFailedRequest.at >= previous.lastSuccessfulRequestAt);

    this.recordHealthFailure(key, message, at, revoked ? "revoked" : failureKind);

    if (failureKind !== "request" || !isAccountLimitMessage(message)) return;
    this.health[key] = { ...this.health[key], nextRetryAt: limitRetryAt(message, at) };
    this.saveHealth();
  }

  public recordHealthSuccess(key: string, at: string): void {
    this.reloadHealth();
    const previous = this.health[key];

    const { nextRetryAt, lastCredentialProbeFailure: _probeFailure, ...rest } = previous ?? {};

    // A limit cooldown runs out on its own clock. A success from a request that
    // was already in flight, or a passing check, does not end it early.
    const coolingDown =
      nextRetryAt !== undefined && Date.parse(nextRetryAt) > this.clock.currentTimeMillisUnsafe();

    this.health[key] = {
      ...rest,
      ...(coolingDown ? { nextRetryAt } : {}),
      lastSuccessfulRequestAt: at,
      healthTest: { status: "passed", checkedAt: at },
    };
    this.saveHealth();
  }

  recordRequestFailure(
    provider: SubscriptionProviderId,
    message: string,
    at = DateTime.formatIso(DateTime.makeUnsafe(this.clock.currentTimeMillisUnsafe())),
    failureKind: "request" | "revoked" = "request",
  ): void {
    this.recordHealthFailure(provider, message, at, failureKind);
  }

  recordAccountRequestFailure(
    provider: SubscriptionProviderId,
    instanceId: string,
    message: string,
    at: string,
  ): void {
    this.recordHealthFailure(credentialKey(provider, instanceId), message, at, "request");
  }

  recordProviderInstanceFailure(
    instanceId: string,
    message: string,
    at = DateTime.formatIso(DateTime.makeUnsafe(this.clock.currentTimeMillisUnsafe())),
    model?: string,
  ): void {
    this.recordHealthFailure(`provider:${instanceId}`, message, at, "request", model);
  }

  recordMcpRequestSuccess(
    serverId: string,
    at = DateTime.formatIso(DateTime.makeUnsafe(this.clock.currentTimeMillisUnsafe())),
  ): void {
    this.recordHealthSuccess(`mcp:${serverId}`, at);
  }

  recordMcpRequestFailure(
    serverId: string,
    message: string,
    at = DateTime.formatIso(DateTime.makeUnsafe(this.clock.currentTimeMillisUnsafe())),
  ): void {
    this.recordHealthFailure(`mcp:${serverId}`, message, at, "request");
  }

  recordImageRequestSuccess(
    provider: "chatgpt" | "grok",
    at = DateTime.formatIso(DateTime.makeUnsafe(this.clock.currentTimeMillisUnsafe())),
  ): void {
    this.recordHealthSuccess(`image:${provider}`, at);
  }

  recordImageCredentialProbeSuccess(
    provider: "chatgpt" | "grok",
    at = DateTime.formatIso(DateTime.makeUnsafe(this.clock.currentTimeMillisUnsafe())),
  ): void {
    this.reloadHealth();
    const key = `image:${provider}`;
    const { lastCredentialProbeFailure: _failure, ...previous } = this.health[key] ?? {};
    this.health[key] = {
      ...previous,
      lastCredentialProbeAt: at,
      healthTest: { status: "passed", checkedAt: at },
    };
    this.saveHealth();
  }

  recordImageCredentialProbeFailure(
    provider: "chatgpt" | "grok",
    message: string,
    at = DateTime.formatIso(DateTime.makeUnsafe(this.clock.currentTimeMillisUnsafe())),
    failureKind: "request" | "revoked" = "request",
  ): void {
    message = this.redactHealthMessage(message);
    const key = `image:${provider}`;
    this.health[key] = {
      ...this.health[key],
      lastCredentialProbeFailure: {
        at,
        message,
        failureKind,
      },
      healthTest: { status: "failed", checkedAt: at },
    };
    this.saveHealth();
  }

  recordImageRequestFailure(
    provider: "chatgpt" | "grok",
    message: string,
    at = DateTime.formatIso(DateTime.makeUnsafe(this.clock.currentTimeMillisUnsafe())),
    failureKind: "request" | "revoked" = "request",
  ): void {
    this.recordHealthFailure(`image:${provider}`, message, at, failureKind);
  }

  /** A completed image generation, which also proves the provider healthy. */
  recordImageGenerationSuccess(
    provider: "chatgpt" | "grok",
    at = DateTime.formatIso(DateTime.makeUnsafe(this.clock.currentTimeMillisUnsafe())),
  ): void {
    this.recordHealthSuccess(`image:${provider}`, at);
    this.health[`image:${provider}`] = {
      ...this.health[`image:${provider}`],
      lastGenerationAt: at,
    };
    this.saveHealth();
  }

  imageLastGenerationAt(provider: "chatgpt" | "grok"): string | undefined {
    return this.health[`image:${provider}`]?.lastGenerationAt;
  }

  /** Image-provider request health, keyed separately from the chat driver. */
  imageRequestHealth(provider: "chatgpt" | "grok"): ImageRequestHealthStatus | undefined {
    const key = `image:${provider}`;
    const requestHealth = this.requestHealth(key);
    const record = this.health[key];

    if (!record) return requestHealth;

    return {
      ...(requestHealth ?? { health: "detected" }),
      ...(record.lastCredentialProbeAt
        ? { lastCredentialProbeAt: record.lastCredentialProbeAt }
        : {}),
      ...(record.lastCredentialProbeFailure
        ? { lastCredentialProbeFailure: record.lastCredentialProbeFailure }
        : {}),
      ...(record.healthTest ? { healthTest: record.healthTest } : {}),
    };
  }

  public redactHealthMessage(message: string): string {
    this.reloadHealth();

    for (const credentialId of Object.keys(this.data)) {
      const credential = credentialAt(this.data, credentialId);

      if (!credential) continue;

      for (const secret of [
        credential.access,
        credential.type === "oauth" ? credential.refresh : undefined,
      ]) {
        if (secret) message = message.replaceAll(secret, "[redacted]");
      }
    }

    return message;
  }

  public recordHealthFailure(
    key: string,
    message: string,
    at: string,
    failureKind: "request" | "revoked",
    model?: string,
  ): void {
    message = this.redactHealthMessage(message);
    const { nextRetryAt: _nextRetryAt, ...previous } = this.health[key] ?? {};
    this.health[key] = {
      ...previous,
      lastFailedRequest: { at, message, ...(model ? { model } : {}) },
      failureKind,
      healthTest: { status: "failed", checkedAt: at },
    };
    this.saveHealth();
  }

  providerInstanceHealth(
    instanceId: string,
  ): "healthy" | "failed" | "failed-first-request" | "recovered" | undefined {
    return this.requestHealth(`provider:${instanceId}`)?.health;
  }

  providerInstanceRequestHealth(instanceId: string): RequestHealthStatus | undefined {
    return this.requestHealth(`provider:${instanceId}`);
  }

  mcpRequestHealth(serverId: string): RequestHealthStatus | undefined {
    return this.requestHealth(`mcp:${serverId}`);
  }

  public requestHealth(key: string): RequestHealthStatus | undefined {
    const health = this.health[key];

    if (!health) return undefined;

    if (
      health.lastFailedRequest &&
      (!health.lastSuccessfulRequestAt ||
        health.lastFailedRequest.at > health.lastSuccessfulRequestAt ||
        (health.lastFailedRequest.at === health.lastSuccessfulRequestAt &&
          health.healthTest?.status === "failed"))
    ) {
      return {
        health: health.lastSuccessfulRequestAt ? "failed" : "failed-first-request",
        ...(health.lastSuccessfulRequestAt
          ? { lastSuccessfulRequestAt: health.lastSuccessfulRequestAt }
          : {}),
        lastFailedRequest: health.lastFailedRequest,
        ...(health.nextRetryAt ? { nextRetryAt: health.nextRetryAt } : {}),
      };
    }

    if (
      health.lastSuccessfulRequestAt &&
      health.lastFailedRequest &&
      (health.lastSuccessfulRequestAt > health.lastFailedRequest.at ||
        (health.lastSuccessfulRequestAt === health.lastFailedRequest.at &&
          health.healthTest?.status === "passed"))
    ) {
      return {
        health: "recovered",
        lastSuccessfulRequestAt: health.lastSuccessfulRequestAt,
        lastFailedRequest: health.lastFailedRequest,
        ...(health.nextRetryAt ? { nextRetryAt: health.nextRetryAt } : {}),
      };
    }

    return health.lastSuccessfulRequestAt
      ? {
          health: "healthy",
          lastSuccessfulRequestAt: health.lastSuccessfulRequestAt,
          ...(health.lastFailedRequest ? { lastFailedRequest: health.lastFailedRequest } : {}),
          ...(health.nextRetryAt ? { nextRetryAt: health.nextRetryAt } : {}),
        }
      : undefined;
  }

  public async isCurrentHealthCredential(
    key: string,
    credential: ApiKeyCredential | OAuthCredential,
    version: number,
  ): Promise<boolean> {
    if (this.healthProbeVersions.get(key) !== version) return false;
    await this.reloadAsync();

    if (this.healthProbeVersions.get(key) !== version) return false;
    const current = credentialAt(this.data, key);

    if (current?.type !== credential.type || current.access !== credential.access) return false;

    return credential.type === "api-key"
      ? current.type === "api-key" && current.baseUrl === credential.baseUrl
      : current.type === "oauth" && current.refresh === credential.refresh;
  }

  async testHealth(provider: SubscriptionProviderId, instanceId?: string): Promise<void> {
    await this.reloadAsync();
    const key = credentialKey(provider, instanceId);
    const version = (this.healthProbeVersions.get(key) ?? 0) + 1;
    this.healthProbeVersions.set(key, version);
    const credential = credentialAt(this.data, key);

    if (!credential) {
      this.recordOAuthFailure(
        provider,
        this.store.current().loadError?.message ?? "No account is connected.",
        "request",
        instanceId,
      );

      return;
    }

    if (credential.type === "api-key") {
      const defaultBaseUrls: Partial<Record<SubscriptionProviderId, string>> = {
        anthropic: "https://api.anthropic.com/v1",
        "openai-codex": "https://api.openai.com/v1",
        xai: "https://api.x.ai/v1",
        "kimi-for-coding": "https://api.kimi.com/coding/v1",
        "opencode-go": "https://opencode.ai/zen/go/v1",
      };

      const url =
        provider === "opencode-go" && !credential.baseUrl
          ? OPENCODE_GO_USAGE_URL
          : provider === "anthropic"
            ? `${anthropicApiBaseUrl(credential.baseUrl)}/v1/models`
            : `${credential.baseUrl ?? defaultBaseUrls[provider]}/models`;

      try {
        const response = await probeHealth(url, {
          ...(provider === "anthropic"
            ? { "x-api-key": credential.access, "anthropic-version": "2023-06-01" }
            : { Authorization: `Bearer ${credential.access}` }),
          "User-Agent": OPENCODE_GO_USER_AGENT,
          ...(provider === "opencode-go" ? { "x-opencode-client": "akeru-bot" } : {}),
        });

        if (!(await this.isCurrentHealthCredential(key, credential, version))) return;

        if (!response.ok) {
          this.recordHealthFailure(
            key,
            `The provider rejected the API-key check (${response.status}).`,
            DateTime.formatIso(DateTime.makeUnsafe(this.clock.currentTimeMillisUnsafe())),
            response.status === 401 || response.status === 403 ? "revoked" : "request",
          );
        } else {
          this.recordHealthSuccess(
            key,
            DateTime.formatIso(DateTime.makeUnsafe(this.clock.currentTimeMillisUnsafe())),
          );
        }
      } catch {
        if (!(await this.isCurrentHealthCredential(key, credential, version))) return;
        this.recordHealthFailure(
          key,
          "The API-key check failed. Check the base URL and connection.",
          DateTime.formatIso(DateTime.makeUnsafe(this.clock.currentTimeMillisUnsafe())),
          "request",
        );
      }

      return;
    }

    let testedCredential = credential;

    try {
      const refreshed =
        credential.expires > this.clock.currentTimeMillisUnsafe()
          ? credential
          : await runRefresh(provider, credential);

      if (!(await this.isCurrentHealthCredential(key, credential, version))) return;

      if (refreshed !== credential) {
        // Save under the store lock only while the tested credential is still stored,
        // so a logout or replacement that lands meanwhile is never undone.
        const saved = await this.updateCredentials((data) => {
          const latest = credentialAt(data, key);

          return latest?.type === "oauth" &&
            latest.access === credential.access &&
            latest.refresh === credential.refresh
            ? { ...data, [key]: refreshedCredential(latest, refreshed) }
            : data;
        });

        const stored = credentialAt(saved, key);

        if (stored?.type !== "oauth" || stored.access !== refreshed.access) return;
      }

      testedCredential = { type: "oauth", ...refreshed };
      const request = oauthHealthRequest(provider, refreshed);

      if (!request) throw new Error("This subscription does not expose a health endpoint.");

      const response = await probeHealth(request.url, request.headers);

      if (!(await this.isCurrentHealthCredential(key, testedCredential, version))) return;

      if (!response.ok) {
        throw new Error(`The provider rejected the health request (${response.status}).`);
      }

      this.recordHealthSuccess(
        key,
        DateTime.formatIso(DateTime.makeUnsafe(this.clock.currentTimeMillisUnsafe())),
      );

      const checkedAt = DateTime.formatIso(
        DateTime.makeUnsafe(this.clock.currentTimeMillisUnsafe()),
      );

      this.reloadHealth();
      this.health[key] = {
        ...this.health[key],
        oauthCheck: { status: "passed", checkedAt },
      };
      this.saveHealth();
    } catch (cause) {
      if (!(await this.isCurrentHealthCredential(key, testedCredential, version))) return;
      this.recordHealthFailure(
        key,
        cause instanceof Error ? cause.message : "The provider rejected the health request.",
        DateTime.formatIso(DateTime.makeUnsafe(this.clock.currentTimeMillisUnsafe())),
        oauthFailureKind(cause),
      );
    }
  }

  /**
   * Start the post-login health check in the background. It keeps running when the
   * client that finished the login disconnects; `awaitHealthCheck` observes it.
   */
  public startHealthCheck(provider: SubscriptionProviderId, instanceId?: string): LoginPollStatus {
    if (!this.checkHealthOnConnect) return { status: "connected" };
    const key = credentialKey(provider, instanceId);
    this.reloadHealth();
    this.health[key] = {
      ...this.health[key],
      healthCheckStartedAt: DateTime.formatIso(
        DateTime.makeUnsafe(this.clock.currentTimeMillisUnsafe()),
      ),
    };
    this.saveHealth();

    const check = this.testHealth(provider, instanceId)
      .finally(() => {
        if (this.healthChecks.get(key) !== check) return;
        this.reloadHealth();
        const current = this.health[key];

        if (current?.healthCheckStartedAt === undefined) return;
        const { healthCheckStartedAt: _startedAt, ...rest } = current;
        this.health[key] = rest;
        this.saveHealth();
      })
      .catch(() => undefined)
      .finally(() => {
        if (this.healthChecks.get(key) === check) this.healthChecks.delete(key);
      });

    this.healthChecks.set(key, check);

    return { status: "connected", health: "checking" };
  }

  /** Resolves when the post-login health check for `provider` has recorded its result. */
  awaitHealthCheck(provider: SubscriptionProviderId, instanceId?: string): Promise<void> {
    return this.healthChecks.get(credentialKey(provider, instanceId)) ?? Promise.resolve();
  }

  public recordOAuthFailure(
    provider: SubscriptionProviderId,
    message: string,
    failureKind: "request" | "revoked" = "request",
    instanceId?: string,
  ): void {
    this.recordOAuthFailureForKey(credentialKey(provider, instanceId), message, failureKind);
  }

  public recordOAuthFailureForKey(
    key: string,
    message: string,
    failureKind: "request" | "revoked",
  ): void {
    const checkedAt = DateTime.formatIso(DateTime.makeUnsafe(this.clock.currentTimeMillisUnsafe()));
    this.reloadHealth();
    const { nextRetryAt: _nextRetryAt, ...previous } = this.health[key] ?? {};
    this.health[key] = {
      ...previous,
      lastFailedRequest: { at: checkedAt, message },
      failureKind,
      oauthCheck: { status: "failed", checkedAt },
    };
    this.saveHealth();
  }

  /**
   * Drops a usage limit recorded only on the provider's default instance once
   * one of its backup accounts is removed or signs in again, so the provider
   * is not left blocked by a limit no remaining account carries.
   */
  public clearProviderLimit(provider: SubscriptionProviderId, key: string): void {
    if (key.startsWith("instance:")) return;
    const providerKey = `provider:${defaultInstanceByProvider[provider]}`;
    const failure = this.health[providerKey]?.lastFailedRequest;

    if (failure !== undefined && isAccountLimitMessage(failure.message))
      delete this.health[providerKey];
  }

  public clearImageHealth(provider: SubscriptionProviderId, instanceId?: string): void {
    if (credentialKey(provider, instanceId) !== provider) return;

    if (provider === "openai-codex") delete this.health["image:chatgpt"];

    if (provider === "xai") delete this.health["image:grok"];
  }
}

async function probeHealth(url: string, headers: Record<string, string>) {
  const signal = AbortSignal.timeout(HEALTH_CHECK_TIMEOUT_MS);

  try {
    return await runOAuthPromise(
      HttpClient.HttpClient.use((client) =>
        HttpClient.withScope(client).get(url, { headers }),
      ).pipe(
        Effect.map((response) => ({
          status: response.status,
          ok: response.status >= 200 && response.status < 300,
        })),
        Effect.scoped,
        Effect.provideService(FetchHttpClient.RequestInit, { redirect: "error" }),
        Effect.provideService(HttpClient.TracerPropagationEnabled, false),
      ),
      signal,
    );
  } catch (cause) {
    if (signal.aborted) throw signal.reason;

    if (HttpClientError.isHttpClientError(cause) && "cause" in cause.reason) {
      throw cause.reason.cause;
    }

    throw cause;
  }
}