Skip to content

Open source alternative to Semrush and Ahrefs

13.6 KB
Raw Blame
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { aiObservations, projectCompetitors } from "@/db/schema";
import { DataforseoChargedTaskError } from "@/server/lib/dataforseo/envelope";
import { AppError } from "@/server/lib/errors";
import { AiVisibilityRepository as repo } from "../repositories/AiVisibilityRepository";
import {
  collectAiLiveBatch,
  collectAiModelBatch,
  collectAiRound,
  finalizeAiRun,
  postAiBatch,
  prepareAiRun,
} from "./aiVisibilityCollection";
import { configuration, customer } from "./aiVisibilityTestFixtures";

const { testDb, dataforseo } = await vi.hoisted(async () => {
  const { createAiVisibilityTestDb } = await import("../aiVisibilityTestDb");
  return {
    testDb: await createAiVisibilityTestDb(),
    dataforseo: {
      trackingTaskPost: vi.fn(),
      trackingLiveBatch: vi.fn(),
      fetchTaskResult: vi.fn(),
      llmResponse: vi.fn(),
    },
  };
});
vi.mock("cloudflare:workers", () => ({ env: { DATABASE_PROVIDER: "d1" } }));
vi.mock("cloudflare:workflows", () => ({
  NonRetryableError: class extends Error {},
}));
vi.mock("@/db", () => ({ db: testDb.db }));
vi.mock("@/db/runBatch", () => ({ runBatch: testDb.runBatch }));
vi.mock("@/server/lib/runtime-env", () => ({
  isHostedServerAuthMode: async () => false,
}));
vi.mock("@/server/lib/dataforseo", () => ({
  MAX_TASKS_PER_POST: 100,
  createDataforseoClient: () => ({
    aiSearch: {
      trackingTaskPost: dataforseo.trackingTaskPost,
      trackingLiveBatch: dataforseo.trackingLiveBatch,
      llmResponse: dataforseo.llmResponse,
    },
  }),
  fetchAiTrackingTaskResult: dataforseo.fetchTaskResult,
}));
vi.mock("@/server/lib/dataforseo/llm-models", () => ({
  resolveLatestLlmModelName: async () => "claude-sonnet-5",
}));

const runId = "run";
const market = { locationCode: 2840, languageCode: "en" };
const pendingTask = {
  tag: "answer-0",
  taskId: "task",
  engine: "chatgpt",
  prompt: "Which SEO tools?",
  attempts: 1,
} as const;
const liveAnswer = {
  status: "fulfilled",
  value: { status: "completed", result: { markdown: "OpenSEO", sources: [] } },
} as const;
const providerFailure = {
  status: "failed",
  message: "Internal Server Error.",
  code: "UPSTREAM_UNAVAILABLE",
  retryable: true,
} as const;

afterEach(() => {
  vi.useRealTimers();
});

beforeEach(async () => {
  await testDb.seedProject();
  await testDb.db.insert(projectCompetitors).values({
    id: "competitor",
    projectId: "project",
    domain: "ahrefs.com",
    name: "Ahrefs",
    updatedBy: "user",
  });
  const rows = configuration({
    engines: ["chatgpt"],
    prompts: [{ text: "Which SEO tools?" }, { text: "Best rank tracker?" }],
  });
  await repo.saveConfiguration(rows);
  await repo.createRun(
    {
      id: runId,
      trackerId: rows.tracker.id,
      projectId: "project",
      trigger: "manual",
      status: "running",
      ...market,
      createdAt: "2026-09-05T12:00:00.000Z",
    },
    rows.prompts.map((prompt, index) => ({
      id: `answer-${index}`,
      runId,
      promptId: prompt.id,
      engine: "chatgpt",
      branded: false,
    })),
  );
});

describe("AI answer collection", () => {
  it("saves an answer with its citations and each brand's mention and citation", async () => {
    dataforseo.fetchTaskResult.mockResolvedValue({
      status: "completed",
      result: {
        markdown:
          "Try **Ahrefs** or OpenSEO [1](https://openseo.so/pricing?utm_source=chatgpt)",
        sources: [
          {
            url: "https://openseo.so/pricing?utm_source=chatgpt",
            title: "Pricing",
          },
        ],
      },
    });

    const round = await collectAiRound(runId, [pendingTask], true);

    expect(round).toEqual({ pending: [], retry: [] });
    expect(await repo.getObservation("answer-0")).toMatchObject({
      status: "completed",
    });
    const evidence = await repo.getEvidence(["answer-0"]);
    expect(evidence.sources).toMatchObject([
      { url: "https://openseo.so/pricing", domain: "openseo.so", position: 1 },
    ]);
    expect(evidence.matches).toEqual(
      expect.arrayContaining([
        expect.objectContaining({
          domain: "openseo.so",
          own: true,
          mentioned: true,
          cited: true,
          firstMention: 14,
        }),
        expect.objectContaining({
          domain: "ahrefs.com",
          own: false,
          mentioned: true,
          cited: false,
          firstMention: 4,
        }),
      ]),
    );
    // A retried step skips the answer an earlier attempt already saved.
    await collectAiRound(runId, [pendingTask], true);
    expect(dataforseo.fetchTaskResult).toHaveBeenCalledTimes(1);
  });

  it("fails the answers DataForSEO did not accept and collects the rest", async () => {
    dataforseo.trackingTaskPost.mockResolvedValue([
      { tag: "answer-0", taskId: "task" },
    ]);

    const pending = await postAiBatch(runId, customer, market, {
      engine: "chatgpt",
      tasks: [
        { tag: "answer-0", prompt: "Which SEO tools?" },
        { tag: "answer-1", prompt: "Best rank tracker?" },
      ],
    });

    expect(pending).toEqual([pendingTask]);
    expect(await repo.getObservation("answer-1")).toMatchObject({
      status: "failed",
    });
  });

  it("counts the attempt of an answer posted again, so its retries stop at 3", async () => {
    dataforseo.trackingTaskPost.mockResolvedValue([
      { tag: "answer-0", taskId: "task" },
    ]);

    const pending = await postAiBatch(runId, customer, market, {
      engine: "chatgpt",
      tasks: [{ tag: "answer-0", prompt: "Which SEO tools?", attempts: 2 }],
    });

    expect(pending).toEqual([{ ...pendingTask, attempts: 3 }]);
  });

  it("returns a queued answer DataForSEO's backend failed for a new post, and fails it after the last attempt", async () => {
    dataforseo.fetchTaskResult.mockResolvedValue(providerFailure);

    const round = await collectAiRound(
      runId,
      [
        pendingTask,
        {
          ...pendingTask,
          tag: "answer-1",
          prompt: "Best rank tracker?",
          attempts: 3,
        },
      ],
      true,
    );

    expect(round.retry).toEqual([
      {
        engine: "chatgpt",
        tasks: [{ tag: "answer-0", prompt: "Which SEO tools?", attempts: 1 }],
      },
    ]);
    expect(await repo.getObservation("answer-1")).toMatchObject({
      status: "failed",
      error: "UPSTREAM_UNAVAILABLE",
    });
  });

  it("saves each model answer without waiting for slower calls", async () => {
    dataforseo.llmResponse.mockImplementation(
      ({ userPrompt }: { userPrompt: string }) =>
        userPrompt === "Best rank tracker?"
          ? new Promise(() => {})
          : Promise.resolve({ items: [] }),
    );

    void collectAiModelBatch(runId, customer, market, {
      engine: "claude",
      tasks: [
        { tag: "answer-0", prompt: "Which SEO tools?" },
        { tag: "answer-1", prompt: "Best rank tracker?" },
      ],
    });

    await vi.waitFor(async () =>
      expect(await repo.getObservation("answer-0")).toMatchObject({
        status: "completed",
      }),
    );
    expect(await repo.getObservation("answer-1")).toMatchObject({
      status: "pending",
    });
  });

  it("saves model answers with their cited pages and fails the calls that error", async () => {
    dataforseo.llmResponse.mockImplementation(
      async ({ userPrompt }: { userPrompt: string }) => {
        if (userPrompt === "Best rank tracker?") throw new Error("Timed out");
        return {
          items: [
            {
              type: "message",
              sections: [
                {
                  text: "Try OpenSEO.",
                  annotations: [
                    { url: "https://openseo.so/", title: "OpenSEO" },
                  ],
                },
              ],
            },
          ],
        };
      },
    );

    await collectAiModelBatch(runId, customer, market, {
      engine: "claude",
      tasks: [
        { tag: "answer-0", prompt: "Which SEO tools?" },
        { tag: "answer-1", prompt: "Best rank tracker?" },
      ],
    });

    expect(dataforseo.llmResponse).toHaveBeenCalledWith(
      expect.objectContaining({
        modelSlug: "claude",
        webSearch: true,
        webSearchCountryCode: "US",
      }),
    );
    expect(await repo.getObservation("answer-0")).toMatchObject({
      status: "completed",
      answerMarkdown: "Try OpenSEO.",
    });
    expect((await repo.getEvidence(["answer-0"])).sources).toMatchObject([
      { url: "https://openseo.so/", domain: "openseo.so" },
    ]);
    expect(await repo.getObservation("answer-1")).toMatchObject({
      status: "failed",
      error: "INTERNAL_ERROR",
    });
  });

  it("asks a model answer again only when DataForSEO returned the failed task", async () => {
    vi.useFakeTimers({ toFake: ["setTimeout"] });
    dataforseo.llmResponse.mockImplementation(
      async ({ userPrompt }: { userPrompt: string }) => {
        if (userPrompt === "Best rank tracker?")
          throw new AppError("UPSTREAM_UNAVAILABLE", "DataForSEO HTTP 502");
        throw new DataforseoChargedTaskError(
          "Internal Server Error.",
          { path: [], costUsd: 0 },
          false,
          "UPSTREAM_UNAVAILABLE",
        );
      },
    );

    const collecting = collectAiModelBatch(runId, customer, market, {
      engine: "claude",
      tasks: [
        { tag: "answer-0", prompt: "Which SEO tools?" },
        { tag: "answer-1", prompt: "Best rank tracker?" },
      ],
    });
    await vi.runAllTimersAsync();
    await collecting;

    expect(dataforseo.llmResponse).toHaveBeenCalledTimes(4);
    expect(await repo.getObservation("answer-0")).toMatchObject({
      status: "failed",
      error: "UPSTREAM_UNAVAILABLE",
    });
    expect(await repo.getObservation("answer-1")).toMatchObject({
      status: "failed",
      error: "UPSTREAM_UNAVAILABLE",
    });
  });

  it("sends Claude to the model API, not the live scraper, on a manual run", async () => {
    const [{ promptId }] = await repo.getObservations([runId]);
    await testDb.db.insert(aiObservations).values({
      id: "answer-claude",
      runId,
      promptId,
      engine: "claude",
      branded: false,
    });

    const plan = await prepareAiRun(runId);

    expect(plan.modelBatches).toMatchObject([
      { engine: "claude", tasks: [{ tag: "answer-claude" }] },
    ]);
    expect(plan.batches.flat()).not.toContainEqual(
      expect.objectContaining({ engine: "claude" }),
    );
  });

  it("collects a manual run live, saving answers and failing rejected calls", async () => {
    expect(await prepareAiRun(runId)).toMatchObject({ live: true });
    dataforseo.trackingLiveBatch.mockResolvedValue([
      liveAnswer,
      { status: "rejected", reason: new Error("Timed out") },
    ]);

    await collectAiLiveBatch(runId, customer, market, [
      { tag: "answer-0", prompt: "Which SEO tools?", engine: "chatgpt" },
      { tag: "answer-1", prompt: "Best rank tracker?", engine: "chatgpt" },
    ]);

    expect(await repo.getObservation("answer-0")).toMatchObject({
      status: "completed",
    });
    expect(await repo.getObservation("answer-1")).toMatchObject({
      status: "failed",
      error: "INTERNAL_ERROR",
    });
    expect(dataforseo.trackingLiveBatch).toHaveBeenCalledOnce();
  });

  it("asks a live answer again when DataForSEO's backend fails it, up to 3 attempts", async () => {
    vi.useFakeTimers({ toFake: ["setTimeout"] });
    const failed = { status: "fulfilled", value: providerFailure };
    dataforseo.trackingLiveBatch
      .mockResolvedValueOnce([failed, failed])
      .mockResolvedValueOnce([liveAnswer, failed])
      .mockResolvedValueOnce([failed]);

    const collecting = collectAiLiveBatch(runId, customer, market, [
      { tag: "answer-0", prompt: "Which SEO tools?", engine: "chatgpt" },
      { tag: "answer-1", prompt: "Best rank tracker?", engine: "chatgpt" },
    ]);
    await vi.runAllTimersAsync();
    await collecting;

    expect(dataforseo.trackingLiveBatch).toHaveBeenCalledTimes(3);
    expect(await repo.getObservation("answer-0")).toMatchObject({
      status: "completed",
    });
    expect(await repo.getObservation("answer-1")).toMatchObject({
      status: "failed",
      error: "UPSTREAM_UNAVAILABLE",
    });
  });

  it("saves the rest of a paid live batch when one answer cannot be saved", async () => {
    dataforseo.trackingLiveBatch.mockResolvedValue([liveAnswer, liveAnswer]);
    vi.spyOn(repo, "persistAnswer").mockRejectedValueOnce(new Error("D1"));
    vi.spyOn(console, "error").mockImplementation(() => {});

    await collectAiLiveBatch(runId, customer, market, [
      { tag: "answer-0", prompt: "Which SEO tools?", engine: "chatgpt" },
      { tag: "answer-1", prompt: "Best rank tracker?", engine: "chatgpt" },
    ]);

    expect(await repo.getObservation("answer-0")).toMatchObject({
      status: "pending",
    });
    expect(await repo.getObservation("answer-1")).toMatchObject({
      status: "completed",
    });
  });

  it("fails answers that never arrived and finishes the run as partial", async () => {
    dataforseo.fetchTaskResult.mockResolvedValue(liveAnswer.value);
    await collectAiRound(runId, [pendingTask], true);

    await finalizeAiRun(runId);

    expect(await repo.getRunInternal(runId)).toMatchObject({
      status: "partial",
    });
    expect(await repo.getObservation("answer-1")).toMatchObject({
      status: "failed",
      error: "No answer arrived within the collection window.",
    });
  });

  it("tells the tracker the check could not collect answers when none arrived", async () => {
    await finalizeAiRun(runId);

    expect(await repo.getRunInternal(runId)).toMatchObject({
      status: "failed",
    });
    expect((await repo.getTracker("project"))?.lastSkipReason).toBe(
      "The last check could not collect answers.",
    );
  });
});