Skip to content

Open source alternative to Semrush and Ahrefs

Run onboarding prompts once and gray out prompts without results (EVE-397) (#867)

Carter Smith95f775996fea
diff --git a/plugins/openseo/.claude-plugin/plugin.json b/plugins/openseo/.claude-plugin/plugin.json
index e77212b7..f7feeaef 100644
--- a/plugins/openseo/.claude-plugin/plugin.json
+++ b/plugins/openseo/.claude-plugin/plugin.json
@@ -1,7 +1,7 @@
{
"name": "openseo",
"displayName": "OpenSEO",
- "version": "1.0.20",
+ "version": "1.0.21",
"description": "Research keywords and competitors, audit sites, track rankings and AI visibility, and use connected Search Console data from your AI assistant.",
"author": {
"name": "OpenSEO",
diff --git a/plugins/openseo/.codex-plugin/plugin.json b/plugins/openseo/.codex-plugin/plugin.json
index 040c7af9..0b12facc 100644
--- a/plugins/openseo/.codex-plugin/plugin.json
+++ b/plugins/openseo/.codex-plugin/plugin.json
@@ -1,6 +1,6 @@
{
"name": "openseo",
- "version": "1.0.20",
+ "version": "1.0.21",
"description": "Research keywords and competitors, audit sites, track rankings and AI visibility, and use connected Search Console data from your AI assistant.",
"author": {
"name": "OpenSEO",
diff --git a/plugins/openseo/.cursor-plugin/plugin.json b/plugins/openseo/.cursor-plugin/plugin.json
index b8b6ba44..96162ba3 100644
--- a/plugins/openseo/.cursor-plugin/plugin.json
+++ b/plugins/openseo/.cursor-plugin/plugin.json
@@ -1,6 +1,6 @@
{
"name": "openseo",
- "version": "1.0.20",
+ "version": "1.0.21",
"description": "Research keywords and competitors, audit sites, track rankings and AI visibility, and use connected Search Console data from your AI assistant.",
"author": {
"name": "Every App, Inc",
diff --git a/src/client/features/ai-visibility/AiResearchSetupGate.tsx b/src/client/features/ai-visibility/AiResearchSetupGate.tsx
index 3e776b46..d19f3035 100644
--- a/src/client/features/ai-visibility/AiResearchSetupGate.tsx
+++ b/src/client/features/ai-visibility/AiResearchSetupGate.tsx
@@ -139,7 +139,8 @@ export function AiResearchSetupGate({
{failed ? "Try again" : "Start research"}
</Button>
<p className="text-xs leading-relaxed text-muted-foreground">
- Research uses a small amount of usage credits.
+ Research and a first run of your prompts use a small amount of
+ usage credits.
</p>
</div>
</CardContent>
diff --git a/src/client/features/ai-visibility/CitationsTab.tsx b/src/client/features/ai-visibility/CitationsTab.tsx
index 0d827445..d4d7dd02 100644
--- a/src/client/features/ai-visibility/CitationsTab.tsx
+++ b/src/client/features/ai-visibility/CitationsTab.tsx
@@ -37,10 +37,12 @@ export function CitationsTab({
projectId,
state,
runId,
+ collecting,
}: {
projectId: string;
state: AiTrackerState;
runId: string | undefined;
+ collecting: boolean;
}) {
const [groupBy, setGroupBy] = useState<"url" | "domain">("url");
const [ownership, setOwnership] = useState<
@@ -150,7 +152,9 @@ export function CitationsTab({
/>
) : !query.data.rows.length ? (
<p className="p-10 text-center text-sm text-muted-foreground">
- No citations match these filters.
+ {collecting
+ ? "Collecting answers. Cited pages appear here as answers arrive."
+ : "No citations match these filters."}
</p>
) : (
<SourcesTable
diff --git a/src/client/features/ai-visibility/CompetitorsTab.tsx b/src/client/features/ai-visibility/CompetitorsTab.tsx
index f4ab15b7..24120087 100644
--- a/src/client/features/ai-visibility/CompetitorsTab.tsx
+++ b/src/client/features/ai-visibility/CompetitorsTab.tsx
@@ -24,10 +24,12 @@ export function CompetitorsTab({
projectId,
state,
runId,
+ collecting,
}: {
projectId: string;
state: AiTrackerState;
runId: string | undefined;
+ collecting: boolean;
}) {
const [selectedEngines, setSelectedEngines] = useState<AiEngine[] | null>(
null,
@@ -81,7 +83,9 @@ export function CompetitorsTab({
<AiQueryError error={query.error} />
) : runId && !rows.length ? (
<p className="p-10 text-center text-sm text-muted-foreground">
- No answers match these filters.
+ {collecting
+ ? "Collecting answers. Brands appear here as answers arrive."
+ : "No answers match these filters."}
</p>
) : (
<Table>
diff --git a/src/client/features/ai-visibility/PromptAnalysis.tsx b/src/client/features/ai-visibility/PromptAnalysis.tsx
index 1dae5a4f..4eca30b4 100644
--- a/src/client/features/ai-visibility/PromptAnalysis.tsx
+++ b/src/client/features/ai-visibility/PromptAnalysis.tsx
@@ -1,9 +1,21 @@
import { useState } from "react";
import { sort } from "remeda";
import { Button } from "@/client/components/ui/button";
-import type { AiPrompt } from "@/shared/ai-visibility";
+import {
+ Select,
+ SelectContent,
+ SelectItem,
+ SelectTrigger,
+ SelectValue,
+} from "@/client/components/ui/select";
+import {
+ aiObservationStatusLabel,
+ type AiPrompt,
+} from "@/shared/ai-visibility";
import { summarizeAiBrands } from "@/shared/ai-visibility-results";
import type { getAiVisibilityResults } from "@/serverFunctions/ai-visibility";
+import { aiEngineSchema } from "@/types/schemas/ai-visibility";
+import { EngineLabel } from "./EngineLabel";
import { PromptAnswers } from "./PromptAnswers";
import {
PromptDateRange,
@@ -11,8 +23,12 @@ import {
inPromptPeriod,
promptPeriod,
} from "./PromptDateRange";
-import { PromptExecutions } from "./PromptExecutions";
+import { PromptExecutions, type PromptExecution } from "./PromptExecutions";
import { PromptSummary } from "./PromptSummary";
+import { aiDate } from "./shared";
+
+const answered = ({ observation }: PromptExecution) =>
+ observation.status === "completed" && observation.answerStatus === "answered";
export function PromptAnalysis({
projectId,
@@ -43,11 +59,7 @@ export function PromptAnalysis({
);
const selected =
executions.find(({ observation }) => observation.id === selectedId) ??
- executions.find(
- ({ observation }) =>
- observation.status === "completed" &&
- observation.answerStatus === "answered",
- ) ??
+ executions.find(answered) ??
executions[0];
const summaries = summarizeAiBrands(
executions.map(({ observation }) => observation),
@@ -58,19 +70,23 @@ export function PromptAnalysis({
<PromptSummary summaries={summaries} />
{selected ? (
<>
- <PromptExecutions
- executions={executions}
- selectedId={selected.observation.id}
- onSelect={setSelectedId}
- />
<PromptAnswers
projectId={projectId}
prompt={prompt}
observation={selected.observation}
- executionDate={
- selected.observation.collectedAt ?? selected.run.createdAt
+ controls={
+ <ExecutionSelects
+ executions={executions}
+ selected={selected}
+ onSelect={setSelectedId}
+ />
}
/>
+ <PromptExecutions
+ executions={executions}
+ selectedId={selected.observation.id}
+ onSelect={setSelectedId}
+ />
</>
) : (
<div className="space-y-3 rounded-lg border border-border bg-card p-8 text-center">
@@ -104,3 +120,79 @@ export function PromptAnalysis({
</div>
);
}
+
+/** Pick an answer by AI provider, then by when it was collected. */
+function ExecutionSelects({
+ executions,
+ selected,
+ onSelect,
+}: {
+ executions: PromptExecution[];
+ selected: PromptExecution;
+ onSelect: (id: string) => void;
+}) {
+ const engine = selected.observation.engine;
+ const engineItems = aiEngineSchema.options
+ .filter((value) =>
+ executions.some(({ observation }) => observation.engine === value),
+ )
+ .map((value) => ({ value, label: <EngineLabel engine={value} /> }));
+ const dateItems = executions
+ .filter(({ observation }) => observation.engine === engine)
+ .map(({ observation, run }) => ({
+ value: observation.id,
+ label: `${aiDate(observation.collectedAt ?? run.createdAt)}${
+ observation.status === "completed"
+ ? ""
+ : ` · ${aiObservationStatusLabel(observation.status)}`
+ }`,
+ }));
+ return (
+ <div className="flex flex-wrap items-center gap-2">
+ <Select
+ items={engineItems}
+ value={engine}
+ onValueChange={(next) => {
+ // Keep the same run when that provider answered it.
+ const forEngine = executions.filter(
+ ({ observation }) => observation.engine === next,
+ );
+ const match =
+ forEngine.find(({ run }) => run.id === selected.run.id) ??
+ forEngine.find(answered) ??
+ forEngine[0];
+ if (match) onSelect(match.observation.id);
+ }}
+ >
+ <SelectTrigger size="sm" aria-label="AI provider">
+ <SelectValue />
+ </SelectTrigger>
+ <SelectContent>
+ {engineItems.map((item) => (
+ <SelectItem key={item.value} value={item.value}>
+ {item.label}
+ </SelectItem>
+ ))}
+ </SelectContent>
+ </Select>
+ <Select
+ items={dateItems}
+ value={selected.observation.id}
+ onValueChange={(id) => {
+ if (id) onSelect(id);
+ }}
+ >
+ <SelectTrigger size="sm" aria-label="Answer date">
+ <SelectValue />
+ </SelectTrigger>
+ <SelectContent>
+ {dateItems.map((item) => (
+ <SelectItem key={item.value} value={item.value}>
+ {item.label}
+ </SelectItem>
+ ))}
+ </SelectContent>
+ </Select>
+ </div>
+ );
+}
diff --git a/src/client/features/ai-visibility/PromptAnswers.tsx b/src/client/features/ai-visibility/PromptAnswers.tsx
index 37f352f3..8aa66021 100644
--- a/src/client/features/ai-visibility/PromptAnswers.tsx
+++ b/src/client/features/ai-visibility/PromptAnswers.tsx
@@ -1,3 +1,4 @@
+import type { ReactNode } from "react";
import {
AI_ENGINE_LABELS,
aiObservationStatusLabel,
@@ -5,19 +6,18 @@ import {
type AiPrompt,
} from "@/shared/ai-visibility";
import { AnswerContent } from "./AnswerContent";
-import { EngineLabel } from "./EngineLabel";
-import { aiDate } from "./shared";
export function PromptAnswers({
projectId,
prompt,
observation,
- executionDate,
+ controls,
}: {
projectId: string;
prompt: AiPrompt | undefined;
observation: AiObservationRow;
- executionDate: string;
+ /** Chooses which answer is shown. */
+ controls: ReactNode;
}) {
return (
<section
@@ -26,11 +26,7 @@ export function PromptAnswers({
>
<div className="flex flex-wrap items-center gap-3 border-b px-5 py-3 border-border">
<h2 className="mr-auto text-sm font-semibold">Answer</h2>
- <div className="flex flex-wrap items-center gap-2 text-xs text-muted-foreground">
- <EngineLabel engine={observation.engine} />
- <span aria-hidden="true">·</span>
- <time dateTime={executionDate}>{aiDate(executionDate)}</time>
- </div>
+ {controls}
</div>
{prompt && observation.prompt !== prompt.text && (
<p
diff --git a/src/client/features/ai-visibility/PromptExecutions.tsx b/src/client/features/ai-visibility/PromptExecutions.tsx
index 6a2df5f5..48325357 100644
--- a/src/client/features/ai-visibility/PromptExecutions.tsx
+++ b/src/client/features/ai-visibility/PromptExecutions.tsx
@@ -9,7 +9,7 @@ import {
import { EngineLabel } from "./EngineLabel";
import { aiDate, AiMatchBadge } from "./shared";
-interface PromptExecution {
+export interface PromptExecution {
observation: AiObservationRow;
run: Pick<AiRun, "id" | "createdAt" | "trigger">;
}
diff --git a/src/client/features/ai-visibility/PromptInventory.tsx b/src/client/features/ai-visibility/PromptInventory.tsx
index 5628a5cc..fb96a7a5 100644
--- a/src/client/features/ai-visibility/PromptInventory.tsx
+++ b/src/client/features/ai-visibility/PromptInventory.tsx
@@ -1,4 +1,5 @@
import { useState } from "react";
+import { cn } from "cn";
import { Link } from "@tanstack/react-router";
import { ChevronDown, ChevronRight } from "lucide-react";
import {
@@ -11,6 +12,7 @@ import {
import { ConfirmDialog } from "@/client/components/ConfirmDialog";
import { Button } from "@/client/components/ui/button";
import { Skeleton } from "@/client/components/ui/skeleton";
+import { Spinner } from "@/client/components/ui/spinner";
import {
Table,
TableBody,
@@ -168,23 +170,46 @@ export function PromptInventory({
const promptRows = rows?.filter(
(row) => row.promptId === prompt.id,
);
+ const ready = promptHasResults(prompt, rows);
+ const running =
+ !ready &&
+ !!promptRows?.some((row) => row.status === "pending");
+ // Without answers there is nothing to inspect yet, so the
+ // prompt and its result cells are grayed out.
+ const dim = ready ? undefined : "opacity-60";
return (
<TableRow key={prompt.id}>
- <TableCell className="min-w-72 py-4 pl-10 first:pl-10">
- <Link
- to="/p/$projectId/ai-visibility/prompts/$promptId"
- params={{ projectId, promptId: prompt.id }}
- className="whitespace-pre-wrap text-sm font-medium underline-offset-4 hover:text-primary hover:underline"
- data-ph-mask
- >
- {prompt.text}
- </Link>
- <p className="mt-1 text-xs text-muted-foreground">
- {prompt.paused ? "Paused" : "Active"} ·{" "}
- {prompt.branded ? "Branded" : "Non-branded"}
+ <TableCell
+ className={cn("min-w-72 py-4 pl-10 first:pl-10", dim)}
+ >
+ {ready ? (
+ <Link
+ to="/p/$projectId/ai-visibility/prompts/$promptId"
+ params={{ projectId, promptId: prompt.id }}
+ className="whitespace-pre-wrap text-sm font-medium underline-offset-4 hover:text-primary hover:underline"
+ data-ph-mask
+ >
+ {prompt.text}
+ </Link>
+ ) : (
+ <span
+ className="whitespace-pre-wrap text-sm font-medium text-muted-foreground"
+ data-ph-mask
+ >
+ {prompt.text}
+ </span>
+ )}
+ <p className="mt-1 flex items-center gap-1.5 text-xs text-muted-foreground">
+ {running && <Spinner className="size-3" />}
+ {running
+ ? "Collecting answers"
+ : prompt.paused
+ ? "Paused"
+ : "Active"}{" "}
+ · {prompt.branded ? "Branded" : "Non-branded"}
</p>
</TableCell>
- <TableCell>
+ <TableCell className={dim}>
<div className="flex min-w-64 items-center gap-3">
{engines.map((engine) => (
<EngineResult
@@ -197,14 +222,14 @@ export function PromptInventory({
))}
</div>
</TableCell>
- <TableCell>
+ <TableCell className={dim}>
<PromptRate
rows={promptRows}
kind="mentioned"
loading={loading}
/>
</TableCell>
- <TableCell>
+ <TableCell className={dim}>
<PromptRate
rows={promptRows}
kind="cited"
@@ -276,6 +301,22 @@ export function PromptInventory({
);
}
+/**
+ * The tracker state knows earlier runs; answers arriving in the current run
+ * unlock a prompt before the state refreshes.
+ */
+export function promptHasResults(
+ prompt: AiPrompt,
+ rows: AiObservationRow[] | undefined,
+) {
+ return (
+ prompt.hasResults ||
+ !!rows?.some(
+ (row) => row.promptId === prompt.id && row.answerStatus === "answered",
+ )
+ );
+}
+
function PromptRate({
rows,
kind,
diff --git a/src/client/features/ai-visibility/PromptList.tsx b/src/client/features/ai-visibility/PromptList.tsx
index 1130aed2..c1ef74c7 100644
--- a/src/client/features/ai-visibility/PromptList.tsx
+++ b/src/client/features/ai-visibility/PromptList.tsx
@@ -1,9 +1,10 @@
import { useState } from "react";
import { useMutation, useQuery } from "@tanstack/react-query";
-import { Plus } from "lucide-react";
+import { Info, Plus } from "lucide-react";
import { toast } from "sonner";
import { ExportMenu } from "@/client/components/ExportMenu";
import { DataTableToolbar } from "@/client/components/table/DataTableToolbar";
+import { Alert, AlertDescription } from "@/client/components/ui/alert";
import { Button } from "@/client/components/ui/button";
import { Input } from "@/client/components/ui/input";
import { exportAiVisibilityData } from "@/serverFunctions/ai-visibility";
@@ -11,13 +12,14 @@ import { safeHttpUrl } from "@/shared/safe-url";
import { getStandardErrorMessage } from "@/client/lib/error-messages";
import type { AiPrompt, AiRun, AiTrackerState } from "@/shared/ai-visibility";
import type { AiTrackerPatch } from "@/types/schemas/ai-visibility";
-import { PromptInventory } from "./PromptInventory";
+import { PromptInventory, promptHasResults } from "./PromptInventory";
import { AiQueryError, aiRunResultsQueryOptions } from "./shared";
export function PromptList({
projectId,
state,
currentRun,
+ collecting,
onSetup,
onEdit,
onReduce,
@@ -27,6 +29,7 @@ export function PromptList({
projectId: string;
state: AiTrackerState;
currentRun: AiRun | null;
+ collecting: boolean;
onSetup: () => void;
onEdit: (prompt: AiPrompt) => void;
onReduce: (patch: AiTrackerPatch) => void;
@@ -97,6 +100,16 @@ export function PromptList({
}}
/>
)}
+ {/* Only before any prompt has results; rows gray out individually. */}
+ {!collecting &&
+ !prompts.some((prompt) => promptHasResults(prompt, results.data)) && (
+ <Alert variant="info" banner>
+ <Info />
+ <AlertDescription>
+ Run prompts before interacting with results.
+ </AlertDescription>
+ </Alert>
+ )}
<PromptInventory
projectId={projectId}
prompts={prompts}
diff --git a/src/client/features/ai-visibility/PromptTrackingPage.tsx b/src/client/features/ai-visibility/PromptTrackingPage.tsx
index 586de08d..7af0b9cf 100644
--- a/src/client/features/ai-visibility/PromptTrackingPage.tsx
+++ b/src/client/features/ai-visibility/PromptTrackingPage.tsx
@@ -237,7 +237,7 @@ function PromptTrackingContent({
</p>
)}
</div>
- <VisibilityTrend projectId={projectId} />
+ <VisibilityTrend projectId={projectId} collecting={busy} />
<PromptTrackingTabs projectId={projectId} tab={tab} />
{tab === "prompts" && (
<PromptList
@@ -245,6 +245,7 @@ function PromptTrackingContent({
projectId={projectId}
state={state}
currentRun={currentRun}
+ collecting={busy}
onSetup={() => setEditorMode(TrackerEditorMode.Prompts)}
onEdit={setEditingPrompt}
onReduce={(patch) => reduceTracking.mutate(patch)}
@@ -259,6 +260,7 @@ function PromptTrackingContent({
projectId={projectId}
state={state}
runId={currentRun?.id}
+ collecting={busy}
/>
)}
{tab === "citations" && (
@@ -267,6 +269,7 @@ function PromptTrackingContent({
projectId={projectId}
state={state}
runId={currentRun?.id}
+ collecting={busy}
/>
)}
</div>
diff --git a/src/client/features/ai-visibility/VisibilityTrend.tsx b/src/client/features/ai-visibility/VisibilityTrend.tsx
index 7d50550b..08e525fc 100644
--- a/src/client/features/ai-visibility/VisibilityTrend.tsx
+++ b/src/client/features/ai-visibility/VisibilityTrend.tsx
@@ -28,7 +28,7 @@ const periods = [
const rate = (value: number | null) =>
value === null ? "—" : `${Math.round(value)}%`;
-function notice(trend: AiTrend) {
+function notice(trend: AiTrend, collecting: boolean) {
const { days } = trend;
switch (trend.comparison) {
case "incomplete":
@@ -36,7 +36,9 @@ function notice(trend: AiTrend) {
case "scope_changed":
return `No prompt and engine has answers for the same market and brand in both periods. The tracked scope changed, so the periods cannot be compared.`;
case "no_data":
- return `There are no finished runs in the last ${days} days.`;
+ return collecting
+ ? "Collecting answers. Visibility appears here when the run finishes."
+ : `There are no finished runs in the last ${days} days.`;
default:
return null;
}
@@ -51,11 +53,17 @@ export const aiTrendQueryOptions = (projectId: string, days: AiTrendDays) =>
});
/** The trend panel inside the Prompt tracking card, laid out like rank tracking's overview. */
-export function VisibilityTrend({ projectId }: { projectId: string }) {
+export function VisibilityTrend({
+ projectId,
+ collecting,
+}: {
+ projectId: string;
+ collecting: boolean;
+}) {
const [days, setDays] = useState<AiTrendDays>(7);
const query = useQuery(aiTrendQueryOptions(projectId, days));
const trend = query.data;
- const message = trend && notice(trend);
+ const message = trend && notice(trend, collecting);
// Incomplete collection and a changed scope need attention; missing history does not.
const warning =
trend?.comparison === "incomplete" || trend?.comparison === "scope_changed";
diff --git a/src/client/features/projects/ProjectWebsiteSetup.tsx b/src/client/features/projects/ProjectWebsiteSetup.tsx
index c6ccfa9a..b0becd67 100644
--- a/src/client/features/projects/ProjectWebsiteSetup.tsx
+++ b/src/client/features/projects/ProjectWebsiteSetup.tsx
@@ -132,7 +132,8 @@ export function ProjectWebsiteSetup({ projectId }: { projectId: string }) {
Start research
</Button>
<p className="text-xs leading-relaxed text-muted-foreground">
- Research uses a small amount of usage credits.
+ Research and a first run of your prompts use a small amount of
+ usage credits.
</p>
</div>
</form>
diff --git a/src/env.d.ts b/src/env.d.ts
index 08a47a1f..28b172ae 100644
--- a/src/env.d.ts
+++ b/src/env.d.ts
@@ -62,7 +62,10 @@ declare namespace Cloudflare {
// DataForSEO API Basic auth value (base64 of login:password)
DATAFORSEO_API_KEY: string;
AI_VISIBILITY_WORKFLOW: Workflow<
- ({ runId: string } | { setupProjectId: string }) & {
+ (
+ | { runId: string }
+ | { setupProjectId: string; runSeededTracking?: boolean }
+ ) & {
customer: {
organizationId: string;
userId: string;
diff --git a/src/server/features/ai-visibility/repositories/AiVisibilityRepository.ts b/src/server/features/ai-visibility/repositories/AiVisibilityRepository.ts
index 92f322ec..06d83ba2 100644
--- a/src/server/features/ai-visibility/repositories/AiVisibilityRepository.ts
+++ b/src/server/features/ai-visibility/repositories/AiVisibilityRepository.ts
@@ -161,6 +161,26 @@ async function getEvidence(observationIds: string[]) {
return { sources, matches };
}
+/**
+ * Prompts with at least one collected answer, in any run. A completed
+ * collection with no answer text (no AI Overview shown) does not count.
+ */
+async function listAnsweredPromptIds(trackerId: string) {
+ const rows = await db
+ .selectDistinct({ promptId: aiObservations.promptId })
+ .from(aiObservations)
+ .innerJoin(aiPrompts, eq(aiPrompts.id, aiObservations.promptId))
+ .where(
+ and(
+ eq(aiPrompts.trackerId, trackerId),
+ eq(aiObservations.status, "completed"),
+ // SQL `<> ''` is also false for NULL, so this excludes both.
+ ne(aiObservations.answerMarkdown, ""),
+ ),
+ );
+ return rows.map((row) => row.promptId);
+}
+
/** Replaces an answer's evidence, so a retried collect step stays idempotent. */
async function persistAnswer(input: {
observationId: string;
@@ -275,6 +295,7 @@ export const AiVisibilityRepository = {
db.update(aiRuns).set(values).where(eq(aiRuns.id, id)),
getObservations,
getObservation,
+ listAnsweredPromptIds,
getObservationStatuses: (runIds: string[]) =>
runIds.length
? db
diff --git a/src/server/features/ai-visibility/services/AiVisibilityService.ts b/src/server/features/ai-visibility/services/AiVisibilityService.ts
index 310ff488..22539fa8 100644
--- a/src/server/features/ai-visibility/services/AiVisibilityService.ts
+++ b/src/server/features/ai-visibility/services/AiVisibilityService.ts
@@ -25,8 +25,13 @@ export const AiVisibilityService = {
startResearchSetup: async (
input: { projectId: string },
billing: BillingCustomerContext,
+ options?: { runSeededTracking?: boolean },
) =>
- (await import("./aiResearchKeywords")).startAiResearchSetup(input, billing),
+ (await import("./aiResearchKeywords")).startAiResearchSetup(
+ input,
+ billing,
+ options,
+ ),
researchPrompts: researchAiPrompts,
listResearchKeywords: listAiResearchKeywords,
getTracker,
diff --git a/src/server/features/ai-visibility/services/aiResearchKeywords.test.ts b/src/server/features/ai-visibility/services/aiResearchKeywords.test.ts
index 609c1455..a208b812 100644
--- a/src/server/features/ai-visibility/services/aiResearchKeywords.test.ts
+++ b/src/server/features/ai-visibility/services/aiResearchKeywords.test.ts
@@ -1,12 +1,20 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
-const { listCompetitors, listResearchKeywords, research, workflow } =
- vi.hoisted(() => ({
- listCompetitors: vi.fn(),
- listResearchKeywords: vi.fn(),
- research: vi.fn(),
- workflow: { get: vi.fn(), create: vi.fn() },
- }));
+const {
+ listCompetitors,
+ listResearchKeywords,
+ research,
+ workflow,
+ prepareWebsiteTracking,
+ startWebsiteTrackingRun,
+} = vi.hoisted(() => ({
+ listCompetitors: vi.fn(),
+ listResearchKeywords: vi.fn(),
+ research: vi.fn(),
+ workflow: { get: vi.fn(), create: vi.fn() },
+ prepareWebsiteTracking: vi.fn(),
+ startWebsiteTrackingRun: vi.fn(),
+}));
vi.mock("cloudflare:workers", () => ({
env: { AI_VISIBILITY_WORKFLOW: workflow },
@@ -23,7 +31,10 @@ vi.mock("../repositories/aiVisibilityConfigurationRepository", () => ({
writeConfiguration: vi.fn(),
writeResearchKeywords: vi.fn(),
}));
-vi.mock("./websiteTrackingSetup", () => ({ prepareWebsiteTracking: vi.fn() }));
+vi.mock("./websiteTrackingSetup", () => ({
+ prepareWebsiteTracking,
+ startWebsiteTrackingRun,
+}));
vi.mock("@/server/features/projects/repositories/ProjectRepository", () => ({
ProjectRepository: {
getProjectForOrganization: async () => ({
@@ -66,6 +77,7 @@ describe("runAiResearchSetup", () => {
beforeEach(() => {
listResearchKeywords.mockResolvedValue([]);
listCompetitors.mockResolvedValue([]);
+ prepareWebsiteTracking.mockResolvedValue(null);
research.mockResolvedValue({
name: "OpenSEO",
domain: "openseo.so",
@@ -92,6 +104,19 @@ describe("runAiResearchSetup", () => {
expect(research).not.toHaveBeenCalled();
});
+ it("runs newly seeded tracking once only for in-app setup", async () => {
+ listCompetitors.mockResolvedValue([{ domain: "ahrefs.com" }]);
+ prepareWebsiteTracking.mockResolvedValue({ prompts: [] });
+
+ await runAiResearchSetup({ projectId: "project" }, customer);
+ expect(startWebsiteTrackingRun).not.toHaveBeenCalled();
+
+ await runAiResearchSetup({ projectId: "project" }, customer, {
+ runSeededTracking: true,
+ });
+ expect(startWebsiteTrackingRun).toHaveBeenCalledWith("project", customer);
+ });
+
it("returns website research for competitor review when competitors are missing", async () => {
const result = await runAiResearchSetup({ projectId: "project" }, customer);
@@ -155,6 +180,7 @@ describe("startAiResearchSetup", () => {
id: "ai-research-setup-project",
params: {
setupProjectId: "project",
+ runSeededTracking: false,
customer: { ...customer, projectId: "project" },
},
});
diff --git a/src/server/features/ai-visibility/services/aiResearchKeywords.ts b/src/server/features/ai-visibility/services/aiResearchKeywords.ts
index 8a7ef888..74bcb6bc 100644
--- a/src/server/features/ai-visibility/services/aiResearchKeywords.ts
+++ b/src/server/features/ai-visibility/services/aiResearchKeywords.ts
@@ -20,7 +20,10 @@ import {
writeConfiguration,
writeResearchKeywords,
} from "../repositories/aiVisibilityConfigurationRepository";
-import { prepareWebsiteTracking } from "./websiteTrackingSetup";
+import {
+ prepareWebsiteTracking,
+ startWebsiteTrackingRun,
+} from "./websiteTrackingSetup";
/**
* Prompt Research needs saved keywords. Website setup writes them; a
@@ -100,6 +103,9 @@ export async function getAiResearchSetup(projectId: string) {
export async function startAiResearchSetup(
input: { projectId: string },
customer: BillingCustomerContext,
+ // In-app onboarding runs seeded tracking once. MCP setup never spends on
+ // answers. A restarted setup keeps the flag it was first started with.
+ { runSeededTracking = false } = {},
) {
const setup = await getAiResearchSetup(input.projectId);
if (setup.status !== "none" && setup.status !== "failed") return setup;
@@ -122,6 +128,7 @@ export async function startAiResearchSetup(
// Only the billing fields: the request context is not serializable.
params: {
setupProjectId: input.projectId,
+ runSeededTracking,
customer: {
organizationId: customer.organizationId,
userId: customer.userId,
@@ -137,6 +144,7 @@ export async function startAiResearchSetup(
export async function runAiResearchSetup(
input: { projectId: string },
customer: BillingCustomerContext,
+ { runSeededTracking = false } = {},
): Promise<{ review: WebsiteResearch | null }> {
const project = await ProjectRepository.getProjectForOrganization(
input.projectId,
@@ -177,5 +185,7 @@ export async function runAiResearchSetup(
...review.suggestedKeywords,
]),
]);
+ if (tracking && runSeededTracking)
+ await startWebsiteTrackingRun(input.projectId, customer);
return { review: null };
}
diff --git a/src/server/features/ai-visibility/services/aiVisibilityRuns.test.ts b/src/server/features/ai-visibility/services/aiVisibilityRuns.test.ts
index 88a42250..6a24cf35 100644
--- a/src/server/features/ai-visibility/services/aiVisibilityRuns.test.ts
+++ b/src/server/features/ai-visibility/services/aiVisibilityRuns.test.ts
@@ -3,6 +3,7 @@ import { beforeEach, describe, expect, it, vi } from "vitest";
import { aiObservations, aiRuns } from "@/db/schema";
import { AiVisibilityRepository as repo } from "../repositories/AiVisibilityRepository";
import { runCheck } from "./aiVisibilityRuns";
+import { getTracker } from "./aiVisibilityState";
import { configuration, customer } from "./aiVisibilityTestFixtures";
const { testDb, workflow } = await vi.hoisted(async () => {
@@ -72,6 +73,29 @@ describe("AI visibility checks", () => {
).toEqual([{ status: "failed" }]);
});
+ it("marks only prompts with collected answer text as having results", async () => {
+ const run = await runCheck({ projectId, maxCostUsd: 1 }, customer);
+ const answers = await repo.getObservations([run.id]);
+ for (const answer of answers)
+ await testDb.db
+ .update(aiObservations)
+ .set({
+ status: "completed",
+ // A completed collection without text means no answer was shown.
+ answerMarkdown:
+ answer.prompt === "Is OpenSEO good?" ? "OpenSEO is good." : null,
+ })
+ .where(eq(aiObservations.id, answer.id));
+
+ const { prompts } = await getTracker({ projectId });
+
+ expect(
+ Object.fromEntries(
+ prompts.map((prompt) => [prompt.text, prompt.hasResults]),
+ ),
+ ).toEqual({ "Is OpenSEO good?": true, "Which SEO tools?": false });
+ });
+
it("refuses a check that now costs more than the approved maximum", async () => {
await expect(
runCheck({ projectId, maxCostUsd: 0.001 }, customer),
diff --git a/src/server/features/ai-visibility/services/aiVisibilityState.ts b/src/server/features/ai-visibility/services/aiVisibilityState.ts
index bbb88e1b..956bead2 100644
--- a/src/server/features/ai-visibility/services/aiVisibilityState.ts
+++ b/src/server/features/ai-visibility/services/aiVisibilityState.ts
@@ -39,7 +39,12 @@ export async function getTracker(input: {
);
// A run whose workflow stopped is failed here, so it never looks stuck.
const runs = await Promise.all(recent.map((run) => reconcileAiRun(run)));
- const observations = await repo.getObservationStatuses(runs.map((r) => r.id));
+ const [observations, answered] = await Promise.all([
+ repo.getObservationStatuses(runs.map((r) => r.id)),
+ config
+ ? repo.listAnsweredPromptIds(config.tracker.id)
+ : Promise.resolve<string[]>([]),
+ ]);
const brands = project ? aiBrands(project, competitors) : [];
const own = brands.find((b) => b.own);
const capabilities = aiEngineSchema.options.map((engine) => ({
@@ -84,6 +89,7 @@ export async function getTracker(input: {
paused: p.paused,
archived: p.archived,
branded: !!own && aiPromptIsBranded(p.text, own),
+ hasResults: answered.includes(p.id),
})),
brands,
engines: trackerEngines(tracker),
diff --git a/src/server/features/ai-visibility/services/websiteTrackingSetup.test.ts b/src/server/features/ai-visibility/services/websiteTrackingSetup.test.ts
index 3a2321ba..d5afaf2d 100644
--- a/src/server/features/ai-visibility/services/websiteTrackingSetup.test.ts
+++ b/src/server/features/ai-visibility/services/websiteTrackingSetup.test.ts
@@ -1,10 +1,23 @@
-import { describe, expect, it, vi } from "vitest";
+import { beforeEach, describe, expect, it, vi } from "vitest";
+const { getConfiguration, startRun } = vi.hoisted(() => ({
+ getConfiguration: vi.fn(),
+ startRun: vi.fn(),
+}));
vi.mock("../repositories/AiVisibilityRepository", () => ({
- AiVisibilityRepository: { getConfiguration: async () => null },
+ AiVisibilityRepository: { getConfiguration },
}));
+vi.mock("./aiVisibilityRuns", () => ({ startRun }));
+
+import {
+ prepareWebsiteTracking,
+ startWebsiteTrackingRun,
+} from "./websiteTrackingSetup";
+import { configuration, customer } from "./aiVisibilityTestFixtures";
-import { prepareWebsiteTracking } from "./websiteTrackingSetup";
+beforeEach(() => {
+ getConfiguration.mockResolvedValue(null);
+});
const topic = (name: string) => ({
name,
@@ -35,3 +48,16 @@ describe("prepareWebsiteTracking", () => {
expect(tracking?.prompts).toHaveLength(15);
});
});
+
+describe("startWebsiteTrackingRun", () => {
+ it("starts the baseline, and a run that cannot start leaves setup saved", async () => {
+ const config = configuration();
+ getConfiguration.mockResolvedValue(config);
+ startRun.mockRejectedValue(new Error("DATAFORSEO_API_KEY is missing"));
+
+ await expect(
+ startWebsiteTrackingRun("project", customer),
+ ).resolves.toBeUndefined();
+ expect(startRun).toHaveBeenCalledWith(config, "baseline", customer);
+ });
+});
diff --git a/src/server/features/ai-visibility/services/websiteTrackingSetup.ts b/src/server/features/ai-visibility/services/websiteTrackingSetup.ts
index 1440739f..61c4d141 100644
--- a/src/server/features/ai-visibility/services/websiteTrackingSetup.ts
+++ b/src/server/features/ai-visibility/services/websiteTrackingSetup.ts
@@ -5,6 +5,8 @@ import { aiEnginesWithoutLocation } from "@/shared/ai-visibility";
import { DEFAULT_LOCATION_CODE } from "@/shared/keyword-locations";
import type { SaveProjectWebsiteSetup } from "@/types/schemas/projectWebsite";
import { AppError } from "@/server/lib/errors";
+import type { BillingCustomerContext } from "@/server/billing/subscription";
+import { startRun } from "./aiVisibilityRuns";
/** Seeds weekly tracking from website setup's first three suggested topics. */
export async function prepareWebsiteTracking(
@@ -51,3 +53,22 @@ export async function prepareWebsiteTracking(
now: new Date().toISOString(),
}).rows;
}
+
+/**
+ * In-app onboarding collects the seeded prompts once, so Prompt Tracking
+ * opens on a run in progress rather than an empty tracker. The run is the
+ * tracker's baseline; collection checks credits before paid work. Setup is
+ * already saved, so a run that cannot start leaves the prompts ready to run.
+ */
+export async function startWebsiteTrackingRun(
+ projectId: string,
+ billing: BillingCustomerContext,
+) {
+ const config = await repo.getConfiguration(projectId);
+ if (!config) return;
+ try {
+ await startRun(config, "baseline", billing);
+ } catch (error) {
+ console.warn(`[ai-visibility] ${projectId} onboarding run:`, error);
+ }
+}
diff --git a/src/server/features/projects/services/ProjectWebsiteService.ts b/src/server/features/projects/services/ProjectWebsiteService.ts
index 58f7b83d..3a0fd184 100644
--- a/src/server/features/projects/services/ProjectWebsiteService.ts
+++ b/src/server/features/projects/services/ProjectWebsiteService.ts
@@ -11,7 +11,10 @@ import { saveWebsiteSetup } from "@/server/features/projects/repositories/Projec
import { researchWebsite } from "@/server/features/projects/services/websiteResearch";
import { normalizeBacklinksTarget } from "@/server/lib/dataforseoBacklinksTarget";
import { AppError } from "@/server/lib/errors";
-import { prepareWebsiteTracking } from "@/server/features/ai-visibility/services/websiteTrackingSetup";
+import {
+ prepareWebsiteTracking,
+ startWebsiteTrackingRun,
+} from "@/server/features/ai-visibility/services/websiteTrackingSetup";
import {
saveProjectWebsiteSetupSchema,
type SaveProjectWebsiteSetup,
@@ -54,6 +57,9 @@ async function save(
input: SaveProjectWebsiteSetup,
customer: BillingCustomerContext,
author: ContextAuthor,
+ // In-app onboarding runs seeded tracking once. MCP setup never spends;
+ // agents get approval before a run.
+ { runSeededTracking = false } = {},
) {
const accepted = saveProjectWebsiteSetupSchema.parse(input);
const project = await requireProject(customer, accepted.projectId);
@@ -85,6 +91,8 @@ async function save(
];
const tracking = await prepareWebsiteTracking(accepted, project);
await saveWebsiteSetup(customer.organizationId, accepted, author, tracking);
+ if (tracking && runSeededTracking)
+ await startWebsiteTrackingRun(accepted.projectId, customer);
return requireProject(customer, accepted.projectId);
}
export const ProjectWebsiteService = { research, save };
diff --git a/src/server/mcp/tools/ai-visibility-output-schemas.ts b/src/server/mcp/tools/ai-visibility-output-schemas.ts
index 56e1024b..d0280618 100644
--- a/src/server/mcp/tools/ai-visibility-output-schemas.ts
+++ b/src/server/mcp/tools/ai-visibility-output-schemas.ts
@@ -47,6 +47,7 @@ export const aiTrackerOutput = z.looseObject({
paused: flag,
archived: flag,
branded: flag,
+ hasResults: flag,
}),
),
brands: z.array(z.looseObject({ name: text, domain: text, own: flag })),
diff --git a/src/server/workflows/AiVisibilityWorkflow.ts b/src/server/workflows/AiVisibilityWorkflow.ts
index c8e1fd53..f6f65c0d 100644
--- a/src/server/workflows/AiVisibilityWorkflow.ts
+++ b/src/server/workflows/AiVisibilityWorkflow.ts
@@ -17,7 +17,10 @@ import type { BillingCustomerContext } from "@/server/billing/subscription";
// One binding serves answer collection runs and the one-time research setup,
// so setup needs no new workflow infrastructure.
-type AiVisibilityPayload = ({ runId: string } | { setupProjectId: string }) & {
+type AiVisibilityPayload = (
+ | { runId: string }
+ | { setupProjectId: string; runSeededTracking?: boolean }
+) & {
customer: BillingCustomerContext;
};
@@ -63,6 +66,7 @@ export class AiVisibilityWorkflow extends WorkflowEntrypoint<
runAiResearchSetup(
{ projectId: payload.setupProjectId },
payload.customer,
+ { runSeededTracking: payload.runSeededTracking },
),
);
const { runId, customer } = payload;
diff --git a/src/serverFunctions/ai-visibility.ts b/src/serverFunctions/ai-visibility.ts
index b968e4a9..778ad2b7 100644
--- a/src/serverFunctions/ai-visibility.ts
+++ b/src/serverFunctions/ai-visibility.ts
@@ -53,6 +53,7 @@ export const startAiResearchSetup = createServerFn({ method: "POST" })
AiVisibilityService.startResearchSetup(
{ projectId: context.projectId },
context,
+ { runSeededTracking: true },
),
);
diff --git a/src/serverFunctions/projectWebsite.ts b/src/serverFunctions/projectWebsite.ts
index 16ea2f7f..f8d03cba 100644
--- a/src/serverFunctions/projectWebsite.ts
+++ b/src/serverFunctions/projectWebsite.ts
@@ -20,5 +20,6 @@ export const saveProjectWebsiteSetup = createServerFn({ method: "POST" })
{ ...data, projectId: context.projectId },
context,
"user",
+ { runSeededTracking: true },
),
);
diff --git a/src/shared/ai-visibility.ts b/src/shared/ai-visibility.ts
index a04766be..d129fb0c 100644
--- a/src/shared/ai-visibility.ts
+++ b/src/shared/ai-visibility.ts
@@ -25,6 +25,8 @@ export interface AiPrompt {
paused: boolean;
archived: boolean;
branded: boolean;
+ /** Whether any run has collected answer text for this prompt. */
+ hasResults: boolean;
}
export interface AiBrand {
name: string;