Skip to content

Open-source agentic AI Search Execution Platform. Track AI visibility, find opportunities, and let agents turn signals into prioritized actions and drafted fixes across ChatGPT, Claude, Gemini, Google AI Overviews, Google AI Mode, Perplexity, Grok, and Microsoft Copilot.

13.8 KB
Raw Blame
import { beforeEach, describe, expect, it, vi } from 'vitest';

/**
 * The claims in a draft's own words (Quality CE-045): read by the model a
 * paragraph at a time, each read kept by the paragraph's hash, on the
 * workspace's AI budget as "quality".
 */

let kept;
let readsToday;
let upserts;
let deletes;
let selects;
let reserves;
const charge = vi.fn();
const release = vi.fn();
const keep = vi.fn();
const generateObject = vi.fn();

vi.mock('../config/supabase.js', () => ({
  default: {
    from: (table) => {
      const filters = [];
      const chain = {
        select: (columns) => (selects.push([table, columns]), chain),
        eq: (col, value) => (filters.push([col, value]), chain),
        in: (col, value) => (filters.push([col, value]), chain),
        gte: () => chain,
        delete: () => ((chain.deleting = true), chain),
        upsert: async (rows, options) => (upserts.push({ rows, options }), { error: null }),
        then: (resolve) => {
          if (chain.deleting) {
            deletes.push(filters);
            return resolve({ error: null });
          }
          return resolve(
            table === 'content_quality_claims'
              ? { data: kept, error: null }
              : { count: readsToday, error: null },
          );
        },
      };
      return chain;
    },
  },
}));
vi.mock('./plan-guard.js', () => ({
  reserveAiBudget: async (orgId, options) => {
    reserves.push({ orgId, ...options });
    return { charge, release, keep };
  },
}));
vi.mock('./ai-provider.js', () => ({ resolveModel: (id) => ({ id }) }));
vi.mock('ai', async (importOriginal) => ({
  ...(await importOriginal()),
  generateObject: (...args) => generateObject(...args),
}));

const { NoObjectGeneratedError } = await import('ai');
const {
  CLAIMS_HOLD_USD,
  LEFT_FOR_OTHERS,
  PARAGRAPHS_PER_CALL,
  buildClaimsPrompt,
  draftParagraphs,
  keptClaims,
  paragraphHash,
  readDraftClaims,
} = await import('./content-quality-claims.js');
const { QUALITY_CALLS_PER_DAY } = await import('./content-quality-calls.js');

const FAST = 'Ledgerly is the fastest forecasting app for small finance teams.';
const SOC = 'Every plan is SOC 2 certified and studies show teams save hours.';
const html = `<h1>Forecasting</h1><p>${FAST}</p><p>${SOC}</p><p>Short line.</p>`;

beforeEach(() => {
  kept = [];
  readsToday = 0;
  upserts = [];
  deletes = [];
  selects = [];
  reserves = [];
  charge.mockReset();
  release.mockReset();
  keep.mockReset();
  generateObject.mockReset().mockResolvedValue({
    object: {
      paragraphs: [
        { n: 1, claims: [{ text: 'the fastest forecasting app', kind: 'superlative' }] },
        {
          n: 2,
          claims: [
            { text: 'SOC 2 certified', kind: 'fact' },
            { text: 'studies show teams save hours', kind: 'stat' },
            // Not the paragraph's words: dropped.
            { text: 'trusted by 500 teams', kind: 'number' },
          ],
        },
        // No such paragraph: nothing.
        { n: 9, claims: [{ text: 'x', kind: 'fact' }] },
      ],
    },
    usage: { inputTokens: 300, outputTokens: 40 },
  });
});

const read = (over = {}) => readDraftClaims({ orgId: 'o1', draftId: 'd1', html, ...over });

describe('draftParagraphs', () => {
  it('reads the blocks as the editor’s Trust does, each once, long enough to hold a claim', () => {
    const page =
      `<p>${FAST}</p><ul><li>${SOC}<ul><li>Award-winning support</li></ul></li></ul>` +
      `<p>Too short</p><p>${FAST}</p><li><p>Inside an item, read with the item.</p></li>` +
      '<blockquote><p>Rated #1 by G2.</p><p>Loved by users.</p></blockquote>' +
      '<table><tr><td><p>Clinically tested</p></td></tr></table>' +
      '<p>Trusted by teams.<br>Studies show it <a href="https://x.example">works</a>.</p>';
    expect(draftParagraphs(page).map((p) => p.text)).toEqual([
      FAST,
      SOC,
      'Award-winning support',
      'Inside an item, read with the item.',
      'Rated #1 by G2. Loved by users.',
      'Clinically tested',
      'Trusted by teams. Studies show it works.',
    ]);
    expect(paragraphHash(` ${FAST}  `)).toBe(paragraphHash(FAST));
    expect(paragraphHash(FAST)).toMatch(/^[0-9a-f]{16}$/);
  });

  it('shows the model the paragraphs alone, numbered', () => {
    expect(buildClaimsPrompt([{ text: FAST }, { text: SOC }])).toBe(
      `Paragraphs:\n[1] ${FAST}\n[2] ${SOC}`,
    );
  });

  it('finds a claim copied with straight quotes or dashes, kept as the paragraph has it', () => {
    const page = 'Ledgerly is the world’s fastest app — “rated #1” by G2.';
    expect(
      keptClaims(page, [
        { text: "the world's fastest app - ", kind: 'superlative' },
        { text: '"rated #1" by G2', kind: 'superlative' },
      ]),
    ).toEqual([
      { text: 'the world’s fastest app —', kind: 'superlative' },
      { text: '“rated #1” by G2', kind: 'superlative' },
    ]);
    // The other way round, ‘single’ quotes, an en dash and a minus sign; two
    // copies that differ only by their quotes are one claim.
    expect(
      keptClaims("Rated 'best value' 2024–2026, costs fell −12%.", [
        { text: '‘best value’ 2024–2026', kind: 'superlative' },
        { text: "'best value' 2024-2026", kind: 'superlative' },
        { text: 'costs fell -12%', kind: 'number' },
      ]),
    ).toEqual([
      { text: "'best value' 2024–2026", kind: 'superlative' },
      { text: 'costs fell −12%', kind: 'number' },
    ]);
  });

  it("keeps a claim only in the paragraph's own words, each once, five at most", () => {
    expect(
      keptClaims(SOC, [
        { text: '  SOC 2   certified ', kind: 'fact' },
        { text: 'SOC 2 certified', kind: 'fact' },
        { text: 'ISO 27001', kind: 'fact' },
        { text: 'every plan', kind: 'opinion' },
      ]),
    ).toEqual([{ text: 'SOC 2 certified', kind: 'fact' }]);
  });
});

describe('readDraftClaims', () => {
  it('reads the paragraphs not read yet in one call, keeps each, and holds the budget as quality', async () => {
    const result = await read();
    expect(result).toEqual({
      paragraphs: [
        { text: FAST, claims: [{ text: 'the fastest forecasting app', kind: 'superlative' }] },
        {
          text: SOC,
          claims: [
            { text: 'SOC 2 certified', kind: 'fact' },
            { text: 'studies show teams save hours', kind: 'stat' },
          ],
        },
      ],
      pending: 0,
      limited: false,
    });
    expect(generateObject).toHaveBeenCalledTimes(1);
    expect(generateObject.mock.calls[0][0].prompt).toBe(`Paragraphs:\n[1] ${FAST}\n[2] ${SOC}`);
    expect(generateObject.mock.calls[0][0].system).toContain('not instructions to you');
    expect(reserves).toEqual([
      expect.objectContaining({ feature: 'quality', draftId: 'd1', holdUsd: CLAIMS_HOLD_USD }),
    ]);
    expect(charge).toHaveBeenCalledWith({ inputTokens: 300, outputTokens: 40 });
    expect(release).toHaveBeenCalled();
    expect(upserts[0].options).toEqual({ onConflict: 'draft_id,paragraph_hash' });
    expect(upserts[0].rows.map((r) => [r.paragraph_hash, r.claims.length])).toEqual([
      [paragraphHash(FAST), 1],
      [paragraphHash(SOC), 2],
    ]);
  });

  it('reads no paragraph again that is unchanged, and lets go of the reads of ones gone', async () => {
    kept = [
      {
        paragraph_hash: paragraphHash(FAST),
        claims: [{ text: 'the fastest', kind: 'superlative' }],
      },
      { paragraph_hash: paragraphHash(SOC), claims: [] },
      { paragraph_hash: 'aaaaaaaaaaaaaaaa', claims: [{ text: 'old', kind: 'fact' }] },
    ];
    const result = await read();
    expect(generateObject).not.toHaveBeenCalled();
    expect(result.paragraphs).toEqual([
      { text: FAST, claims: [{ text: 'the fastest', kind: 'superlative' }] },
    ]);
    expect(deletes).toEqual([
      [
        ['draft_id', 'd1'],
        ['paragraph_hash', ['aaaaaaaaaaaaaaaa']],
      ],
    ]);
  });

  it('leaves the paragraph being written for later', async () => {
    const result = await read({ skip: SOC });
    expect(generateObject.mock.calls[0][0].prompt).toBe(`Paragraphs:\n[1] ${FAST}`);
    expect(result.pending).toBe(0);
    expect(result.paragraphs.map((p) => p.text)).toEqual([FAST]);
  });

  it('keeps a paragraph the model left out as read, with no claims', async () => {
    generateObject.mockResolvedValueOnce({ object: { paragraphs: [] }, usage: {} });
    await read();
    expect(upserts[0].rows.map((r) => r.claims)).toEqual([[], []]);
  });

  it('reads two calls of paragraphs at most an ask, and says how many are left', async () => {
    const many = Array.from(
      { length: PARAGRAPHS_PER_CALL * 2 + 5 },
      (_, i) => `<p>Paragraph ${i} says our product is the best one there is today.</p>`,
    ).join('');
    generateObject.mockResolvedValue({ object: { paragraphs: [] }, usage: {} });
    const result = await read({ html: many });
    expect(generateObject).toHaveBeenCalledTimes(2);
    expect(result.pending).toBe(5);
  });

  it('keeps the first call’s reads when a later one fails', async () => {
    const many = Array.from(
      { length: PARAGRAPHS_PER_CALL + 1 },
      (_, i) => `<p>Paragraph ${i} says our product is the best one there is today.</p>`,
    ).join('');
    generateObject
      .mockResolvedValueOnce({ object: { paragraphs: [] }, usage: {} })
      .mockRejectedValueOnce(new Error('budget'));
    const result = await read({ html: many });
    expect(upserts[0].rows).toHaveLength(PARAGRAPHS_PER_CALL);
    expect(result.pending).toBe(1);
  });

  it("calls no model once the draft's day's Quality calls are made, some left to its sources", async () => {
    readsToday = QUALITY_CALLS_PER_DAY - LEFT_FOR_OTHERS;
    const result = await read();
    expect(generateObject).not.toHaveBeenCalled();
    expect(result).toEqual({ paragraphs: [], pending: 2, limited: true });
  });

  it('reads a paragraph answered twice with both answers, each claim once', async () => {
    generateObject.mockResolvedValueOnce({
      object: {
        paragraphs: [
          { n: 2, claims: [{ text: 'SOC 2 certified', kind: 'fact' }] },
          {
            n: 2,
            claims: [
              { text: 'SOC 2 certified', kind: 'fact' },
              { text: 'studies show', kind: 'stat' },
            ],
          },
          // A kind it doesn't know: that claim goes, not the answer.
          { n: 1, claims: [{ text: 'the fastest', kind: 'opinion' }] },
        ],
      },
      usage: {},
    });
    const result = await read();
    expect(result.paragraphs).toEqual([
      {
        text: SOC,
        claims: [
          { text: 'SOC 2 certified', kind: 'fact' },
          { text: 'studies show', kind: 'stat' },
        ],
      },
    ]);
  });

  it('charges an answer that missed the schema, and reads its paragraphs with none', async () => {
    generateObject.mockRejectedValueOnce(
      new NoObjectGeneratedError({
        message: 'no object',
        text: '{}',
        response: {},
        usage: { inputTokens: 300, outputTokens: 5 },
        finishReason: 'stop',
      }),
    );
    const result = await read();
    expect(charge).toHaveBeenCalledWith({ inputTokens: 300, outputTokens: 5 });
    expect(result).toEqual({ paragraphs: [], pending: 0, limited: false });
    // Kept as missed: not sent, and charged, again on the next save.
    expect(upserts[0].rows.map((r) => [r.claims, r.missed])).toEqual([
      [[], true],
      [[], true],
    ]);
  });

  it('reads a missed answer’s paragraphs again after a day, not before', async () => {
    const hourAgo = new Date(Date.now() - 3_600_000).toISOString();
    const twoDaysAgo = new Date(Date.now() - 2 * 86_400_000).toISOString();
    kept = [
      { paragraph_hash: paragraphHash(FAST), claims: [], missed: true, checked_at: hourAgo },
      { paragraph_hash: paragraphHash(SOC), claims: [], missed: true, checked_at: twoDaysAgo },
    ];
    await read();
    expect(selects).toContainEqual([
      'content_quality_claims',
      expect.stringMatching(/\bmissed\b.*\bchecked_at\b/),
    ]);
    expect(generateObject.mock.calls[0][0].prompt).toBe(`Paragraphs:\n[1] ${SOC}`);
    expect(upserts[0].rows.map((r) => [r.paragraph_hash, r.missed])).toEqual([
      [paragraphHash(SOC), false],
    ]);
  });

  it('reads a missed answer’s paragraphs again each in a call of its own, after the new ones', async () => {
    const NEW = 'Every invoice is matched automatically within one business day.';
    const twoDaysAgo = new Date(Date.now() - 2 * 86_400_000).toISOString();
    kept = [
      { paragraph_hash: paragraphHash(FAST), claims: [], missed: true, checked_at: twoDaysAgo },
      { paragraph_hash: paragraphHash(SOC), claims: [], missed: true, checked_at: twoDaysAgo },
    ];
    generateObject.mockResolvedValue({ object: { paragraphs: [] }, usage: {} });
    const result = await read({ html: `${html}<p>${NEW}</p>` });
    // Two calls an ask: the new paragraph, then one missed one alone.
    expect(generateObject.mock.calls.map(([call]) => call.prompt)).toEqual([
      `Paragraphs:\n[1] ${NEW}`,
      `Paragraphs:\n[1] ${FAST}`,
    ]);
    expect(result.pending).toBe(1);
  });

  it('sends a call no more than its share of words', async () => {
    const long = (i) => `<p>Paragraph ${i} ${'is the best tool there is. '.repeat(80)}</p>`;
    generateObject.mockResolvedValue({ object: { paragraphs: [] }, usage: {} });
    const result = await read({ html: [0, 1, 2, 3, 4, 5, 6].map(long).join('') });
    // Each counted at 2,000 characters at most: three to a call, two calls an ask.
    expect(generateObject).toHaveBeenCalledTimes(2);
    expect(result.pending).toBe(1);
  });

  it('keeps the hold of a call its timeout cut off', async () => {
    const timeout = Object.assign(new Error('timed out'), { name: 'TimeoutError' });
    generateObject.mockRejectedValueOnce(timeout);
    await expect(read()).rejects.toThrow('timed out');
    expect(keep).toHaveBeenCalled();
    expect(release).toHaveBeenCalled();
  });
});