Skip to content

Open source website builder and Webflow alternative. Webstudio is an advanced visual builder that connects to any headless CMS, supports all CSS properties, and can be hosted anywhere, including with us.

9.0 KB
Raw Blame
import { describe, expect, test } from "vitest";
import type { IssueReportInput } from "@webstudio-is/protocol";
import {
  createGitHubInstallationToken,
  formatIssueReport,
  publishIssueReport,
} from "./github-issue-report.server";

const report: IssueReportInput = {
  trigger: "automatic-friction",
  category: "schema-or-docs-mismatch",
  deduplicationKey: "update-props-input-contract",
  title: "fix: Clarify the update-props input contract",
  agent: {
    client: "Codex",
    clientVersion: "1.2.3",
    provider: "OpenAI",
    model: "gpt-5.6-sol",
    reasoningEffort: "medium",
  },
  runtime: {
    projectId: "project-123",
    cliVersion: "1.2.3",
    nodeVersion: "22.14.0",
    os: "linux",
    osVersion: "6",
    architecture: "arm64",
    executionMode: "mcp",
    apiContractVersion: "public-api:client",
    serverApiContractVersion: "public-api:server",
    bundleVersion: "bundle:client",
    recentFailure: {
      entityIds: [
        { field: "pageId", id: "page-123" },
        { field: "instanceId", id: "instance-123" },
      ],
      tool: "preview.start",
      code: "PROJECT_BUNDLE_INVALID",
      errorType: "AssetRevisionConflictError",
      requestId: "abc123-LIS",
      httpStatus: 504,
      elapsedMs: 17_000,
      issues: [
        {
          path: ["assets", "0", "type"],
          code: "invalid_value",
          constraint: 'one of "font"|"image"|"video"|"file"',
        },
      ],
    },
    recentTargets: [
      { tool: "screenshot", entityIds: [{ field: "pageId", id: "page-456" }] },
      {
        tool: "inspect-instance",
        entityIds: [{ field: "instanceId", id: "instance-456" }],
      },
    ],
    session: {
      staleNamespaces: ["styles"],
      missingNamespaces: ["instances"],
      invalidatedNamespaces: [],
    },
    preview: {
      stale: true,
      hasRenderedVersion: true,
      renderedVersionMatchesSession: false,
    },
  },
  report: {
    userStory:
      "As a Webstudio user, I want routine MCP edits to complete without corrective retries.",
    summary: "A documented operation required a corrected retry.",
    attemptedWorkflow: ["Inspect the component.", "Attempt the update."],
    expectedBehavior: "The documented input should be accepted.",
    actualResult: "The initial call returned BAD_REQUEST.",
    recoveryAttempts: ["Inspect the schema and retry."],
    userImpact: "The edit required extra tool calls.",
    technicalContext: "The update-props input shape was ambiguous.",
    acceptanceCriteria: ["The documented input succeeds."],
  },
};

describe("GitHub issue reports", () => {
  test("formats the complete LLM-authored workflow and agent runtime", () => {
    const body = formatIssueReport(report);

    expect(body).toContain("## User story");
    expect(body).toContain(report.report.userStory);
    expect(body).toContain("## Expected behavior");
    expect(body).toContain(report.report.actualResult);
    expect(body).toContain("- Client: Codex 1.2.3");
    expect(body).toContain("- CLI: 1.2.3");
    expect(body).toContain("- Project ID: `project-123`");
    expect(body).toContain("- Recent targeted tool: `screenshot`");
    expect(body).toContain("  - pageId: `page-456`");
    expect(body).toContain("- Recent targeted tool: `inspect-instance`");
    expect(body).toContain("  - instanceId: `instance-456`");
    expect(body).toContain("- pageId: `page-123`");
    expect(body).toContain("- instanceId: `instance-123`");
    expect(body).toContain("- Node.js: 22.14.0");
    expect(body).toContain("- Operating system: linux 6 (arm64)");
    expect(body).toContain("- Execution mode: mcp");
    expect(body).toContain("- API contract: public-api:client");
    expect(body).toContain("- Server API contract: public-api:server");
    expect(body).toContain("- Bundle contract: bundle:client");
    expect(body).toContain("## Captured failure diagnostics");
    expect(body).toContain("- Tool: `preview.start`");
    expect(body).toContain("- Error code: `PROJECT_BUNDLE_INVALID`");
    expect(body).toContain("- Error type: AssetRevisionConflictError");
    expect(body).toContain("- Request ID: `abc123-LIS`");
    expect(body).toContain("- HTTP status: `504`");
    expect(body).toContain("- Duration: `17000ms`");
    expect(body).toContain("stale=styles");
    expect(body).toContain("version matches session=false");
    expect(body).toContain(
      '- `assets.0.type`: `invalid_value` (one of "font"|"image"|"video"|"file")'
    );
    expect(body).toContain("- Model: gpt-5.6-sol");
    expect(body).toContain("- Reasoning effort: medium");
    expect(body).toContain(
      "<!-- webstudio-issue-report:update-props-input-contract -->"
    );
  });

  test("keeps version fields visible for reports from legacy clients", () => {
    const legacyReport = structuredClone(report);
    delete legacyReport.runtime;

    const body = formatIssueReport(legacyReport);

    expect(body).toContain("## Technical runtime");
    expect(body).toContain("- CLI: unknown");
    expect(body).toContain("- Node.js: unknown");
  });

  test("formats response and browser diagnostics without raw data", () => {
    const responseReport = structuredClone(report);
    if (responseReport.runtime?.recentFailure === undefined) {
      throw new Error("Expected failure diagnostics");
    }
    responseReport.runtime.recentFailure.response = {
      format: "json",
      envelope: "result",
      batchSize: 1,
    };
    responseReport.runtime.recentFailure.browser = {
      exitSignal: "SIGABRT",
      attempts: [{ browser: "chromium", source: "path" }],
    };
    responseReport.runtime.recentFailure.previewInstall = {
      processErrorCode: "ETIMEDOUT",
      terminated: true,
    };

    const body = formatIssueReport(responseReport);
    expect(body).toContain("Response: json; envelope=result; batch size=1");
    expect(body).toContain("Browser exit: SIGABRT");
    expect(body).toContain("Browser attempts: chromium (path)");
    expect(body).toContain("Preview install: ETIMEDOUT; process terminated");

    responseReport.runtime.recentFailure.previewInstall = {};
    expect(formatIssueReport(responseReport)).not.toContain("Preview install:");
  });

  test("discovers the repository installation before creating its token", async () => {
    const requests: Array<{ url: URL; init?: RequestInit }> = [];
    const request: typeof fetch = async (input, init) => {
      const url = new URL(input.toString());
      requests.push({ url, init });
      if (url.pathname === "/repos/webstudio-is/webstudio/installation") {
        return Response.json({ id: 123 });
      }
      return Response.json({ token: "installation-token" });
    };

    await expect(
      createGitHubInstallationToken({
        appId: "app-id",
        privateKey: "private-key",
        createJwt: () => "app-jwt",
        request,
      })
    ).resolves.toBe("installation-token");
    expect(requests.map(({ url }) => url.pathname)).toEqual([
      "/repos/webstudio-is/webstudio/installation",
      "/app/installations/123/access_tokens",
    ]);
    expect(requests[0]?.init?.headers).toMatchObject({
      Authorization: "Bearer app-jwt",
    });
  });

  test("returns an existing issue with the same deduplication key", async () => {
    const requests: URL[] = [];
    const request: typeof fetch = async (input) => {
      requests.push(new URL(input.toString()));
      return Response.json({
        items: [
          {
            number: 6020,
            html_url: "https://github.com/webstudio-is/webstudio/issues/6020",
          },
        ],
      });
    };

    await expect(
      publishIssueReport(report, {
        getInstallationToken: async () => "installation-token",
        request,
      })
    ).resolves.toEqual({
      status: "existing",
      issueNumber: 6020,
      issueUrl: "https://github.com/webstudio-is/webstudio/issues/6020",
    });
    expect(requests).toHaveLength(1);
    expect(requests[0]?.searchParams.get("q")).toContain(
      "webstudio-issue-report:update-props-input-contract"
    );
  });

  test("creates a new issue using only the authored report", async () => {
    const requests: Array<{ url: URL; init?: RequestInit }> = [];
    const request: typeof fetch = async (input, init) => {
      const url = new URL(input.toString());
      requests.push({ url, init });
      if (url.pathname === "/search/issues") {
        return Response.json({ items: [] });
      }
      return Response.json({
        number: 6021,
        html_url: "https://github.com/webstudio-is/webstudio/issues/6021",
      });
    };

    await expect(
      publishIssueReport(report, {
        getInstallationToken: async () => "installation-token",
        request,
      })
    ).resolves.toEqual({
      status: "created",
      issueNumber: 6021,
      issueUrl: "https://github.com/webstudio-is/webstudio/issues/6021",
    });
    const createRequest = requests[1];
    expect(createRequest?.url.pathname).toBe(
      "/repos/webstudio-is/webstudio/issues"
    );
    expect(JSON.parse(String(createRequest?.init?.body))).toEqual({
      title: report.title,
      body: formatIssueReport(report),
    });
  });
});