Skip to content

The open-source app everyone uses to manage agents at work

feat(search): add bulk extract endpoint (#9507)

Dottaae7790861811
diff --git a/doc/SPEC-implementation.md b/doc/SPEC-implementation.md
index b705aa9da4..fabb879125 100644
--- a/doc/SPEC-implementation.md
+++ b/doc/SPEC-implementation.md
@@ -924,6 +924,10 @@ The current app also exposes V1-supporting surfaces for:
- issue thread interactions (`suggest_tasks`, `ask_user_questions`, `request_confirmation`)
- issue approvals, issue references/search, labels, read state, inbox/archive state, and work products
+- company search through `GET /companies/:companyId/search` plus agent-oriented bulk extraction through
+ `GET /companies/:companyId/search/extract`; extraction accepts a server-escaped literal `contains`, optional
+ server-owned URL expansion, issue/comment/document scopes, status/date filters, issue-level pagination, and
+ explicit issue/match truncation flags
- execution workspaces, project workspaces, workspace runtime services, and workspace operations
- task watchdog configuration and reusable watchdog issue orchestration for explicitly watched issue subtrees
- routines and scheduled/API/webhook triggers
diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts
index caefc73521..0802e3e238 100644
--- a/packages/shared/src/index.ts
+++ b/packages/shared/src/index.ts
@@ -658,6 +658,12 @@ export type {
ProjectManagedByPlugin,
ProjectWorkspace,
CompanySearchCountType,
+ CompanySearchExtractIssueResult,
+ CompanySearchExtractKind,
+ CompanySearchExtractMatch,
+ CompanySearchExtractResponse,
+ CompanySearchExtractScope,
+ CompanySearchExtractSourceRef,
CompanySearchFilterOptionCounts,
CompanySearchHighlight,
CompanySearchArtifactSummary,
@@ -1209,7 +1215,13 @@ export type {
QuotaWindow,
ProviderQuotaResult,
} from "./types/index.js";
-export { COMPANY_SEARCH_SCOPES, COMPANY_SEARCH_SORTS, COMPANY_SEARCH_UPDATED_WITHIN_OPTIONS } from "./types/index.js";
+export {
+ COMPANY_SEARCH_EXTRACT_KINDS,
+ COMPANY_SEARCH_EXTRACT_SCOPES,
+ COMPANY_SEARCH_SCOPES,
+ COMPANY_SEARCH_SORTS,
+ COMPANY_SEARCH_UPDATED_WITHIN_OPTIONS,
+} from "./types/index.js";
export {
ISSUE_REFERENCE_IDENTIFIER_RE,
buildIssueReferenceHref,
@@ -1466,12 +1478,18 @@ export {
type CreateDocumentAnnotationComment,
type CreateDocumentAnnotationThread,
type UpdateDocumentAnnotationThread,
+ companySearchExtractQuerySchema,
companySearchQuerySchema,
+ COMPANY_SEARCH_EXTRACT_DEFAULT_LIMIT,
+ COMPANY_SEARCH_EXTRACT_MAX_LIMIT,
+ COMPANY_SEARCH_EXTRACT_MAX_MATCHES_PER_ISSUE,
+ COMPANY_SEARCH_EXTRACT_MAX_OFFSET,
COMPANY_SEARCH_DEFAULT_LIMIT,
COMPANY_SEARCH_MAX_LIMIT,
COMPANY_SEARCH_MAX_OFFSET,
COMPANY_SEARCH_MAX_QUERY_LENGTH,
COMPANY_SEARCH_MAX_TOKENS,
+ type CompanySearchExtractQuery,
type CompanySearchQuery,
createIssueSchema,
createIssueInputSchema,
diff --git a/packages/shared/src/types/index.ts b/packages/shared/src/types/index.ts
index 8506983117..74537a5a33 100644
--- a/packages/shared/src/types/index.ts
+++ b/packages/shared/src/types/index.ts
@@ -251,6 +251,12 @@ export type {
export type { Project, ProjectBudgetSummary, ProjectCodebase, ProjectCodebaseOrigin, ProjectGoalRef, ProjectManagedByPlugin, ProjectWorkspace } from "./project.js";
export type {
CompanySearchCountType,
+ CompanySearchExtractIssueResult,
+ CompanySearchExtractKind,
+ CompanySearchExtractMatch,
+ CompanySearchExtractResponse,
+ CompanySearchExtractScope,
+ CompanySearchExtractSourceRef,
CompanySearchFilterOptionCounts,
CompanySearchHighlight,
CompanySearchArtifactSummary,
@@ -266,7 +272,13 @@ export type {
CompanySearchZeroResults,
CompanySearchZeroResultsLoosenSuggestion,
} from "./search.js";
-export { COMPANY_SEARCH_SCOPES, COMPANY_SEARCH_SORTS, COMPANY_SEARCH_UPDATED_WITHIN_OPTIONS } from "./search.js";
+export {
+ COMPANY_SEARCH_EXTRACT_KINDS,
+ COMPANY_SEARCH_EXTRACT_SCOPES,
+ COMPANY_SEARCH_SCOPES,
+ COMPANY_SEARCH_SORTS,
+ COMPANY_SEARCH_UPDATED_WITHIN_OPTIONS,
+} from "./search.js";
export type {
ExecutionWorkspace,
ExecutionWorkspaceSummary,
diff --git a/packages/shared/src/types/search.ts b/packages/shared/src/types/search.ts
index 7ed7790424..1b812bba5d 100644
--- a/packages/shared/src/types/search.ts
+++ b/packages/shared/src/types/search.ts
@@ -108,3 +108,45 @@ export interface CompanySearchResponse {
zeroResults: CompanySearchZeroResults | null;
hasMore: boolean;
}
+
+export const COMPANY_SEARCH_EXTRACT_SCOPES = ["all", "issues", "comments", "documents"] as const;
+export type CompanySearchExtractScope = (typeof COMPANY_SEARCH_EXTRACT_SCOPES)[number];
+
+export const COMPANY_SEARCH_EXTRACT_KINDS = ["literal", "url"] as const;
+export type CompanySearchExtractKind = (typeof COMPANY_SEARCH_EXTRACT_KINDS)[number];
+
+export type CompanySearchExtractSourceRef =
+ | { type: "issue"; issueId: string }
+ | { type: "comment"; commentId: string }
+ | { type: "document"; documentId: string; documentKey: string };
+
+export interface CompanySearchExtractMatch {
+ value: string;
+ field: "title" | "description" | "comment" | "document_title" | "document_body";
+ label: string;
+ excerpt: string;
+ excerptTruncated: boolean;
+ source: CompanySearchExtractSourceRef;
+}
+
+export interface CompanySearchExtractIssueResult {
+ issueId: string;
+ identifier: string | null;
+ title: string;
+ status: IssueStatus;
+ assigneeAgentId: string | null;
+ updatedAt: string;
+ matches: CompanySearchExtractMatch[];
+ matchesTruncated: boolean;
+}
+
+export interface CompanySearchExtractResponse {
+ contains: string;
+ kind: CompanySearchExtractKind;
+ scope: CompanySearchExtractScope;
+ limit: number;
+ offset: number;
+ results: CompanySearchExtractIssueResult[];
+ hasMore: boolean;
+ truncated: boolean;
+}
diff --git a/packages/shared/src/validators/index.ts b/packages/shared/src/validators/index.ts
index d3ccb2aa70..2fa890e13e 100644
--- a/packages/shared/src/validators/index.ts
+++ b/packages/shared/src/validators/index.ts
@@ -420,12 +420,18 @@ export {
} from "./issue.js";
export {
+ COMPANY_SEARCH_EXTRACT_DEFAULT_LIMIT,
+ COMPANY_SEARCH_EXTRACT_MAX_LIMIT,
+ COMPANY_SEARCH_EXTRACT_MAX_MATCHES_PER_ISSUE,
+ COMPANY_SEARCH_EXTRACT_MAX_OFFSET,
COMPANY_SEARCH_DEFAULT_LIMIT,
COMPANY_SEARCH_MAX_LIMIT,
COMPANY_SEARCH_MAX_OFFSET,
COMPANY_SEARCH_MAX_QUERY_LENGTH,
COMPANY_SEARCH_MAX_TOKENS,
companySearchQuerySchema,
+ companySearchExtractQuerySchema,
+ type CompanySearchExtractQuery,
type CompanySearchQuery,
} from "./search.js";
diff --git a/packages/shared/src/validators/search.ts b/packages/shared/src/validators/search.ts
index 5c995dd823..b54922c1e6 100644
--- a/packages/shared/src/validators/search.ts
+++ b/packages/shared/src/validators/search.ts
@@ -1,13 +1,22 @@
import { z } from "zod";
import { ISSUE_PRIORITIES, ISSUE_STATUSES } from "../constants.js";
import { isUuidLike } from "../agent-url-key.js";
-import { COMPANY_SEARCH_SCOPES, COMPANY_SEARCH_SORTS } from "../types/search.js";
+import {
+ COMPANY_SEARCH_EXTRACT_KINDS,
+ COMPANY_SEARCH_EXTRACT_SCOPES,
+ COMPANY_SEARCH_SCOPES,
+ COMPANY_SEARCH_SORTS,
+} from "../types/search.js";
export const COMPANY_SEARCH_MAX_QUERY_LENGTH = 200;
export const COMPANY_SEARCH_MAX_TOKENS = 8;
export const COMPANY_SEARCH_DEFAULT_LIMIT = 20;
export const COMPANY_SEARCH_MAX_LIMIT = 50;
export const COMPANY_SEARCH_MAX_OFFSET = 200;
+export const COMPANY_SEARCH_EXTRACT_DEFAULT_LIMIT = 100;
+export const COMPANY_SEARCH_EXTRACT_MAX_LIMIT = 200;
+export const COMPANY_SEARCH_EXTRACT_MAX_OFFSET = 5_000;
+export const COMPANY_SEARCH_EXTRACT_MAX_MATCHES_PER_ISSUE = 20;
const UPDATED_WITHIN_RE = /^[1-9]\d{0,2}(h|d|w|m)$/;
@@ -180,3 +189,71 @@ export const companySearchQuerySchema = z.object({
});
export type CompanySearchQuery = z.infer<typeof companySearchQuerySchema>;
+
+export const companySearchExtractQuerySchema = z.object({
+ contains: z.unknown().transform((value, ctx) => {
+ const normalized = parseOptionalString(value, ctx, "contains");
+ if (!normalized || normalized.length < 2) {
+ ctx.addIssue({ code: z.ZodIssueCode.custom, message: "contains must be at least 2 characters" });
+ return "";
+ }
+ if (normalized.length > COMPANY_SEARCH_MAX_QUERY_LENGTH) {
+ ctx.addIssue({
+ code: z.ZodIssueCode.custom,
+ message: `contains must be at most ${COMPANY_SEARCH_MAX_QUERY_LENGTH} characters`,
+ });
+ }
+ return normalized.slice(0, COMPANY_SEARCH_MAX_QUERY_LENGTH);
+ }),
+ kind: z.unknown()
+ .optional()
+ .transform((value, ctx) => {
+ const normalized = parseOptionalString(value, ctx, "kind") ?? "literal";
+ if (!(COMPANY_SEARCH_EXTRACT_KINDS as readonly string[]).includes(normalized)) {
+ ctx.addIssue({ code: z.ZodIssueCode.custom, message: "kind must be literal or url" });
+ return "literal";
+ }
+ return normalized as (typeof COMPANY_SEARCH_EXTRACT_KINDS)[number];
+ }),
+ scope: z.unknown()
+ .optional()
+ .transform((value, ctx) => {
+ const normalized = parseOptionalString(value, ctx, "scope") ?? "all";
+ if (!(COMPANY_SEARCH_EXTRACT_SCOPES as readonly string[]).includes(normalized)) {
+ ctx.addIssue({ code: z.ZodIssueCode.custom, message: "scope must be all, issues, comments, or documents" });
+ return "all";
+ }
+ return normalized as (typeof COMPANY_SEARCH_EXTRACT_SCOPES)[number];
+ }),
+ limit: z.unknown()
+ .optional()
+ .transform((value, ctx) => parseIntegerQuery(
+ value,
+ ctx,
+ "limit",
+ COMPANY_SEARCH_EXTRACT_DEFAULT_LIMIT,
+ 1,
+ COMPANY_SEARCH_EXTRACT_MAX_LIMIT,
+ )),
+ offset: z.unknown()
+ .optional()
+ .transform((value, ctx) => parseIntegerQuery(value, ctx, "offset", 0, 0, COMPANY_SEARCH_EXTRACT_MAX_OFFSET)),
+ status: z.unknown()
+ .optional()
+ .transform((value, ctx) => parseEnumList(value, ctx, "status", ISSUE_STATUSES)),
+ updatedWithin: z.unknown()
+ .optional()
+ .transform((value, ctx) => parseUpdatedWithin(value, ctx)),
+ updatedAfter: z.unknown()
+ .optional()
+ .transform((value, ctx) => parseUpdatedAfter(value, ctx)),
+}).superRefine((value, ctx) => {
+ if (value.updatedWithin && value.updatedAfter) {
+ ctx.addIssue({
+ code: z.ZodIssueCode.custom,
+ message: "updatedWithin and updatedAfter cannot be used together",
+ });
+ }
+});
+
+export type CompanySearchExtractQuery = z.infer<typeof companySearchExtractQuerySchema>;
diff --git a/server/src/__tests__/company-search-extract-routes.test.ts b/server/src/__tests__/company-search-extract-routes.test.ts
new file mode 100644
index 0000000000..67a7144c96
--- /dev/null
+++ b/server/src/__tests__/company-search-extract-routes.test.ts
@@ -0,0 +1,96 @@
+import express from "express";
+import request from "supertest";
+import { describe, expect, it, vi } from "vitest";
+import type {
+ CompanySearchExtractQuery,
+ CompanySearchExtractResponse,
+ CompanySearchQuery,
+ CompanySearchResponse,
+} from "@paperclipai/shared";
+import { issueRoutes } from "../routes/issues.js";
+import { createCompanySearchRateLimiter } from "../services/company-search-rate-limit.js";
+
+function extractResponse(query: CompanySearchExtractQuery): CompanySearchExtractResponse {
+ return {
+ contains: query.contains,
+ kind: query.kind,
+ scope: query.scope,
+ limit: query.limit,
+ offset: query.offset,
+ results: [],
+ hasMore: false,
+ truncated: false,
+ };
+}
+
+function unusedSearch(_companyId: string, _query: CompanySearchQuery): Promise<CompanySearchResponse> {
+ throw new Error("interactive search should not be called");
+}
+
+function createApp(companyIds: string[], extract: (companyId: string, query: CompanySearchExtractQuery) => Promise<CompanySearchExtractResponse>) {
+ const app = express();
+ app.use((req, _res, next) => {
+ req.actor = {
+ type: "board",
+ userId: "user-1",
+ companyIds,
+ source: "session",
+ isInstanceAdmin: true,
+ };
+ next();
+ });
+ app.use("/api", issueRoutes({} as never, {} as never, {
+ searchService: { search: unusedSearch, extract },
+ searchRateLimiter: createCompanySearchRateLimiter({
+ maxRequests: 1,
+ windowMs: 60_000,
+ now: () => 1_000,
+ }),
+ }));
+ return app;
+}
+
+describe("company extract-search route", () => {
+ it("parses the extraction query and invokes the service", async () => {
+ const extract = vi.fn(async (_companyId: string, query: CompanySearchExtractQuery) => extractResponse(query));
+ const app = createApp(["company-1"], extract);
+
+ const response = await request(app)
+ .get("/api/companies/company-1/search/extract")
+ .query({ contains: "github.com/example/repo/pull", kind: "url", scope: "comments", limit: "200" })
+ .expect(200);
+
+ expect(extract).toHaveBeenCalledWith("company-1", expect.objectContaining({
+ contains: "github.com/example/repo/pull",
+ kind: "url",
+ scope: "comments",
+ limit: 200,
+ }));
+ expect(response.body).toMatchObject({ kind: "url", scope: "comments", truncated: false });
+ });
+
+ it("denies cross-company access before invoking the service", async () => {
+ const extract = vi.fn(async (_companyId: string, query: CompanySearchExtractQuery) => extractResponse(query));
+ const app = createApp(["company-1"], extract);
+
+ await request(app)
+ .get("/api/companies/company-2/search/extract?contains=needle")
+ .expect(403);
+
+ expect(extract).not.toHaveBeenCalled();
+ });
+
+ it("shares the company-search rate limiter", async () => {
+ const extract = vi.fn(async (_companyId: string, query: CompanySearchExtractQuery) => extractResponse(query));
+ const app = createApp(["company-1"], extract);
+
+ await request(app).get("/api/companies/company-1/search/extract?contains=needle").expect(200);
+ const limited = await request(app)
+ .get("/api/companies/company-1/search/extract?contains=needle")
+ .expect(429);
+
+ expect(extract).toHaveBeenCalledTimes(1);
+ expect(limited.body).toEqual({ error: "Search rate limit exceeded", retryAfterSeconds: 60 });
+ expect(limited.headers["retry-after"]).toBe("60");
+ });
+});
diff --git a/server/src/__tests__/company-search-extract-service.test.ts b/server/src/__tests__/company-search-extract-service.test.ts
new file mode 100644
index 0000000000..e03b7c6de6
--- /dev/null
+++ b/server/src/__tests__/company-search-extract-service.test.ts
@@ -0,0 +1,244 @@
+import { randomUUID } from "node:crypto";
+import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";
+import {
+ companies,
+ createDb,
+ documents,
+ issueComments,
+ issueDocuments,
+ issues,
+} from "@paperclipai/db";
+import {
+ COMPANY_SEARCH_EXTRACT_MAX_MATCHES_PER_ISSUE,
+ companySearchExtractQuerySchema,
+} from "@paperclipai/shared";
+import {
+ getEmbeddedPostgresTestSupport,
+ startEmbeddedPostgresTestDatabase,
+} from "./helpers/embedded-postgres.js";
+import { companySearchExtractService } from "../services/company-search-extract.js";
+
+const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
+const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
+
+if (!embeddedPostgresSupport.supported) {
+ console.warn(
+ `Skipping embedded Postgres extract-search tests on this host: ${embeddedPostgresSupport.reason ?? "unsupported environment"}`,
+ );
+}
+
+describe("extract-search query validation", () => {
+ it("accepts supported extraction filters and rejects unsafe or ambiguous input", () => {
+ const parsed = companySearchExtractQuerySchema.parse({
+ contains: "github.com/paperclipai/paperclip/pull",
+ kind: "url",
+ scope: "comments",
+ status: "in_progress,in_review",
+ limit: "200",
+ offset: "5000",
+ updatedWithin: "30d",
+ });
+
+ expect(parsed.kind).toBe("url");
+ expect(parsed.scope).toBe("comments");
+ expect(parsed.status).toEqual(["in_progress", "in_review"]);
+ expect(() => companySearchExtractQuerySchema.parse({ contains: ".*", kind: "regex" })).toThrow();
+ expect(() => companySearchExtractQuerySchema.parse({ contains: "x" })).toThrow();
+ expect(() => companySearchExtractQuerySchema.parse({ contains: "needle", limit: "201" })).toThrow();
+ expect(() => companySearchExtractQuerySchema.parse({
+ contains: "needle",
+ updatedWithin: "30d",
+ updatedAfter: "2026-01-01T00:00:00.000Z",
+ })).toThrow();
+ });
+});
+
+describeEmbeddedPostgres("companySearchExtractService", () => {
+ let db!: ReturnType<typeof createDb>;
+ let svc!: ReturnType<typeof companySearchExtractService>;
+ let tempDb: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;
+
+ beforeAll(async () => {
+ tempDb = await startEmbeddedPostgresTestDatabase("paperclip-company-search-extract-");
+ db = createDb(tempDb.connectionString);
+ svc = companySearchExtractService(db);
+ }, 20_000);
+
+ afterEach(async () => {
+ await db.delete(issueDocuments);
+ await db.delete(documents);
+ await db.delete(issueComments);
+ await db.delete(issues);
+ await db.delete(companies);
+ });
+
+ afterAll(async () => {
+ await tempDb?.cleanup();
+ });
+
+ async function createCompany(name = "Paperclip") {
+ const companyId = randomUUID();
+ await db.insert(companies).values({
+ id: companyId,
+ name,
+ issuePrefix: `E${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`,
+ requireBoardApprovalForNewAgents: false,
+ });
+ return companyId;
+ }
+
+ async function createIssue(companyId: string, values: Partial<typeof issues.$inferInsert> = {}) {
+ const id = values.id ?? randomUUID();
+ await db.insert(issues).values({
+ id,
+ companyId,
+ identifier: values.identifier ?? "EXT-1",
+ title: values.title ?? "Extract target",
+ description: values.description ?? null,
+ status: values.status ?? "in_progress",
+ priority: values.priority ?? "medium",
+ ...values,
+ });
+ return id;
+ }
+
+ it("expands and deduplicates URLs across issue, comment, and document sources", async () => {
+ const companyId = await createCompany();
+ const firstUrl = "https://github.com/paperclipai/paperclip/pull/123";
+ const secondUrl = "https://github.com/paperclipai/paperclip/pull/456";
+ const thirdUrl = "https://github.com/paperclipai/paperclip/pull/789";
+ const issueId = await createIssue(companyId, {
+ description: `Primary ${firstUrl} and duplicate ${firstUrl}.`,
+ });
+ await db.insert(issueComments).values({
+ companyId,
+ issueId,
+ body: `Review ${secondUrl} and repeat ${firstUrl}`,
+ });
+ const documentId = randomUUID();
+ await db.insert(documents).values({
+ id: documentId,
+ companyId,
+ title: `PR notes ${thirdUrl}`,
+ latestBody: `Also see [the second PR](${secondUrl}).`,
+ });
+ await db.insert(issueDocuments).values({
+ companyId,
+ issueId,
+ documentId,
+ key: "plan",
+ });
+
+ const result = await svc.extract(companyId, companySearchExtractQuerySchema.parse({
+ contains: "github.com/paperclipai/paperclip/pull",
+ kind: "url",
+ }));
+
+ expect(result.results).toHaveLength(1);
+ expect(result.results[0]?.matches.map((match) => match.value)).toEqual([firstUrl, secondUrl, thirdUrl]);
+ expect(result.results[0]?.matches.map((match) => match.field)).toEqual([
+ "description",
+ "comment",
+ "document_title",
+ ]);
+ expect(result.results[0]?.matchesTruncated).toBe(false);
+ expect(result.truncated).toBe(false);
+ });
+
+ it("keeps URL sources selected by scheme-less queries", async () => {
+ const companyId = await createCompany();
+ const titleUrl = "https://github.com/paperclipai/paperclip/pull/101";
+ const descriptionUrl = "https://github.com/paperclipai/paperclip/pull/102";
+ const documentTitleUrl = "https://github.com/paperclipai/paperclip/pull/103";
+ const documentBodyUrl = "https://github.com/paperclipai/paperclip/pull/104";
+ const issueId = await createIssue(companyId, {
+ title: `Review ${titleUrl}`,
+ description: `Then merge ${descriptionUrl}`,
+ });
+ const documentId = randomUUID();
+ await db.insert(documents).values({
+ id: documentId,
+ companyId,
+ title: `Tracking ${documentTitleUrl}`,
+ latestBody: `Final follow-up ${documentBodyUrl}`,
+ });
+ await db.insert(issueDocuments).values({
+ companyId,
+ issueId,
+ documentId,
+ key: "plan",
+ });
+
+ const result = await svc.extract(companyId, companySearchExtractQuerySchema.parse({
+ contains: "github.com/paperclipai/paperclip/pull",
+ kind: "url",
+ scope: "all",
+ }));
+
+ expect(result.results[0]?.matches.map((match) => [match.field, match.value])).toEqual([
+ ["title", titleUrl],
+ ["description", descriptionUrl],
+ ["document_title", documentTitleUrl],
+ ["document_body", documentBodyUrl],
+ ]);
+ });
+
+ it("filters by issue update window and status", async () => {
+ const companyId = await createCompany();
+ const now = Date.now();
+ const recentId = await createIssue(companyId, {
+ identifier: "EXT-RECENT",
+ description: "needle",
+ status: "in_review",
+ updatedAt: new Date(now - 24 * 60 * 60 * 1000),
+ });
+ await createIssue(companyId, {
+ identifier: "EXT-OLD",
+ description: "needle",
+ status: "in_review",
+ updatedAt: new Date(now - 60 * 24 * 60 * 60 * 1000),
+ });
+ await createIssue(companyId, {
+ identifier: "EXT-DONE",
+ description: "needle",
+ status: "done",
+ updatedAt: new Date(now - 24 * 60 * 60 * 1000),
+ });
+
+ const result = await svc.extract(companyId, companySearchExtractQuerySchema.parse({
+ contains: "needle",
+ updatedWithin: "30d",
+ status: "in_review",
+ }));
+
+ expect(result.results.map((row) => row.issueId)).toEqual([recentId]);
+ });
+
+ it("caps distinct matches per issue and marks truncation explicitly", async () => {
+ const companyId = await createCompany();
+ const urls = Array.from(
+ { length: COMPANY_SEARCH_EXTRACT_MAX_MATCHES_PER_ISSUE + 1 },
+ (_, index) => `https://github.com/paperclipai/paperclip/pull/${index + 1}`,
+ );
+ await createIssue(companyId, { description: urls.join(" ") });
+
+ const result = await svc.extract(companyId, companySearchExtractQuerySchema.parse({
+ contains: "github.com/paperclipai/paperclip/pull",
+ kind: "url",
+ }));
+
+ expect(result.results[0]?.matches).toHaveLength(COMPANY_SEARCH_EXTRACT_MAX_MATCHES_PER_ISSUE);
+ expect(result.results[0]?.matchesTruncated).toBe(true);
+ expect(result.truncated).toBe(true);
+ });
+
+ it("does not return matching issues from another company", async () => {
+ const companyId = await createCompany();
+ const otherCompanyId = await createCompany("Other");
+ await createIssue(otherCompanyId, { description: "needle" });
+
+ const result = await svc.extract(companyId, companySearchExtractQuerySchema.parse({ contains: "needle" }));
+
+ expect(result.results).toEqual([]);
+ });
+});
diff --git a/server/src/routes/issues.ts b/server/src/routes/issues.ts
index 80a99b51c0..a7bf135606 100644
--- a/server/src/routes/issues.ts
+++ b/server/src/routes/issues.ts
@@ -27,6 +27,7 @@ import {
acceptIssueThreadInteractionSchema,
attachmentArtifactWorkProductMetadataSchema,
cancelIssueThreadInteractionSchema,
+ companySearchExtractQuerySchema,
companySearchQuerySchema,
createIssueAttachmentMetadataSchema,
createIssueThreadInteractionSchema,
@@ -63,6 +64,8 @@ import {
isUuidLike,
normalizeIssueIdentifier as normalizeIssueReferenceIdentifier,
type CompactIssue,
+ type CompanySearchExtractQuery,
+ type CompanySearchExtractResponse,
type CompanySearchQuery,
type CompanySearchResponse,
type ExecutionWorkspace,
@@ -251,6 +254,7 @@ type RecoveryRevalidationTrigger =
| "work_product"
| "read_projection";
type CompanySearchService = {
+ extract(companyId: string, query: CompanySearchExtractQuery): Promise<CompanySearchExtractResponse>;
search(companyId: string, query: CompanySearchQuery): Promise<CompanySearchResponse>;
};
type ActivityIssueRelationSummary = {
@@ -4531,6 +4535,40 @@ export function issueRoutes(
});
});
+ router.get("/companies/:companyId/search/extract", async (req, res) => {
+ const companyId = req.params.companyId as string;
+ assertCompanyAccess(req, companyId);
+ const companyScopeDecision = await access.decide({
+ actor: req.actor,
+ action: "company_scope:read",
+ resource: { type: "company", companyId },
+ });
+ if (!companyScopeDecision.allowed) {
+ res.status(403).json({ error: "Company search is outside this actor's authorization boundary" });
+ return;
+ }
+ const parsedQuery = companySearchExtractQuerySchema.safeParse(req.query);
+ if (!parsedQuery.success) {
+ res.status(400).json({
+ error: parsedQuery.error.issues[0]?.message ?? "Invalid extract search query",
+ });
+ return;
+ }
+ const rateLimit = searchRateLimiter.consume(companySearchRateLimitActor(req, companyId));
+ res.setHeader("X-RateLimit-Limit", String(rateLimit.limit));
+ res.setHeader("X-RateLimit-Remaining", String(rateLimit.remaining));
+ if (!rateLimit.allowed) {
+ res.setHeader("Retry-After", String(rateLimit.retryAfterSeconds));
+ res.status(429).json({
+ error: "Search rate limit exceeded",
+ retryAfterSeconds: rateLimit.retryAfterSeconds,
+ });
+ return;
+ }
+ const result = await getSearchService().extract(companyId, parsedQuery.data);
+ res.json(result);
+ });
+
router.get("/companies/:companyId/search", async (req, res) => {
const companyId = req.params.companyId as string;
assertCompanyAccess(req, companyId);
diff --git a/server/src/routes/openapi.ts b/server/src/routes/openapi.ts
index 774d53fa89..bdb8bec2b5 100644
--- a/server/src/routes/openapi.ts
+++ b/server/src/routes/openapi.ts
@@ -5142,6 +5142,7 @@ registerCurrentRoute({
for (const route of [
["get", "/api/companies/import/jobs/{jobId}", "Get company import job status"],
["get", "/api/companies/{companyId}/search", "Search company data"],
+ ["get", "/api/companies/{companyId}/search/extract", "Extract company search matches"],
["get", "/api/companies/{companyId}/issues/count", "Count issues in a company"],
] as const) {
registerCurrentRoute({
diff --git a/server/src/services/company-search-extract.ts b/server/src/services/company-search-extract.ts
new file mode 100644
index 0000000000..6462b33445
--- /dev/null
+++ b/server/src/services/company-search-extract.ts
@@ -0,0 +1,349 @@
+import { and, asc, desc, eq, gte, inArray, isNull, or, sql } from "drizzle-orm";
+import type { SQL, SQLWrapper } from "drizzle-orm";
+import type { Db } from "@paperclipai/db";
+import { documents, issueComments, issueDocuments, issues } from "@paperclipai/db";
+import {
+ COMPANY_SEARCH_EXTRACT_MAX_MATCHES_PER_ISSUE,
+ type CompanySearchExtractIssueResult,
+ type CompanySearchExtractMatch,
+ type CompanySearchExtractQuery,
+ type CompanySearchExtractResponse,
+ type CompanySearchExtractSourceRef,
+} from "@paperclipai/shared";
+import { visibleIssueCondition } from "./issue-visibility.js";
+
+const EXCERPT_MAX_CHARS = 180;
+const URL_PATTERN = /(?:https?:\/\/|www\.)[^\s<>"'`]+|(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z]{2,}\/[^\s<>"'`]+/giu;
+
+type ExtractSource = {
+ issueId: string;
+ field: CompanySearchExtractMatch["field"];
+ label: string;
+ text: string;
+ source: CompanySearchExtractSourceRef;
+};
+
+function escapeLikePattern(value: string): string {
+ return value.replace(/[\\%_]/g, "\\$&");
+}
+
+function escapeRegexPattern(value: string): string {
+ return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+
+function urlContainsPattern(contains: string): string {
+ const literal = escapeRegexPattern(contains);
+ return `(?=[^[:space:]<>"']*${literal})(?:(?:https?://|www\\.)[^[:space:]<>"']+|(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\\.)+[a-z]{2,}/[^[:space:]<>"']+)`;
+}
+
+function contentMatch(
+ column: SQLWrapper,
+ query: CompanySearchExtractQuery,
+ containsPattern: string,
+ urlPattern: string,
+): SQL {
+ return query.kind === "url"
+ ? sql`${column} ~* ${urlPattern}`
+ : sql`${column} ILIKE ${containsPattern}`;
+}
+
+function updatedWithinStart(value: string | undefined, now = new Date()): Date | null {
+ if (!value) return null;
+ const match = /^(\d+)(h|d|w|m)$/.exec(value);
+ if (!match) return null;
+ const amount = Number.parseInt(match[1]!, 10);
+ const unit = match[2];
+ const hours = unit === "h" ? amount : unit === "d" ? amount * 24 : unit === "w" ? amount * 24 * 7 : amount * 24 * 30;
+ return new Date(now.getTime() - hours * 60 * 60 * 1000);
+}
+
+function scopeIncludes(scope: CompanySearchExtractQuery["scope"], source: Exclude<CompanySearchExtractQuery["scope"], "all">) {
+ return scope === "all" || scope === source;
+}
+
+function trimUrlToken(value: string): string {
+ let result = value.replace(/[.,;:!?]+$/g, "");
+ const pairs: Array<[string, string]> = [["(", ")"], ["[", "]"], ["{", "}"]];
+ for (const [open, close] of pairs) {
+ while (result.endsWith(close) && result.split(close).length > result.split(open).length) {
+ result = result.slice(0, -1);
+ }
+ }
+ return result;
+}
+
+function literalOccurrences(text: string, contains: string) {
+ const lowerText = text.toLowerCase();
+ const lowerContains = contains.toLowerCase();
+ const matches: Array<{ value: string; start: number }> = [];
+ let start = 0;
+ while (start <= text.length - contains.length) {
+ const index = lowerText.indexOf(lowerContains, start);
+ if (index < 0) break;
+ matches.push({ value: text.slice(index, index + contains.length), start: index });
+ start = index + Math.max(contains.length, 1);
+ }
+ return matches;
+}
+
+function urlOccurrences(text: string, contains: string) {
+ const lowerContains = contains.toLowerCase();
+ const matches: Array<{ value: string; start: number }> = [];
+ for (const match of text.matchAll(URL_PATTERN)) {
+ const raw = match[0];
+ const value = trimUrlToken(raw);
+ if (!value.toLowerCase().includes(lowerContains)) continue;
+ matches.push({ value, start: match.index ?? 0 });
+ }
+ return matches;
+}
+
+function sourceOccurrences(text: string, query: CompanySearchExtractQuery) {
+ return query.kind === "url"
+ ? urlOccurrences(text, query.contains)
+ : literalOccurrences(text, query.contains);
+}
+
+function excerpt(text: string, start: number, length: number) {
+ if (text.length <= EXCERPT_MAX_CHARS) {
+ return { value: text, truncated: false };
+ }
+ const context = Math.max(0, Math.floor((EXCERPT_MAX_CHARS - length) / 2));
+ let excerptStart = Math.max(0, start - context);
+ let excerptEnd = Math.min(text.length, excerptStart + EXCERPT_MAX_CHARS);
+ excerptStart = Math.max(0, excerptEnd - EXCERPT_MAX_CHARS);
+ const prefix = excerptStart > 0 ? "…" : "";
+ const suffix = excerptEnd < text.length ? "…" : "";
+ return {
+ value: `${prefix}${text.slice(excerptStart, excerptEnd).replace(/\s+/g, " ").trim()}${suffix}`,
+ truncated: true,
+ };
+}
+
+function extractMatches(sources: ExtractSource[], query: CompanySearchExtractQuery) {
+ const matches: CompanySearchExtractMatch[] = [];
+ const seen = new Set<string>();
+ let matchesTruncated = false;
+
+ for (const source of sources) {
+ const occurrences = sourceOccurrences(source.text, query);
+ for (const occurrence of occurrences) {
+ const dedupeKey = occurrence.value.toLowerCase();
+ if (seen.has(dedupeKey)) continue;
+ seen.add(dedupeKey);
+ if (matches.length >= COMPANY_SEARCH_EXTRACT_MAX_MATCHES_PER_ISSUE) {
+ matchesTruncated = true;
+ continue;
+ }
+ const matchExcerpt = excerpt(source.text, occurrence.start, occurrence.value.length);
+ matches.push({
+ value: occurrence.value,
+ field: source.field,
+ label: source.label,
+ excerpt: matchExcerpt.value,
+ excerptTruncated: matchExcerpt.truncated,
+ source: source.source,
+ });
+ }
+ }
+
+ return { matches, matchesTruncated };
+}
+
+export function companySearchExtractService(db: Db) {
+ return {
+ extract: async (companyId: string, query: CompanySearchExtractQuery): Promise<CompanySearchExtractResponse> => {
+ const containsPattern = `%${escapeLikePattern(query.contains)}%`;
+ const urlPattern = urlContainsPattern(query.contains);
+ const scopeConditions: SQL[] = [];
+ if (scopeIncludes(query.scope, "issues")) {
+ scopeConditions.push(or(
+ contentMatch(issues.title, query, containsPattern, urlPattern),
+ contentMatch(issues.description, query, containsPattern, urlPattern),
+ )!);
+ }
+ if (scopeIncludes(query.scope, "comments")) {
+ scopeConditions.push(sql`EXISTS (
+ SELECT 1
+ FROM issue_comments extract_comments
+ WHERE extract_comments.company_id = ${companyId}
+ AND extract_comments.issue_id = ${issues.id}
+ AND extract_comments.deleted_at IS NULL
+ AND ${query.kind === "url"
+ ? sql`extract_comments.body ~* ${urlPattern}`
+ : sql`extract_comments.body ILIKE ${containsPattern}`}
+ )`);
+ }
+ if (scopeIncludes(query.scope, "documents")) {
+ scopeConditions.push(sql`EXISTS (
+ SELECT 1
+ FROM issue_documents extract_issue_documents
+ INNER JOIN documents extract_documents
+ ON extract_documents.id = extract_issue_documents.document_id
+ AND extract_documents.company_id = extract_issue_documents.company_id
+ WHERE extract_issue_documents.company_id = ${companyId}
+ AND extract_issue_documents.issue_id = ${issues.id}
+ AND (
+ ${query.kind === "url"
+ ? sql`extract_documents.title ~* ${urlPattern}`
+ : sql`extract_documents.title ILIKE ${containsPattern}`}
+ OR ${query.kind === "url"
+ ? sql`extract_documents.latest_body ~* ${urlPattern}`
+ : sql`extract_documents.latest_body ILIKE ${containsPattern}`}
+ )
+ )`);
+ }
+
+ const conditions: SQL[] = [
+ eq(issues.companyId, companyId),
+ visibleIssueCondition(),
+ or(...scopeConditions)!,
+ ];
+ if (query.status.length > 0) conditions.push(inArray(issues.status, query.status));
+ const updatedWithin = updatedWithinStart(query.updatedWithin);
+ if (updatedWithin) conditions.push(gte(issues.updatedAt, updatedWithin));
+ if (query.updatedAfter) conditions.push(gte(issues.updatedAt, new Date(query.updatedAfter)));
+
+ const candidateRows = await db
+ .select({
+ id: issues.id,
+ identifier: issues.identifier,
+ title: issues.title,
+ description: issues.description,
+ status: issues.status,
+ assigneeAgentId: issues.assigneeAgentId,
+ updatedAt: issues.updatedAt,
+ })
+ .from(issues)
+ .where(and(...conditions))
+ .orderBy(desc(issues.updatedAt), desc(issues.id))
+ .limit(query.limit + 1)
+ .offset(query.offset);
+
+ const hasMore = candidateRows.length > query.limit;
+ const pageRows = candidateRows.slice(0, query.limit);
+ const issueIds = pageRows.map((row) => row.id);
+ const sourcesByIssue = new Map<string, ExtractSource[]>();
+ const addSource = (source: ExtractSource) => {
+ const sources = sourcesByIssue.get(source.issueId) ?? [];
+ sources.push(source);
+ sourcesByIssue.set(source.issueId, sources);
+ };
+
+ if (scopeIncludes(query.scope, "issues")) {
+ for (const row of pageRows) {
+ if (sourceOccurrences(row.title, query).length > 0) {
+ addSource({
+ issueId: row.id,
+ field: "title",
+ label: "Issue title",
+ text: row.title,
+ source: { type: "issue", issueId: row.id },
+ });
+ }
+ if (row.description && sourceOccurrences(row.description, query).length > 0) {
+ addSource({
+ issueId: row.id,
+ field: "description",
+ label: "Issue description",
+ text: row.description,
+ source: { type: "issue", issueId: row.id },
+ });
+ }
+ }
+ }
+
+ if (issueIds.length > 0 && scopeIncludes(query.scope, "comments")) {
+ const commentRows = await db
+ .select({ id: issueComments.id, issueId: issueComments.issueId, body: issueComments.body })
+ .from(issueComments)
+ .where(and(
+ eq(issueComments.companyId, companyId),
+ inArray(issueComments.issueId, issueIds),
+ isNull(issueComments.deletedAt),
+ contentMatch(issueComments.body, query, containsPattern, urlPattern),
+ ))
+ .orderBy(asc(issueComments.createdAt), asc(issueComments.id));
+ for (const row of commentRows) {
+ addSource({
+ issueId: row.issueId,
+ field: "comment",
+ label: "Comment",
+ text: row.body,
+ source: { type: "comment", commentId: row.id },
+ });
+ }
+ }
+
+ if (issueIds.length > 0 && scopeIncludes(query.scope, "documents")) {
+ const documentRows = await db
+ .select({
+ id: documents.id,
+ issueId: issueDocuments.issueId,
+ key: issueDocuments.key,
+ title: documents.title,
+ body: documents.latestBody,
+ })
+ .from(issueDocuments)
+ .innerJoin(documents, and(
+ eq(documents.id, issueDocuments.documentId),
+ eq(documents.companyId, issueDocuments.companyId),
+ ))
+ .where(and(
+ eq(issueDocuments.companyId, companyId),
+ inArray(issueDocuments.issueId, issueIds),
+ or(
+ contentMatch(documents.title, query, containsPattern, urlPattern),
+ contentMatch(documents.latestBody, query, containsPattern, urlPattern),
+ ),
+ ))
+ .orderBy(asc(issueDocuments.key), asc(documents.id));
+ for (const row of documentRows) {
+ const source = { type: "document" as const, documentId: row.id, documentKey: row.key };
+ if (row.title && sourceOccurrences(row.title, query).length > 0) {
+ addSource({
+ issueId: row.issueId,
+ field: "document_title",
+ label: `Document title (${row.key})`,
+ text: row.title,
+ source,
+ });
+ }
+ if (sourceOccurrences(row.body, query).length > 0) {
+ addSource({
+ issueId: row.issueId,
+ field: "document_body",
+ label: `Document (${row.key})`,
+ text: row.body,
+ source,
+ });
+ }
+ }
+ }
+
+ const results: CompanySearchExtractIssueResult[] = pageRows.map((row) => {
+ const extracted = extractMatches(sourcesByIssue.get(row.id) ?? [], query);
+ return {
+ issueId: row.id,
+ identifier: row.identifier,
+ title: row.title,
+ status: row.status as CompanySearchExtractIssueResult["status"],
+ assigneeAgentId: row.assigneeAgentId,
+ updatedAt: row.updatedAt.toISOString(),
+ ...extracted,
+ };
+ });
+
+ return {
+ contains: query.contains,
+ kind: query.kind,
+ scope: query.scope,
+ limit: query.limit,
+ offset: query.offset,
+ results,
+ hasMore,
+ truncated: hasMore || results.some((result) => result.matchesTruncated),
+ };
+ },
+ };
+}
diff --git a/server/src/services/company-search.ts b/server/src/services/company-search.ts
index 8b90e7b1a8..4de7430cc9 100644
--- a/server/src/services/company-search.ts
+++ b/server/src/services/company-search.ts
@@ -37,6 +37,7 @@ import {
type CompanySearchUpdatedWithinOption,
} from "@paperclipai/shared";
import { companyArtifactsService } from "./company-artifacts.js";
+import { companySearchExtractService } from "./company-search-extract.js";
import { visibleIssueCondition } from "./issue-visibility.js";
const MIN_TOKEN_LENGTH = 2;
@@ -554,7 +555,9 @@ export function companySearchBranchFetchLimit(limit: number, offset = 0) {
}
export function companySearchService(db: Db) {
+ const extractService = companySearchExtractService(db);
return {
+ extract: extractService.extract,
search: async (companyId: string, query: CompanySearchQuery): Promise<CompanySearchResponse> => {
const normalizedQuery = normalizeQuery(query.q);
const hasSearchText = normalizedQuery.length > 0;
diff --git a/server/src/services/index.ts b/server/src/services/index.ts
index bce7184929..b7745b470f 100644
--- a/server/src/services/index.ts
+++ b/server/src/services/index.ts
@@ -1,6 +1,7 @@
export { companyService } from "./companies.js";
export { companyArtifactsService } from "./company-artifacts.js";
export { companySearchService } from "./company-search.js";
+export { companySearchExtractService } from "./company-search-extract.js";
export { feedbackService } from "./feedback.js";
export { companySkillService } from "./company-skills.js";
export { companySkillPolicyService, normalizeSkillPolicySourceType } from "./company-skill-policy.js";