The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
feat(search): add bulk extract endpoint (#9507)
Dottaae7790861811
diff --git a/doc/SPEC-implementation.md b/doc/SPEC-implementation.mdindex b705aa9da4..fabb879125 100644--- a/doc/SPEC-implementation.md+++ b/doc/SPEC-implementation.md@@ -924,6 +924,10 @@ The current app also exposes V1-supporting surfaces for:- issue thread interactions (`suggest_tasks`, `ask_user_questions`, `request_confirmation`)- issue approvals, issue references/search, labels, read state, inbox/archive state, and work products+- company search through `GET /companies/:companyId/search` plus agent-oriented bulk extraction through+ `GET /companies/:companyId/search/extract`; extraction accepts a server-escaped literal `contains`, optional+ server-owned URL expansion, issue/comment/document scopes, status/date filters, issue-level pagination, and+ explicit issue/match truncation flags- execution workspaces, project workspaces, workspace runtime services, and workspace operations- task watchdog configuration and reusable watchdog issue orchestration for explicitly watched issue subtrees- routines and scheduled/API/webhook triggersdiff --git a/packages/shared/src/index.ts b/packages/shared/src/index.tsindex caefc73521..0802e3e238 100644--- a/packages/shared/src/index.ts+++ b/packages/shared/src/index.ts@@ -658,6 +658,12 @@ export type {ProjectManagedByPlugin,ProjectWorkspace,CompanySearchCountType,+ CompanySearchExtractIssueResult,+ CompanySearchExtractKind,+ CompanySearchExtractMatch,+ CompanySearchExtractResponse,+ CompanySearchExtractScope,+ CompanySearchExtractSourceRef,CompanySearchFilterOptionCounts,CompanySearchHighlight,CompanySearchArtifactSummary,@@ -1209,7 +1215,13 @@ export type {QuotaWindow,ProviderQuotaResult,} from "./types/index.js";-export { COMPANY_SEARCH_SCOPES, COMPANY_SEARCH_SORTS, COMPANY_SEARCH_UPDATED_WITHIN_OPTIONS } from "./types/index.js";+export {+ COMPANY_SEARCH_EXTRACT_KINDS,+ COMPANY_SEARCH_EXTRACT_SCOPES,+ COMPANY_SEARCH_SCOPES,+ COMPANY_SEARCH_SORTS,+ COMPANY_SEARCH_UPDATED_WITHIN_OPTIONS,+} from "./types/index.js";export {ISSUE_REFERENCE_IDENTIFIER_RE,buildIssueReferenceHref,@@ -1466,12 +1478,18 @@ export {type CreateDocumentAnnotationComment,type CreateDocumentAnnotationThread,type UpdateDocumentAnnotationThread,+ companySearchExtractQuerySchema,companySearchQuerySchema,+ COMPANY_SEARCH_EXTRACT_DEFAULT_LIMIT,+ COMPANY_SEARCH_EXTRACT_MAX_LIMIT,+ COMPANY_SEARCH_EXTRACT_MAX_MATCHES_PER_ISSUE,+ COMPANY_SEARCH_EXTRACT_MAX_OFFSET,COMPANY_SEARCH_DEFAULT_LIMIT,COMPANY_SEARCH_MAX_LIMIT,COMPANY_SEARCH_MAX_OFFSET,COMPANY_SEARCH_MAX_QUERY_LENGTH,COMPANY_SEARCH_MAX_TOKENS,+ type CompanySearchExtractQuery,type CompanySearchQuery,createIssueSchema,createIssueInputSchema,diff --git a/packages/shared/src/types/index.ts b/packages/shared/src/types/index.tsindex 8506983117..74537a5a33 100644--- a/packages/shared/src/types/index.ts+++ b/packages/shared/src/types/index.ts@@ -251,6 +251,12 @@ export type {export type { Project, ProjectBudgetSummary, ProjectCodebase, ProjectCodebaseOrigin, ProjectGoalRef, ProjectManagedByPlugin, ProjectWorkspace } from "./project.js";export type {CompanySearchCountType,+ CompanySearchExtractIssueResult,+ CompanySearchExtractKind,+ CompanySearchExtractMatch,+ CompanySearchExtractResponse,+ CompanySearchExtractScope,+ CompanySearchExtractSourceRef,CompanySearchFilterOptionCounts,CompanySearchHighlight,CompanySearchArtifactSummary,@@ -266,7 +272,13 @@ export type {CompanySearchZeroResults,CompanySearchZeroResultsLoosenSuggestion,} from "./search.js";-export { COMPANY_SEARCH_SCOPES, COMPANY_SEARCH_SORTS, COMPANY_SEARCH_UPDATED_WITHIN_OPTIONS } from "./search.js";+export {+ COMPANY_SEARCH_EXTRACT_KINDS,+ COMPANY_SEARCH_EXTRACT_SCOPES,+ COMPANY_SEARCH_SCOPES,+ COMPANY_SEARCH_SORTS,+ COMPANY_SEARCH_UPDATED_WITHIN_OPTIONS,+} from "./search.js";export type {ExecutionWorkspace,ExecutionWorkspaceSummary,diff --git a/packages/shared/src/types/search.ts b/packages/shared/src/types/search.tsindex 7ed7790424..1b812bba5d 100644--- a/packages/shared/src/types/search.ts+++ b/packages/shared/src/types/search.ts@@ -108,3 +108,45 @@ export interface CompanySearchResponse {zeroResults: CompanySearchZeroResults | null;hasMore: boolean;}++export const COMPANY_SEARCH_EXTRACT_SCOPES = ["all", "issues", "comments", "documents"] as const;+export type CompanySearchExtractScope = (typeof COMPANY_SEARCH_EXTRACT_SCOPES)[number];++export const COMPANY_SEARCH_EXTRACT_KINDS = ["literal", "url"] as const;+export type CompanySearchExtractKind = (typeof COMPANY_SEARCH_EXTRACT_KINDS)[number];++export type CompanySearchExtractSourceRef =+ | { type: "issue"; issueId: string }+ | { type: "comment"; commentId: string }+ | { type: "document"; documentId: string; documentKey: string };++export interface CompanySearchExtractMatch {+ value: string;+ field: "title" | "description" | "comment" | "document_title" | "document_body";+ label: string;+ excerpt: string;+ excerptTruncated: boolean;+ source: CompanySearchExtractSourceRef;+}++export interface CompanySearchExtractIssueResult {+ issueId: string;+ identifier: string | null;+ title: string;+ status: IssueStatus;+ assigneeAgentId: string | null;+ updatedAt: string;+ matches: CompanySearchExtractMatch[];+ matchesTruncated: boolean;+}++export interface CompanySearchExtractResponse {+ contains: string;+ kind: CompanySearchExtractKind;+ scope: CompanySearchExtractScope;+ limit: number;+ offset: number;+ results: CompanySearchExtractIssueResult[];+ hasMore: boolean;+ truncated: boolean;+}diff --git a/packages/shared/src/validators/index.ts b/packages/shared/src/validators/index.tsindex d3ccb2aa70..2fa890e13e 100644--- a/packages/shared/src/validators/index.ts+++ b/packages/shared/src/validators/index.ts@@ -420,12 +420,18 @@ export {} from "./issue.js";export {+ COMPANY_SEARCH_EXTRACT_DEFAULT_LIMIT,+ COMPANY_SEARCH_EXTRACT_MAX_LIMIT,+ COMPANY_SEARCH_EXTRACT_MAX_MATCHES_PER_ISSUE,+ COMPANY_SEARCH_EXTRACT_MAX_OFFSET,COMPANY_SEARCH_DEFAULT_LIMIT,COMPANY_SEARCH_MAX_LIMIT,COMPANY_SEARCH_MAX_OFFSET,COMPANY_SEARCH_MAX_QUERY_LENGTH,COMPANY_SEARCH_MAX_TOKENS,companySearchQuerySchema,+ companySearchExtractQuerySchema,+ type CompanySearchExtractQuery,type CompanySearchQuery,} from "./search.js";diff --git a/packages/shared/src/validators/search.ts b/packages/shared/src/validators/search.tsindex 5c995dd823..b54922c1e6 100644--- a/packages/shared/src/validators/search.ts+++ b/packages/shared/src/validators/search.ts@@ -1,13 +1,22 @@import { z } from "zod";import { ISSUE_PRIORITIES, ISSUE_STATUSES } from "../constants.js";import { isUuidLike } from "../agent-url-key.js";-import { COMPANY_SEARCH_SCOPES, COMPANY_SEARCH_SORTS } from "../types/search.js";+import {+ COMPANY_SEARCH_EXTRACT_KINDS,+ COMPANY_SEARCH_EXTRACT_SCOPES,+ COMPANY_SEARCH_SCOPES,+ COMPANY_SEARCH_SORTS,+} from "../types/search.js";export const COMPANY_SEARCH_MAX_QUERY_LENGTH = 200;export const COMPANY_SEARCH_MAX_TOKENS = 8;export const COMPANY_SEARCH_DEFAULT_LIMIT = 20;export const COMPANY_SEARCH_MAX_LIMIT = 50;export const COMPANY_SEARCH_MAX_OFFSET = 200;+export const COMPANY_SEARCH_EXTRACT_DEFAULT_LIMIT = 100;+export const COMPANY_SEARCH_EXTRACT_MAX_LIMIT = 200;+export const COMPANY_SEARCH_EXTRACT_MAX_OFFSET = 5_000;+export const COMPANY_SEARCH_EXTRACT_MAX_MATCHES_PER_ISSUE = 20;const UPDATED_WITHIN_RE = /^[1-9]\d{0,2}(h|d|w|m)$/;@@ -180,3 +189,71 @@ export const companySearchQuerySchema = z.object({});export type CompanySearchQuery = z.infer<typeof companySearchQuerySchema>;++export const companySearchExtractQuerySchema = z.object({+ contains: z.unknown().transform((value, ctx) => {+ const normalized = parseOptionalString(value, ctx, "contains");+ if (!normalized || normalized.length < 2) {+ ctx.addIssue({ code: z.ZodIssueCode.custom, message: "contains must be at least 2 characters" });+ return "";+ }+ if (normalized.length > COMPANY_SEARCH_MAX_QUERY_LENGTH) {+ ctx.addIssue({+ code: z.ZodIssueCode.custom,+ message: `contains must be at most ${COMPANY_SEARCH_MAX_QUERY_LENGTH} characters`,+ });+ }+ return normalized.slice(0, COMPANY_SEARCH_MAX_QUERY_LENGTH);+ }),+ kind: z.unknown()+ .optional()+ .transform((value, ctx) => {+ const normalized = parseOptionalString(value, ctx, "kind") ?? "literal";+ if (!(COMPANY_SEARCH_EXTRACT_KINDS as readonly string[]).includes(normalized)) {+ ctx.addIssue({ code: z.ZodIssueCode.custom, message: "kind must be literal or url" });+ return "literal";+ }+ return normalized as (typeof COMPANY_SEARCH_EXTRACT_KINDS)[number];+ }),+ scope: z.unknown()+ .optional()+ .transform((value, ctx) => {+ const normalized = parseOptionalString(value, ctx, "scope") ?? "all";+ if (!(COMPANY_SEARCH_EXTRACT_SCOPES as readonly string[]).includes(normalized)) {+ ctx.addIssue({ code: z.ZodIssueCode.custom, message: "scope must be all, issues, comments, or documents" });+ return "all";+ }+ return normalized as (typeof COMPANY_SEARCH_EXTRACT_SCOPES)[number];+ }),+ limit: z.unknown()+ .optional()+ .transform((value, ctx) => parseIntegerQuery(+ value,+ ctx,+ "limit",+ COMPANY_SEARCH_EXTRACT_DEFAULT_LIMIT,+ 1,+ COMPANY_SEARCH_EXTRACT_MAX_LIMIT,+ )),+ offset: z.unknown()+ .optional()+ .transform((value, ctx) => parseIntegerQuery(value, ctx, "offset", 0, 0, COMPANY_SEARCH_EXTRACT_MAX_OFFSET)),+ status: z.unknown()+ .optional()+ .transform((value, ctx) => parseEnumList(value, ctx, "status", ISSUE_STATUSES)),+ updatedWithin: z.unknown()+ .optional()+ .transform((value, ctx) => parseUpdatedWithin(value, ctx)),+ updatedAfter: z.unknown()+ .optional()+ .transform((value, ctx) => parseUpdatedAfter(value, ctx)),+}).superRefine((value, ctx) => {+ if (value.updatedWithin && value.updatedAfter) {+ ctx.addIssue({+ code: z.ZodIssueCode.custom,+ message: "updatedWithin and updatedAfter cannot be used together",+ });+ }+});++export type CompanySearchExtractQuery = z.infer<typeof companySearchExtractQuerySchema>;diff --git a/server/src/__tests__/company-search-extract-routes.test.ts b/server/src/__tests__/company-search-extract-routes.test.tsnew file mode 100644index 0000000000..67a7144c96--- /dev/null+++ b/server/src/__tests__/company-search-extract-routes.test.ts@@ -0,0 +1,96 @@+import express from "express";+import request from "supertest";+import { describe, expect, it, vi } from "vitest";+import type {+ CompanySearchExtractQuery,+ CompanySearchExtractResponse,+ CompanySearchQuery,+ CompanySearchResponse,+} from "@paperclipai/shared";+import { issueRoutes } from "../routes/issues.js";+import { createCompanySearchRateLimiter } from "../services/company-search-rate-limit.js";++function extractResponse(query: CompanySearchExtractQuery): CompanySearchExtractResponse {+ return {+ contains: query.contains,+ kind: query.kind,+ scope: query.scope,+ limit: query.limit,+ offset: query.offset,+ results: [],+ hasMore: false,+ truncated: false,+ };+}++function unusedSearch(_companyId: string, _query: CompanySearchQuery): Promise<CompanySearchResponse> {+ throw new Error("interactive search should not be called");+}++function createApp(companyIds: string[], extract: (companyId: string, query: CompanySearchExtractQuery) => Promise<CompanySearchExtractResponse>) {+ const app = express();+ app.use((req, _res, next) => {+ req.actor = {+ type: "board",+ userId: "user-1",+ companyIds,+ source: "session",+ isInstanceAdmin: true,+ };+ next();+ });+ app.use("/api", issueRoutes({} as never, {} as never, {+ searchService: { search: unusedSearch, extract },+ searchRateLimiter: createCompanySearchRateLimiter({+ maxRequests: 1,+ windowMs: 60_000,+ now: () => 1_000,+ }),+ }));+ return app;+}++describe("company extract-search route", () => {+ it("parses the extraction query and invokes the service", async () => {+ const extract = vi.fn(async (_companyId: string, query: CompanySearchExtractQuery) => extractResponse(query));+ const app = createApp(["company-1"], extract);++ const response = await request(app)+ .get("/api/companies/company-1/search/extract")+ .query({ contains: "github.com/example/repo/pull", kind: "url", scope: "comments", limit: "200" })+ .expect(200);++ expect(extract).toHaveBeenCalledWith("company-1", expect.objectContaining({+ contains: "github.com/example/repo/pull",+ kind: "url",+ scope: "comments",+ limit: 200,+ }));+ expect(response.body).toMatchObject({ kind: "url", scope: "comments", truncated: false });+ });++ it("denies cross-company access before invoking the service", async () => {+ const extract = vi.fn(async (_companyId: string, query: CompanySearchExtractQuery) => extractResponse(query));+ const app = createApp(["company-1"], extract);++ await request(app)+ .get("/api/companies/company-2/search/extract?contains=needle")+ .expect(403);++ expect(extract).not.toHaveBeenCalled();+ });++ it("shares the company-search rate limiter", async () => {+ const extract = vi.fn(async (_companyId: string, query: CompanySearchExtractQuery) => extractResponse(query));+ const app = createApp(["company-1"], extract);++ await request(app).get("/api/companies/company-1/search/extract?contains=needle").expect(200);+ const limited = await request(app)+ .get("/api/companies/company-1/search/extract?contains=needle")+ .expect(429);++ expect(extract).toHaveBeenCalledTimes(1);+ expect(limited.body).toEqual({ error: "Search rate limit exceeded", retryAfterSeconds: 60 });+ expect(limited.headers["retry-after"]).toBe("60");+ });+});diff --git a/server/src/__tests__/company-search-extract-service.test.ts b/server/src/__tests__/company-search-extract-service.test.tsnew file mode 100644index 0000000000..e03b7c6de6--- /dev/null+++ b/server/src/__tests__/company-search-extract-service.test.ts@@ -0,0 +1,244 @@+import { randomUUID } from "node:crypto";+import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";+import {+ companies,+ createDb,+ documents,+ issueComments,+ issueDocuments,+ issues,+} from "@paperclipai/db";+import {+ COMPANY_SEARCH_EXTRACT_MAX_MATCHES_PER_ISSUE,+ companySearchExtractQuerySchema,+} from "@paperclipai/shared";+import {+ getEmbeddedPostgresTestSupport,+ startEmbeddedPostgresTestDatabase,+} from "./helpers/embedded-postgres.js";+import { companySearchExtractService } from "../services/company-search-extract.js";++const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();+const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;++if (!embeddedPostgresSupport.supported) {+ console.warn(+ `Skipping embedded Postgres extract-search tests on this host: ${embeddedPostgresSupport.reason ?? "unsupported environment"}`,+ );+}++describe("extract-search query validation", () => {+ it("accepts supported extraction filters and rejects unsafe or ambiguous input", () => {+ const parsed = companySearchExtractQuerySchema.parse({+ contains: "github.com/paperclipai/paperclip/pull",+ kind: "url",+ scope: "comments",+ status: "in_progress,in_review",+ limit: "200",+ offset: "5000",+ updatedWithin: "30d",+ });++ expect(parsed.kind).toBe("url");+ expect(parsed.scope).toBe("comments");+ expect(parsed.status).toEqual(["in_progress", "in_review"]);+ expect(() => companySearchExtractQuerySchema.parse({ contains: ".*", kind: "regex" })).toThrow();+ expect(() => companySearchExtractQuerySchema.parse({ contains: "x" })).toThrow();+ expect(() => companySearchExtractQuerySchema.parse({ contains: "needle", limit: "201" })).toThrow();+ expect(() => companySearchExtractQuerySchema.parse({+ contains: "needle",+ updatedWithin: "30d",+ updatedAfter: "2026-01-01T00:00:00.000Z",+ })).toThrow();+ });+});++describeEmbeddedPostgres("companySearchExtractService", () => {+ let db!: ReturnType<typeof createDb>;+ let svc!: ReturnType<typeof companySearchExtractService>;+ let tempDb: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;++ beforeAll(async () => {+ tempDb = await startEmbeddedPostgresTestDatabase("paperclip-company-search-extract-");+ db = createDb(tempDb.connectionString);+ svc = companySearchExtractService(db);+ }, 20_000);++ afterEach(async () => {+ await db.delete(issueDocuments);+ await db.delete(documents);+ await db.delete(issueComments);+ await db.delete(issues);+ await db.delete(companies);+ });++ afterAll(async () => {+ await tempDb?.cleanup();+ });++ async function createCompany(name = "Paperclip") {+ const companyId = randomUUID();+ await db.insert(companies).values({+ id: companyId,+ name,+ issuePrefix: `E${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`,+ requireBoardApprovalForNewAgents: false,+ });+ return companyId;+ }++ async function createIssue(companyId: string, values: Partial<typeof issues.$inferInsert> = {}) {+ const id = values.id ?? randomUUID();+ await db.insert(issues).values({+ id,+ companyId,+ identifier: values.identifier ?? "EXT-1",+ title: values.title ?? "Extract target",+ description: values.description ?? null,+ status: values.status ?? "in_progress",+ priority: values.priority ?? "medium",+ ...values,+ });+ return id;+ }++ it("expands and deduplicates URLs across issue, comment, and document sources", async () => {+ const companyId = await createCompany();+ const firstUrl = "https://github.com/paperclipai/paperclip/pull/123";+ const secondUrl = "https://github.com/paperclipai/paperclip/pull/456";+ const thirdUrl = "https://github.com/paperclipai/paperclip/pull/789";+ const issueId = await createIssue(companyId, {+ description: `Primary ${firstUrl} and duplicate ${firstUrl}.`,+ });+ await db.insert(issueComments).values({+ companyId,+ issueId,+ body: `Review ${secondUrl} and repeat ${firstUrl}`,+ });+ const documentId = randomUUID();+ await db.insert(documents).values({+ id: documentId,+ companyId,+ title: `PR notes ${thirdUrl}`,+ latestBody: `Also see [the second PR](${secondUrl}).`,+ });+ await db.insert(issueDocuments).values({+ companyId,+ issueId,+ documentId,+ key: "plan",+ });++ const result = await svc.extract(companyId, companySearchExtractQuerySchema.parse({+ contains: "github.com/paperclipai/paperclip/pull",+ kind: "url",+ }));++ expect(result.results).toHaveLength(1);+ expect(result.results[0]?.matches.map((match) => match.value)).toEqual([firstUrl, secondUrl, thirdUrl]);+ expect(result.results[0]?.matches.map((match) => match.field)).toEqual([+ "description",+ "comment",+ "document_title",+ ]);+ expect(result.results[0]?.matchesTruncated).toBe(false);+ expect(result.truncated).toBe(false);+ });++ it("keeps URL sources selected by scheme-less queries", async () => {+ const companyId = await createCompany();+ const titleUrl = "https://github.com/paperclipai/paperclip/pull/101";+ const descriptionUrl = "https://github.com/paperclipai/paperclip/pull/102";+ const documentTitleUrl = "https://github.com/paperclipai/paperclip/pull/103";+ const documentBodyUrl = "https://github.com/paperclipai/paperclip/pull/104";+ const issueId = await createIssue(companyId, {+ title: `Review ${titleUrl}`,+ description: `Then merge ${descriptionUrl}`,+ });+ const documentId = randomUUID();+ await db.insert(documents).values({+ id: documentId,+ companyId,+ title: `Tracking ${documentTitleUrl}`,+ latestBody: `Final follow-up ${documentBodyUrl}`,+ });+ await db.insert(issueDocuments).values({+ companyId,+ issueId,+ documentId,+ key: "plan",+ });++ const result = await svc.extract(companyId, companySearchExtractQuerySchema.parse({+ contains: "github.com/paperclipai/paperclip/pull",+ kind: "url",+ scope: "all",+ }));++ expect(result.results[0]?.matches.map((match) => [match.field, match.value])).toEqual([+ ["title", titleUrl],+ ["description", descriptionUrl],+ ["document_title", documentTitleUrl],+ ["document_body", documentBodyUrl],+ ]);+ });++ it("filters by issue update window and status", async () => {+ const companyId = await createCompany();+ const now = Date.now();+ const recentId = await createIssue(companyId, {+ identifier: "EXT-RECENT",+ description: "needle",+ status: "in_review",+ updatedAt: new Date(now - 24 * 60 * 60 * 1000),+ });+ await createIssue(companyId, {+ identifier: "EXT-OLD",+ description: "needle",+ status: "in_review",+ updatedAt: new Date(now - 60 * 24 * 60 * 60 * 1000),+ });+ await createIssue(companyId, {+ identifier: "EXT-DONE",+ description: "needle",+ status: "done",+ updatedAt: new Date(now - 24 * 60 * 60 * 1000),+ });++ const result = await svc.extract(companyId, companySearchExtractQuerySchema.parse({+ contains: "needle",+ updatedWithin: "30d",+ status: "in_review",+ }));++ expect(result.results.map((row) => row.issueId)).toEqual([recentId]);+ });++ it("caps distinct matches per issue and marks truncation explicitly", async () => {+ const companyId = await createCompany();+ const urls = Array.from(+ { length: COMPANY_SEARCH_EXTRACT_MAX_MATCHES_PER_ISSUE + 1 },+ (_, index) => `https://github.com/paperclipai/paperclip/pull/${index + 1}`,+ );+ await createIssue(companyId, { description: urls.join(" ") });++ const result = await svc.extract(companyId, companySearchExtractQuerySchema.parse({+ contains: "github.com/paperclipai/paperclip/pull",+ kind: "url",+ }));++ expect(result.results[0]?.matches).toHaveLength(COMPANY_SEARCH_EXTRACT_MAX_MATCHES_PER_ISSUE);+ expect(result.results[0]?.matchesTruncated).toBe(true);+ expect(result.truncated).toBe(true);+ });++ it("does not return matching issues from another company", async () => {+ const companyId = await createCompany();+ const otherCompanyId = await createCompany("Other");+ await createIssue(otherCompanyId, { description: "needle" });++ const result = await svc.extract(companyId, companySearchExtractQuerySchema.parse({ contains: "needle" }));++ expect(result.results).toEqual([]);+ });+});diff --git a/server/src/routes/issues.ts b/server/src/routes/issues.tsindex 80a99b51c0..a7bf135606 100644--- a/server/src/routes/issues.ts+++ b/server/src/routes/issues.ts@@ -27,6 +27,7 @@ import {acceptIssueThreadInteractionSchema,attachmentArtifactWorkProductMetadataSchema,cancelIssueThreadInteractionSchema,+ companySearchExtractQuerySchema,companySearchQuerySchema,createIssueAttachmentMetadataSchema,createIssueThreadInteractionSchema,@@ -63,6 +64,8 @@ import {isUuidLike,normalizeIssueIdentifier as normalizeIssueReferenceIdentifier,type CompactIssue,+ type CompanySearchExtractQuery,+ type CompanySearchExtractResponse,type CompanySearchQuery,type CompanySearchResponse,type ExecutionWorkspace,@@ -251,6 +254,7 @@ type RecoveryRevalidationTrigger =| "work_product"| "read_projection";type CompanySearchService = {+ extract(companyId: string, query: CompanySearchExtractQuery): Promise<CompanySearchExtractResponse>;search(companyId: string, query: CompanySearchQuery): Promise<CompanySearchResponse>;};type ActivityIssueRelationSummary = {@@ -4531,6 +4535,40 @@ export function issueRoutes(});});+ router.get("/companies/:companyId/search/extract", async (req, res) => {+ const companyId = req.params.companyId as string;+ assertCompanyAccess(req, companyId);+ const companyScopeDecision = await access.decide({+ actor: req.actor,+ action: "company_scope:read",+ resource: { type: "company", companyId },+ });+ if (!companyScopeDecision.allowed) {+ res.status(403).json({ error: "Company search is outside this actor's authorization boundary" });+ return;+ }+ const parsedQuery = companySearchExtractQuerySchema.safeParse(req.query);+ if (!parsedQuery.success) {+ res.status(400).json({+ error: parsedQuery.error.issues[0]?.message ?? "Invalid extract search query",+ });+ return;+ }+ const rateLimit = searchRateLimiter.consume(companySearchRateLimitActor(req, companyId));+ res.setHeader("X-RateLimit-Limit", String(rateLimit.limit));+ res.setHeader("X-RateLimit-Remaining", String(rateLimit.remaining));+ if (!rateLimit.allowed) {+ res.setHeader("Retry-After", String(rateLimit.retryAfterSeconds));+ res.status(429).json({+ error: "Search rate limit exceeded",+ retryAfterSeconds: rateLimit.retryAfterSeconds,+ });+ return;+ }+ const result = await getSearchService().extract(companyId, parsedQuery.data);+ res.json(result);+ });+router.get("/companies/:companyId/search", async (req, res) => {const companyId = req.params.companyId as string;assertCompanyAccess(req, companyId);diff --git a/server/src/routes/openapi.ts b/server/src/routes/openapi.tsindex 774d53fa89..bdb8bec2b5 100644--- a/server/src/routes/openapi.ts+++ b/server/src/routes/openapi.ts@@ -5142,6 +5142,7 @@ registerCurrentRoute({for (const route of [["get", "/api/companies/import/jobs/{jobId}", "Get company import job status"],["get", "/api/companies/{companyId}/search", "Search company data"],+ ["get", "/api/companies/{companyId}/search/extract", "Extract company search matches"],["get", "/api/companies/{companyId}/issues/count", "Count issues in a company"],] as const) {registerCurrentRoute({diff --git a/server/src/services/company-search-extract.ts b/server/src/services/company-search-extract.tsnew file mode 100644index 0000000000..6462b33445--- /dev/null+++ b/server/src/services/company-search-extract.ts@@ -0,0 +1,349 @@+import { and, asc, desc, eq, gte, inArray, isNull, or, sql } from "drizzle-orm";+import type { SQL, SQLWrapper } from "drizzle-orm";+import type { Db } from "@paperclipai/db";+import { documents, issueComments, issueDocuments, issues } from "@paperclipai/db";+import {+ COMPANY_SEARCH_EXTRACT_MAX_MATCHES_PER_ISSUE,+ type CompanySearchExtractIssueResult,+ type CompanySearchExtractMatch,+ type CompanySearchExtractQuery,+ type CompanySearchExtractResponse,+ type CompanySearchExtractSourceRef,+} from "@paperclipai/shared";+import { visibleIssueCondition } from "./issue-visibility.js";++const EXCERPT_MAX_CHARS = 180;+const URL_PATTERN = /(?:https?:\/\/|www\.)[^\s<>"'`]+|(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z]{2,}\/[^\s<>"'`]+/giu;++type ExtractSource = {+ issueId: string;+ field: CompanySearchExtractMatch["field"];+ label: string;+ text: string;+ source: CompanySearchExtractSourceRef;+};++function escapeLikePattern(value: string): string {+ return value.replace(/[\\%_]/g, "\\$&");+}++function escapeRegexPattern(value: string): string {+ return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");+}++function urlContainsPattern(contains: string): string {+ const literal = escapeRegexPattern(contains);+ return `(?=[^[:space:]<>"']*${literal})(?:(?:https?://|www\\.)[^[:space:]<>"']+|(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\\.)+[a-z]{2,}/[^[:space:]<>"']+)`;+}++function contentMatch(+ column: SQLWrapper,+ query: CompanySearchExtractQuery,+ containsPattern: string,+ urlPattern: string,+): SQL {+ return query.kind === "url"+ ? sql`${column} ~* ${urlPattern}`+ : sql`${column} ILIKE ${containsPattern}`;+}++function updatedWithinStart(value: string | undefined, now = new Date()): Date | null {+ if (!value) return null;+ const match = /^(\d+)(h|d|w|m)$/.exec(value);+ if (!match) return null;+ const amount = Number.parseInt(match[1]!, 10);+ const unit = match[2];+ const hours = unit === "h" ? amount : unit === "d" ? amount * 24 : unit === "w" ? amount * 24 * 7 : amount * 24 * 30;+ return new Date(now.getTime() - hours * 60 * 60 * 1000);+}++function scopeIncludes(scope: CompanySearchExtractQuery["scope"], source: Exclude<CompanySearchExtractQuery["scope"], "all">) {+ return scope === "all" || scope === source;+}++function trimUrlToken(value: string): string {+ let result = value.replace(/[.,;:!?]+$/g, "");+ const pairs: Array<[string, string]> = [["(", ")"], ["[", "]"], ["{", "}"]];+ for (const [open, close] of pairs) {+ while (result.endsWith(close) && result.split(close).length > result.split(open).length) {+ result = result.slice(0, -1);+ }+ }+ return result;+}++function literalOccurrences(text: string, contains: string) {+ const lowerText = text.toLowerCase();+ const lowerContains = contains.toLowerCase();+ const matches: Array<{ value: string; start: number }> = [];+ let start = 0;+ while (start <= text.length - contains.length) {+ const index = lowerText.indexOf(lowerContains, start);+ if (index < 0) break;+ matches.push({ value: text.slice(index, index + contains.length), start: index });+ start = index + Math.max(contains.length, 1);+ }+ return matches;+}++function urlOccurrences(text: string, contains: string) {+ const lowerContains = contains.toLowerCase();+ const matches: Array<{ value: string; start: number }> = [];+ for (const match of text.matchAll(URL_PATTERN)) {+ const raw = match[0];+ const value = trimUrlToken(raw);+ if (!value.toLowerCase().includes(lowerContains)) continue;+ matches.push({ value, start: match.index ?? 0 });+ }+ return matches;+}++function sourceOccurrences(text: string, query: CompanySearchExtractQuery) {+ return query.kind === "url"+ ? urlOccurrences(text, query.contains)+ : literalOccurrences(text, query.contains);+}++function excerpt(text: string, start: number, length: number) {+ if (text.length <= EXCERPT_MAX_CHARS) {+ return { value: text, truncated: false };+ }+ const context = Math.max(0, Math.floor((EXCERPT_MAX_CHARS - length) / 2));+ let excerptStart = Math.max(0, start - context);+ let excerptEnd = Math.min(text.length, excerptStart + EXCERPT_MAX_CHARS);+ excerptStart = Math.max(0, excerptEnd - EXCERPT_MAX_CHARS);+ const prefix = excerptStart > 0 ? "…" : "";+ const suffix = excerptEnd < text.length ? "…" : "";+ return {+ value: `${prefix}${text.slice(excerptStart, excerptEnd).replace(/\s+/g, " ").trim()}${suffix}`,+ truncated: true,+ };+}++function extractMatches(sources: ExtractSource[], query: CompanySearchExtractQuery) {+ const matches: CompanySearchExtractMatch[] = [];+ const seen = new Set<string>();+ let matchesTruncated = false;++ for (const source of sources) {+ const occurrences = sourceOccurrences(source.text, query);+ for (const occurrence of occurrences) {+ const dedupeKey = occurrence.value.toLowerCase();+ if (seen.has(dedupeKey)) continue;+ seen.add(dedupeKey);+ if (matches.length >= COMPANY_SEARCH_EXTRACT_MAX_MATCHES_PER_ISSUE) {+ matchesTruncated = true;+ continue;+ }+ const matchExcerpt = excerpt(source.text, occurrence.start, occurrence.value.length);+ matches.push({+ value: occurrence.value,+ field: source.field,+ label: source.label,+ excerpt: matchExcerpt.value,+ excerptTruncated: matchExcerpt.truncated,+ source: source.source,+ });+ }+ }++ return { matches, matchesTruncated };+}++export function companySearchExtractService(db: Db) {+ return {+ extract: async (companyId: string, query: CompanySearchExtractQuery): Promise<CompanySearchExtractResponse> => {+ const containsPattern = `%${escapeLikePattern(query.contains)}%`;+ const urlPattern = urlContainsPattern(query.contains);+ const scopeConditions: SQL[] = [];+ if (scopeIncludes(query.scope, "issues")) {+ scopeConditions.push(or(+ contentMatch(issues.title, query, containsPattern, urlPattern),+ contentMatch(issues.description, query, containsPattern, urlPattern),+ )!);+ }+ if (scopeIncludes(query.scope, "comments")) {+ scopeConditions.push(sql`EXISTS (+ SELECT 1+ FROM issue_comments extract_comments+ WHERE extract_comments.company_id = ${companyId}+ AND extract_comments.issue_id = ${issues.id}+ AND extract_comments.deleted_at IS NULL+ AND ${query.kind === "url"+ ? sql`extract_comments.body ~* ${urlPattern}`+ : sql`extract_comments.body ILIKE ${containsPattern}`}+ )`);+ }+ if (scopeIncludes(query.scope, "documents")) {+ scopeConditions.push(sql`EXISTS (+ SELECT 1+ FROM issue_documents extract_issue_documents+ INNER JOIN documents extract_documents+ ON extract_documents.id = extract_issue_documents.document_id+ AND extract_documents.company_id = extract_issue_documents.company_id+ WHERE extract_issue_documents.company_id = ${companyId}+ AND extract_issue_documents.issue_id = ${issues.id}+ AND (+ ${query.kind === "url"+ ? sql`extract_documents.title ~* ${urlPattern}`+ : sql`extract_documents.title ILIKE ${containsPattern}`}+ OR ${query.kind === "url"+ ? sql`extract_documents.latest_body ~* ${urlPattern}`+ : sql`extract_documents.latest_body ILIKE ${containsPattern}`}+ )+ )`);+ }++ const conditions: SQL[] = [+ eq(issues.companyId, companyId),+ visibleIssueCondition(),+ or(...scopeConditions)!,+ ];+ if (query.status.length > 0) conditions.push(inArray(issues.status, query.status));+ const updatedWithin = updatedWithinStart(query.updatedWithin);+ if (updatedWithin) conditions.push(gte(issues.updatedAt, updatedWithin));+ if (query.updatedAfter) conditions.push(gte(issues.updatedAt, new Date(query.updatedAfter)));++ const candidateRows = await db+ .select({+ id: issues.id,+ identifier: issues.identifier,+ title: issues.title,+ description: issues.description,+ status: issues.status,+ assigneeAgentId: issues.assigneeAgentId,+ updatedAt: issues.updatedAt,+ })+ .from(issues)+ .where(and(...conditions))+ .orderBy(desc(issues.updatedAt), desc(issues.id))+ .limit(query.limit + 1)+ .offset(query.offset);++ const hasMore = candidateRows.length > query.limit;+ const pageRows = candidateRows.slice(0, query.limit);+ const issueIds = pageRows.map((row) => row.id);+ const sourcesByIssue = new Map<string, ExtractSource[]>();+ const addSource = (source: ExtractSource) => {+ const sources = sourcesByIssue.get(source.issueId) ?? [];+ sources.push(source);+ sourcesByIssue.set(source.issueId, sources);+ };++ if (scopeIncludes(query.scope, "issues")) {+ for (const row of pageRows) {+ if (sourceOccurrences(row.title, query).length > 0) {+ addSource({+ issueId: row.id,+ field: "title",+ label: "Issue title",+ text: row.title,+ source: { type: "issue", issueId: row.id },+ });+ }+ if (row.description && sourceOccurrences(row.description, query).length > 0) {+ addSource({+ issueId: row.id,+ field: "description",+ label: "Issue description",+ text: row.description,+ source: { type: "issue", issueId: row.id },+ });+ }+ }+ }++ if (issueIds.length > 0 && scopeIncludes(query.scope, "comments")) {+ const commentRows = await db+ .select({ id: issueComments.id, issueId: issueComments.issueId, body: issueComments.body })+ .from(issueComments)+ .where(and(+ eq(issueComments.companyId, companyId),+ inArray(issueComments.issueId, issueIds),+ isNull(issueComments.deletedAt),+ contentMatch(issueComments.body, query, containsPattern, urlPattern),+ ))+ .orderBy(asc(issueComments.createdAt), asc(issueComments.id));+ for (const row of commentRows) {+ addSource({+ issueId: row.issueId,+ field: "comment",+ label: "Comment",+ text: row.body,+ source: { type: "comment", commentId: row.id },+ });+ }+ }++ if (issueIds.length > 0 && scopeIncludes(query.scope, "documents")) {+ const documentRows = await db+ .select({+ id: documents.id,+ issueId: issueDocuments.issueId,+ key: issueDocuments.key,+ title: documents.title,+ body: documents.latestBody,+ })+ .from(issueDocuments)+ .innerJoin(documents, and(+ eq(documents.id, issueDocuments.documentId),+ eq(documents.companyId, issueDocuments.companyId),+ ))+ .where(and(+ eq(issueDocuments.companyId, companyId),+ inArray(issueDocuments.issueId, issueIds),+ or(+ contentMatch(documents.title, query, containsPattern, urlPattern),+ contentMatch(documents.latestBody, query, containsPattern, urlPattern),+ ),+ ))+ .orderBy(asc(issueDocuments.key), asc(documents.id));+ for (const row of documentRows) {+ const source = { type: "document" as const, documentId: row.id, documentKey: row.key };+ if (row.title && sourceOccurrences(row.title, query).length > 0) {+ addSource({+ issueId: row.issueId,+ field: "document_title",+ label: `Document title (${row.key})`,+ text: row.title,+ source,+ });+ }+ if (sourceOccurrences(row.body, query).length > 0) {+ addSource({+ issueId: row.issueId,+ field: "document_body",+ label: `Document (${row.key})`,+ text: row.body,+ source,+ });+ }+ }+ }++ const results: CompanySearchExtractIssueResult[] = pageRows.map((row) => {+ const extracted = extractMatches(sourcesByIssue.get(row.id) ?? [], query);+ return {+ issueId: row.id,+ identifier: row.identifier,+ title: row.title,+ status: row.status as CompanySearchExtractIssueResult["status"],+ assigneeAgentId: row.assigneeAgentId,+ updatedAt: row.updatedAt.toISOString(),+ ...extracted,+ };+ });++ return {+ contains: query.contains,+ kind: query.kind,+ scope: query.scope,+ limit: query.limit,+ offset: query.offset,+ results,+ hasMore,+ truncated: hasMore || results.some((result) => result.matchesTruncated),+ };+ },+ };+}diff --git a/server/src/services/company-search.ts b/server/src/services/company-search.tsindex 8b90e7b1a8..4de7430cc9 100644--- a/server/src/services/company-search.ts+++ b/server/src/services/company-search.ts@@ -37,6 +37,7 @@ import {type CompanySearchUpdatedWithinOption,} from "@paperclipai/shared";import { companyArtifactsService } from "./company-artifacts.js";+import { companySearchExtractService } from "./company-search-extract.js";import { visibleIssueCondition } from "./issue-visibility.js";const MIN_TOKEN_LENGTH = 2;@@ -554,7 +555,9 @@ export function companySearchBranchFetchLimit(limit: number, offset = 0) {}export function companySearchService(db: Db) {+ const extractService = companySearchExtractService(db);return {+ extract: extractService.extract,search: async (companyId: string, query: CompanySearchQuery): Promise<CompanySearchResponse> => {const normalizedQuery = normalizeQuery(query.q);const hasSearchText = normalizedQuery.length > 0;diff --git a/server/src/services/index.ts b/server/src/services/index.tsindex bce7184929..b7745b470f 100644--- a/server/src/services/index.ts+++ b/server/src/services/index.ts@@ -1,6 +1,7 @@export { companyService } from "./companies.js";export { companyArtifactsService } from "./company-artifacts.js";export { companySearchService } from "./company-search.js";+export { companySearchExtractService } from "./company-search-extract.js";export { feedbackService } from "./feedback.js";export { companySkillService } from "./company-skills.js";export { companySkillPolicyService, normalizeSkillPolicySourceType } from "./company-skill-policy.js";