YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(search): keep accessibility-frame metadata matches (#6647)
Ezra Ellette656a05c68163
diff --git a/apps/screenpipe-app-tauri/lib/hooks/__tests__/use-keyword-search-store.test.ts b/apps/screenpipe-app-tauri/lib/hooks/__tests__/use-keyword-search-store.test.tsindex b0fbb72ec..07aa5612f 100644--- a/apps/screenpipe-app-tauri/lib/hooks/__tests__/use-keyword-search-store.test.ts+++ b/apps/screenpipe-app-tauri/lib/hooks/__tests__/use-keyword-search-store.test.ts@@ -322,6 +322,43 @@ describe("useKeywordSearchStore search scheduling", () => {expect(requestedUrls.some((url) => url.includes("/text?persist=false")),).toBe(false);+ expect(mocks.capture).not.toHaveBeenCalledWith(+ "search_ui_query_failed",+ expect.anything(),+ );+ });++ it("keeps metadata matches from accessibility-captured frames", async () => {+ vi.mocked(localFetch).mockImplementation((input) => {+ const url = String(input);+ if (url.startsWith("/search/keyword?")) {+ return Promise.resolve(jsonResponse(grouped([{+ frame_id: 42,+ timestamp: "2026-08-25T17:00:00.000Z",+ text_positions: [{+ text: "semantic button label",+ confidence: 1,+ bounds: { left: 0.1, top: 0.1, width: 0.2, height: 0.05 },+ }],+ app_name: "Google Chrome",+ window_name: "screenpipe search",+ confidence: 1,+ text: "",+ url: "https://screenpipe.com",+ text_source: "accessibility",+ }])));+ }+ if (url.startsWith("/search?")) {+ return Promise.resolve(jsonResponse({ data: [] }));+ }+ throw new Error(`unexpected request: ${url}`);+ });++ await useKeywordSearchStore.getState().searchKeywords("chrome");++ const [result] = useKeywordSearchStore.getState().searchResults;+ expect(result.frame_id).toBe(42);+ expect(result.text_positions).toEqual([]);});it("omits a semantic accessibility result with a coarse element box", async () => {diff --git a/apps/screenpipe-app-tauri/lib/hooks/use-keyword-search-store.tsx b/apps/screenpipe-app-tauri/lib/hooks/use-keyword-search-store.tsxindex e61cd44dc..d4915e0a3 100644--- a/apps/screenpipe-app-tauri/lib/hooks/use-keyword-search-store.tsx+++ b/apps/screenpipe-app-tauri/lib/hooks/use-keyword-search-store.tsx@@ -138,6 +138,13 @@ function textContainsToken(text: string, token: string): boolean {).test(normalizedText);}+function metadataMatchesQuery(result: SearchMatch, query: string): boolean {+ const tokens = queryHighlightTokens(query);+ return [result.app_name, result.window_name, result.url].some((value) =>+ tokens.some((token) => textContainsToken(value, token)),+ );+}+export function visibleMatchingPositions(positions: SearchMatch["text_positions"],query: string,@@ -157,27 +164,33 @@ export function visibleMatchingPositions(/*** Narrow each result's highlight to the positions that match the query.*- * The API remains authoritative and continues to return accessibility matches,- * but the desktop search UI temporarily omits them. AX labels can describe icon- * buttons or whole containers without rendering those words in the screenshot,- * and their element bounds are not trustworthy pixel highlights. Screenshot OCR- * is not run here because it shares the recorder's single OCR permit. Since- * text_source is frame-level, this is a temporary UI cutoff rather than exact- * per-query match provenance.+ * The desktop search UI omits accessibility-only matches. AX labels can describe+ * icon buttons or whole containers without rendering those words in the+ * screenshot, and their element bounds are not trustworthy pixel highlights.+ * `text_source` is frame-level rather than query provenance, though, so an+ * accessibility-captured frame can still match visible app, window, or URL+ * metadata. Keep those results without an AX highlight.+ *+ * Screenshot OCR is not run here because it shares the recorder's single OCR+ * permit. Exact AX-only attribution still belongs in the API contract.*/export function narrowSearchMatchHighlights(results: SearchMatch[],query: string,): SearchMatch[] {- return results- .filter((result) => result.text_source !== "accessibility")- .map((result) => {- const matchingPositions = visibleMatchingPositions(- result.text_positions,- query,- );- return { ...result, text_positions: matchingPositions };- });+ return results.flatMap((result) => {+ const matchingPositions = visibleMatchingPositions(+ result.text_positions,+ query,+ );+ if (+ result.text_source === "accessibility" &&+ !metadataMatchesQuery(result, query)+ ) {+ return [];+ }+ return [{ ...result, text_positions: matchingPositions }];+ });}export const useKeywordSearchStore = create<KeywordSearchState>((set, get) => ({