Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix(search): keep accessibility-frame metadata matches (#6647)

Ezra Ellette656a05c68163
diff --git a/apps/screenpipe-app-tauri/lib/hooks/__tests__/use-keyword-search-store.test.ts b/apps/screenpipe-app-tauri/lib/hooks/__tests__/use-keyword-search-store.test.ts
index b0fbb72ec..07aa5612f 100644
--- a/apps/screenpipe-app-tauri/lib/hooks/__tests__/use-keyword-search-store.test.ts
+++ b/apps/screenpipe-app-tauri/lib/hooks/__tests__/use-keyword-search-store.test.ts
@@ -322,6 +322,43 @@ describe("useKeywordSearchStore search scheduling", () => {
expect(
requestedUrls.some((url) => url.includes("/text?persist=false")),
).toBe(false);
+ expect(mocks.capture).not.toHaveBeenCalledWith(
+ "search_ui_query_failed",
+ expect.anything(),
+ );
+ });
+
+ it("keeps metadata matches from accessibility-captured frames", async () => {
+ vi.mocked(localFetch).mockImplementation((input) => {
+ const url = String(input);
+ if (url.startsWith("/search/keyword?")) {
+ return Promise.resolve(jsonResponse(grouped([{
+ frame_id: 42,
+ timestamp: "2026-08-25T17:00:00.000Z",
+ text_positions: [{
+ text: "semantic button label",
+ confidence: 1,
+ bounds: { left: 0.1, top: 0.1, width: 0.2, height: 0.05 },
+ }],
+ app_name: "Google Chrome",
+ window_name: "screenpipe search",
+ confidence: 1,
+ text: "",
+ url: "https://screenpipe.com",
+ text_source: "accessibility",
+ }])));
+ }
+ if (url.startsWith("/search?")) {
+ return Promise.resolve(jsonResponse({ data: [] }));
+ }
+ throw new Error(`unexpected request: ${url}`);
+ });
+
+ await useKeywordSearchStore.getState().searchKeywords("chrome");
+
+ const [result] = useKeywordSearchStore.getState().searchResults;
+ expect(result.frame_id).toBe(42);
+ expect(result.text_positions).toEqual([]);
});
it("omits a semantic accessibility result with a coarse element box", async () => {
diff --git a/apps/screenpipe-app-tauri/lib/hooks/use-keyword-search-store.tsx b/apps/screenpipe-app-tauri/lib/hooks/use-keyword-search-store.tsx
index e61cd44dc..d4915e0a3 100644
--- a/apps/screenpipe-app-tauri/lib/hooks/use-keyword-search-store.tsx
+++ b/apps/screenpipe-app-tauri/lib/hooks/use-keyword-search-store.tsx
@@ -138,6 +138,13 @@ function textContainsToken(text: string, token: string): boolean {
).test(normalizedText);
}
+function metadataMatchesQuery(result: SearchMatch, query: string): boolean {
+ const tokens = queryHighlightTokens(query);
+ return [result.app_name, result.window_name, result.url].some((value) =>
+ tokens.some((token) => textContainsToken(value, token)),
+ );
+}
+
export function visibleMatchingPositions(
positions: SearchMatch["text_positions"],
query: string,
@@ -157,27 +164,33 @@ export function visibleMatchingPositions(
/**
* Narrow each result's highlight to the positions that match the query.
*
- * The API remains authoritative and continues to return accessibility matches,
- * but the desktop search UI temporarily omits them. AX labels can describe icon
- * buttons or whole containers without rendering those words in the screenshot,
- * and their element bounds are not trustworthy pixel highlights. Screenshot OCR
- * is not run here because it shares the recorder's single OCR permit. Since
- * text_source is frame-level, this is a temporary UI cutoff rather than exact
- * per-query match provenance.
+ * The desktop search UI omits accessibility-only matches. AX labels can describe
+ * icon buttons or whole containers without rendering those words in the
+ * screenshot, and their element bounds are not trustworthy pixel highlights.
+ * `text_source` is frame-level rather than query provenance, though, so an
+ * accessibility-captured frame can still match visible app, window, or URL
+ * metadata. Keep those results without an AX highlight.
+ *
+ * Screenshot OCR is not run here because it shares the recorder's single OCR
+ * permit. Exact AX-only attribution still belongs in the API contract.
*/
export function narrowSearchMatchHighlights(
results: SearchMatch[],
query: string,
): SearchMatch[] {
- return results
- .filter((result) => result.text_source !== "accessibility")
- .map((result) => {
- const matchingPositions = visibleMatchingPositions(
- result.text_positions,
- query,
- );
- return { ...result, text_positions: matchingPositions };
- });
+ return results.flatMap((result) => {
+ const matchingPositions = visibleMatchingPositions(
+ result.text_positions,
+ query,
+ );
+ if (
+ result.text_source === "accessibility" &&
+ !metadataMatchesQuery(result, query)
+ ) {
+ return [];
+ }
+ return [{ ...result, text_positions: matchingPositions }];
+ });
}
export const useKeywordSearchStore = create<KeywordSearchState>((set, get) => ({