Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

feat(search): index tags for non-document entity types in OpenSearch (#4577)

gbirman5d75449d919e
diff --git a/.github/workspace-dep-closures.json b/.github/workspace-dep-closures.json
index 9db837139b..a2d69c09da 100644
--- a/.github/workspace-dep-closures.json
+++ b/.github/workspace-dep-closures.json
@@ -2758,7 +2758,43 @@
"workspace-hack"
],
"macro_project_utils": [
+ "agent",
+ "ai_toolset",
+ "ai_usage",
+ "attachment",
+ "authentication_service_client",
+ "cowlike",
+ "crm",
+ "document_sub_type",
+ "email",
+ "email_db_client",
+ "email_utils",
+ "entity_access",
+ "entity_access_db_utils",
+ "entity_access_management",
+ "filter_ast",
+ "frecency",
+ "item_filters",
+ "macro_env_var",
"macro_project_utils",
+ "macro_user_id",
+ "macro_uuid",
+ "model",
+ "model-entity",
+ "model-error-response",
+ "model_file_type",
+ "model_user",
+ "models_ai_projection",
+ "models_bulk_upload",
+ "models_email",
+ "models_entity_access_management",
+ "models_pagination",
+ "models_permissions",
+ "non_empty",
+ "prompt",
+ "s3_key",
+ "sqs_client",
+ "unfurl",
"workspace-hack"
],
"macro_queues": [
diff --git a/infra/stacks/opensearch/helpers/constants.ts b/infra/stacks/opensearch/helpers/constants.ts
index a1333ff8c1..d3ca7c9cd1 100644
--- a/infra/stacks/opensearch/helpers/constants.ts
+++ b/infra/stacks/opensearch/helpers/constants.ts
@@ -6,6 +6,7 @@ export const CHATS_ALIAS = 'chats';
export const DOCUMENTS_ALIAS = 'documents';
export const EMAILS_ALIAS = 'emails';
export const CALL_RECORDS_ALIAS = 'call_records';
+export const PROJECTS_ALIAS = 'projects';
// Underlying physical indices (versioned). Bump the suffix to roll a new
// version, then swap the alias atomically.
@@ -14,6 +15,7 @@ export const CHATS_INDEX = 'chats_v2';
export const DOCUMENTS_INDEX = 'documents_v2';
export const EMAILS_INDEX = 'emails_v1';
export const CALL_RECORDS_INDEX = 'call_records_v2';
+export const PROJECTS_INDEX = 'projects_v1';
export const ALIAS_TO_INDEX: Record<string, string> = {
[CHANNELS_ALIAS]: CHANNELS_INDEX,
@@ -21,6 +23,7 @@ export const ALIAS_TO_INDEX: Record<string, string> = {
[DOCUMENTS_ALIAS]: DOCUMENTS_INDEX,
[EMAILS_ALIAS]: EMAILS_INDEX,
[CALL_RECORDS_ALIAS]: CALL_RECORDS_INDEX,
+ [PROJECTS_ALIAS]: PROJECTS_INDEX,
};
// Backward-compat shorthands used by older migration scripts. They now point
diff --git a/infra/stacks/opensearch/helpers/scripts/add_properties_field.ts b/infra/stacks/opensearch/helpers/scripts/add_properties_field.ts
new file mode 100644
index 0000000000..2fad584e68
--- /dev/null
+++ b/infra/stacks/opensearch/helpers/scripts/add_properties_field.ts
@@ -0,0 +1,87 @@
+require('dotenv').config();
+
+import { client } from '../client';
+import { CHATS_ALIAS, EMAILS_ALIAS, IS_DRY_RUN } from '../constants';
+
+// Indexes that receive the nested entity-properties field additively
+// (documents shipped with it; projects_v1 was created with it).
+const SUPPORTED_INDEXES = [EMAILS_ALIAS, CHATS_ALIAS];
+
+async function addPropertiesField(dryRun: boolean) {
+ const index = process.env.INDEX;
+ if (!index || !SUPPORTED_INDEXES.includes(index)) {
+ console.log(
+ `⚠️ Set INDEX to one of: ${SUPPORTED_INDEXES.join(', ')}. Aborting.`
+ );
+ return;
+ }
+
+ const opensearchClient = client();
+
+ console.log('\n' + '='.repeat(60));
+ console.log(
+ `Add nested properties field to "${index}" index ${dryRun ? '(DRY-RUN MODE)' : '(LIVE MODE)'}`
+ );
+ console.log('='.repeat(60));
+
+ if (dryRun) {
+ console.log('\n⚠️ DRY-RUN MODE: No changes will be made');
+ }
+
+ const indexExists = (await opensearchClient.indices.exists({ index })).body;
+
+ if (!indexExists) {
+ console.log(`⚠️ Index "${index}" does not exist. Aborting.`);
+ return;
+ }
+
+ console.log('\nAdding nested properties field mapping...');
+ const mappingUpdate = {
+ properties: {
+ // Entity properties (e.g. tags), same nested shape as the documents
+ // index so the shared property/tag filters apply unchanged. Emails:
+ // thread-level values denormalized onto every message doc. Chats:
+ // values live on the parent chat doc only.
+ properties: {
+ type: 'nested' as const,
+ properties: {
+ definition_id: { type: 'keyword' as const },
+ values: { type: 'keyword' as const },
+ number_value: { type: 'double' as const },
+ date_value: { type: 'date' as const },
+ },
+ },
+ },
+ };
+
+ if (dryRun) {
+ console.log('[DRY-RUN] Would add nested properties field mapping');
+ } else {
+ const putMappingResponse = await opensearchClient.indices.putMapping({
+ index,
+ body: mappingUpdate,
+ });
+
+ if (!putMappingResponse.body.acknowledged) {
+ throw new Error('Failed to add field mapping');
+ }
+ console.log('✓ nested properties field mapping added');
+ }
+
+ console.log('\n' + '='.repeat(60));
+ console.log('Done!');
+ console.log('='.repeat(60));
+
+ if (dryRun) {
+ console.log('\nTo run for real, set DRY_RUN=false environment variable\n');
+ } else {
+ console.log(
+ `\n✓ properties field has been added to the "${index}" index mapping.`
+ );
+ console.log(
+ '✓ To backfill existing tags, POST /internal/backfill/properties with the matching entity_type.\n'
+ );
+ }
+}
+
+addPropertiesField(IS_DRY_RUN);
diff --git a/infra/stacks/opensearch/helpers/scripts/create_indices.ts b/infra/stacks/opensearch/helpers/scripts/create_indices.ts
index 7969cd412e..5d5e436417 100644
--- a/infra/stacks/opensearch/helpers/scripts/create_indices.ts
+++ b/infra/stacks/opensearch/helpers/scripts/create_indices.ts
@@ -11,6 +11,9 @@ import {
DOCUMENTS_INDEX,
EMAILS_ALIAS,
EMAILS_INDEX,
+ IS_DRY_RUN,
+ PROJECTS_ALIAS,
+ PROJECTS_INDEX,
SHARD_SETTINGS,
SLOWLOG_SETTINGS,
} from '../constants';
@@ -141,6 +144,10 @@ async function createIndexWithAlias(
console.log(`${indexName}: ${plan.reason}`);
return;
case 'add_alias':
+ if (IS_DRY_RUN) {
+ console.log(`[DRY-RUN] Would add alias ${aliasName} -> ${indexName}`);
+ return;
+ }
console.log(`Adding alias ${aliasName} -> ${indexName}`);
await opensearchClient.indices.putAlias({
index: indexName,
@@ -148,6 +155,12 @@ async function createIndexWithAlias(
});
return;
case 'create_with_alias':
+ if (IS_DRY_RUN) {
+ console.log(
+ `[DRY-RUN] Would create ${indexName} with alias ${aliasName}`
+ );
+ return;
+ }
console.log(
`${indexName} does not exist, creating with alias ${aliasName}`
);
@@ -158,6 +171,10 @@ async function createIndexWithAlias(
return;
case 'create_without_alias':
console.log(`${indexName}: ${plan.nextStep}`);
+ if (IS_DRY_RUN) {
+ console.log(`[DRY-RUN] Would create ${indexName} without alias`);
+ return;
+ }
await opensearchClient.indices.create({
index: indexName,
body,
@@ -315,6 +332,68 @@ const DOCUMENT_BODY = {
},
};
+// `projects_v1` is flat: one doc per project, `_id` = project id. Projects
+// have no content to chunk, so no join field. Access control follows the
+// documents pattern — only `owner_id` is indexed; the caller's accessible
+// project ids are resolved from Postgres at query time.
+const PROJECTS_BODY = {
+ settings: {
+ ...SHARD_SETTINGS,
+ ...SLOWLOG_SETTINGS,
+ refresh_interval: '1s',
+ },
+ mappings: {
+ dynamic: 'false',
+ properties: {
+ entity_id: {
+ type: 'keyword',
+ },
+ name: {
+ type: 'text',
+ fields: {
+ keyword: {
+ type: 'keyword',
+ ignore_above: 128,
+ },
+ },
+ },
+ owner_id: {
+ type: 'keyword',
+ index: true,
+ doc_values: true,
+ },
+ parent_project_id: {
+ type: 'keyword',
+ index: true,
+ doc_values: true,
+ },
+ created_at_seconds: {
+ type: 'date',
+ format: 'epoch_second',
+ index: false,
+ doc_values: true,
+ },
+ updated_at_seconds: {
+ type: 'date',
+ format: 'epoch_second',
+ index: false,
+ doc_values: true,
+ },
+ // Entity properties (tags, custom). Same nested shape as the documents
+ // index so the shared property/tag query builders apply unchanged.
+ properties: {
+ type: 'nested',
+ properties: {
+ definition_id: { type: 'keyword' },
+ values: { type: 'keyword' },
+ number_value: { type: 'double' },
+ date_value: { type: 'date' },
+ },
+ },
+ },
+ },
+};
+
const EMAIL_BODY = {
settings: {
...SHARD_SETTINGS,
@@ -412,6 +491,18 @@ const EMAIL_BODY = {
type: 'text',
analyzer: 'standard',
},
+ // Thread-level entity properties (e.g. tags), denormalized onto every
+ // message doc of the thread. Same nested shape as the documents index
+ // so the shared property/tag filters apply unchanged.
+ properties: {
+ type: 'nested',
+ properties: {
+ definition_id: { type: 'keyword' },
+ values: { type: 'keyword' },
+ number_value: { type: 'double' },
+ date_value: { type: 'date' },
+ },
+ },
},
},
};
@@ -444,6 +535,18 @@ const CHATS_V2_BODY = {
index: false,
doc_values: true,
},
+ // Parent-only entity properties (tags, custom). Same nested shape as
+ // DOCUMENT_BODY so property filters match definition_id + value within
+ // the same entry rather than cross-matching across properties.
+ properties: {
+ type: 'nested',
+ properties: {
+ definition_id: { type: 'keyword' },
+ values: { type: 'keyword' },
+ number_value: { type: 'double' },
+ date_value: { type: 'date' },
+ },
+ },
// Child-only fields
chat_message_id: { type: 'keyword', index: false, doc_values: true },
content: { type: 'text', analyzer: 'standard' },
@@ -517,7 +620,9 @@ const CALL_RECORDS_V2_BODY = {
async function createIndices() {
const opensearchClient = client();
- console.log('Creating indices...');
+ console.log(
+ `Creating indices... ${IS_DRY_RUN ? '(DRY-RUN MODE — set DRY_RUN=false to apply)' : '(LIVE MODE)'}`
+ );
try {
await createIndexWithAlias(opensearchClient, {
@@ -548,6 +653,11 @@ async function createIndices() {
aliasName: CALL_RECORDS_ALIAS,
body: CALL_RECORDS_V2_BODY,
});
+ await createIndexWithAlias(opensearchClient, {
+ indexName: PROJECTS_INDEX,
+ aliasName: PROJECTS_ALIAS,
+ body: PROJECTS_BODY,
+ });
console.log('done');
} catch (error) {
console.error('Error', error);
diff --git a/js/app/packages/app/component/next-soup/soup-view/filters-bar/search/search-facets.tsx b/js/app/packages/app/component/next-soup/soup-view/filters-bar/search/search-facets.tsx
index 1acc23bced..e2a8b4c95d 100644
--- a/js/app/packages/app/component/next-soup/soup-view/filters-bar/search/search-facets.tsx
+++ b/js/app/packages/app/component/next-soup/soup-view/filters-bar/search/search-facets.tsx
@@ -465,9 +465,9 @@ export function useSearchFacets(
},
});
- // Tags show only where tagging applies (all/documents/tasks), gated behind
- // both the broad tags flag and the search-view rollout flag, and hidden
- // when the caller has no tags defined.
+ // Tags show only where tagging applies (all/documents/tasks/emails), gated
+ // behind both the broad tags flag and the search-view rollout flag, and
+ // hidden when the caller has no tags defined.
const tagFacets = (): SearchFacetVM[] =>
searchTags() && tagSource.enabled() && tagSource.hasTags() ? [tags] : [];
@@ -475,8 +475,8 @@ export function useSearchFacets(
switch (controller.type()) {
case 'email':
return inboxPicker.hasMultiple()
- ? [type, importance, inbox]
- : [type, importance];
+ ? [type, importance, inbox, ...tagFacets()]
+ : [type, importance, ...tagFacets()];
case 'channels':
return [type, channelIn, channelFrom];
case 'calls':
diff --git a/js/app/packages/app/component/next-soup/soup-view/filters-bar/search/search-filters-state.ts b/js/app/packages/app/component/next-soup/soup-view/filters-bar/search/search-filters-state.ts
index c3a862c4e6..c268cc7218 100644
--- a/js/app/packages/app/component/next-soup/soup-view/filters-bar/search/search-filters-state.ts
+++ b/js/app/packages/app/component/next-soup/soup-view/filters-bar/search/search-filters-state.ts
@@ -25,13 +25,14 @@ export type SearchIndexId =
export type SearchTypeValue = SearchIndexId | 'all';
-/** Search types where tag filtering applies: documents/tasks plus the mixed
- * `all` view (tags there narrow the taggable result types and leave the
- * rest untouched). */
+/** Search types where tag filtering applies: documents/tasks, emails, plus
+ * the mixed `all` view (tags there narrow the taggable result types and leave
+ * the rest untouched). */
export const TAG_SEARCH_TYPES = new Set<SearchTypeValue>([
'all',
'task',
'document-or-file',
+ 'email',
]);
/**
@@ -66,9 +67,9 @@ export type SearchFiltersSections = {
export type SearchFiltersState = SearchFiltersSections & {
type: SearchTypeValue;
- // Selected tags. Applied only on the all/document/task types (TAG_SEARCH_TYPES)
- // and read live from the compiled query, so they persist across those types
- // but clear when switching to a type where tags do not apply (email/channels/etc).
+ // Selected tags. Applied only on the TAG_SEARCH_TYPES types and read live
+ // from the compiled query, so they persist across those types but clear
+ // when switching to a type where tags do not apply (channels/calls/etc).
// Each entry carries its owning definition id and option id; combined as one OR.
tags: PropertyFilter[];
};
@@ -94,8 +95,8 @@ export function compileSearchQuery(state: SearchFiltersState): Query {
const include: FieldFilters = { ...baseline.include };
const exclude: FieldFilters = { ...baseline.exclude };
- // Tags apply on the all/document/task types only, so a tag filter never
- // silently empties a search narrowed to emails/channels/calls.
+ // Tags apply on the TAG_SEARCH_TYPES types only, so a tag filter never
+ // silently empties a search narrowed to channels/calls.
if (TAG_SEARCH_TYPES.has(state.type) && state.tags.length) {
include.tagFilters = state.tags;
}
diff --git a/js/app/packages/queries/soup/transform-utils.ts b/js/app/packages/queries/soup/transform-utils.ts
index 86eec73006..2ac650cb06 100644
--- a/js/app/packages/queries/soup/transform-utils.ts
+++ b/js/app/packages/queries/soup/transform-utils.ts
@@ -398,6 +398,7 @@ export const useSearchResponseItemMapper = () => {
participants,
search,
snippet: result.snippet ?? undefined,
+ properties: result.properties ?? undefined,
},
];
}
diff --git a/js/app/packages/service-clients/service-cognition/generated/tools/types.ts b/js/app/packages/service-clients/service-cognition/generated/tools/types.ts
index 78adac1293..719549d4ef 100644
--- a/js/app/packages/service-clients/service-cognition/generated/tools/types.ts
+++ b/js/app/packages/service-clients/service-cognition/generated/tools/types.ts
@@ -1834,9 +1834,10 @@ export interface ProjectMetadata {
viewed_at?: string | null;
}
/**
- * ProjectSearchResponseItem object with project metadata we fetch from macrodb. we don't store these
- * timestamps in opensearch as they would require us to update the project record
- * every time the project updates (specifically for updated_at and viewed_at)
+ * ProjectSearchResponseItem object with project metadata we fetch from macrodb.
+ * The index carries created_at/updated_at for ranking, but viewed_at is
+ * per-user and deleted_at changes without a reindex, so metadata stays
+ * database-sourced.
*/
export interface ProjectSearchResponseItemWithMetadata {
created_at: string;
diff --git a/js/app/packages/service-clients/service-search/generated/models/chatSearchResponseItemWithMetadataAllOf.ts b/js/app/packages/service-clients/service-search/generated/models/chatSearchResponseItemWithMetadataAllOf.ts
index 88ce38cf1c..152beb91ed 100644
--- a/js/app/packages/service-clients/service-search/generated/models/chatSearchResponseItemWithMetadataAllOf.ts
+++ b/js/app/packages/service-clients/service-search/generated/models/chatSearchResponseItemWithMetadataAllOf.ts
@@ -5,7 +5,10 @@
* OpenAPI spec version: 0.1.0
*/
import type { ChatSearchResponseItemWithMetadataAllOfMetadata } from './chatSearchResponseItemWithMetadataAllOfMetadata';
+import type { ChatSearchResponseItemWithMetadataAllOfProperties } from './chatSearchResponseItemWithMetadataAllOfProperties';
export type ChatSearchResponseItemWithMetadataAllOf = {
metadata?: ChatSearchResponseItemWithMetadataAllOfMetadata;
+ /** Entity properties (e.g. tags) on the chat. */
+ properties?: ChatSearchResponseItemWithMetadataAllOfProperties;
};
diff --git a/js/app/packages/service-clients/service-search/generated/models/chatSearchResponseItemWithMetadataAllOfProperties.ts b/js/app/packages/service-clients/service-search/generated/models/chatSearchResponseItemWithMetadataAllOfProperties.ts
new file mode 100644
index 0000000000..c79ce903fc
--- /dev/null
+++ b/js/app/packages/service-clients/service-search/generated/models/chatSearchResponseItemWithMetadataAllOfProperties.ts
@@ -0,0 +1,14 @@
+/**
+ * Generated by orval v7.21.0 🍺
+ * Do not edit manually.
+ * search_service
+ * OpenAPI spec version: 0.1.0
+ */
+import type { SoupProperty } from './soupProperty';
+
+/**
+ * Entity properties (e.g. tags) on the chat.
+ */
+export type ChatSearchResponseItemWithMetadataAllOfProperties =
+ | SoupProperty[]
+ | null;
diff --git a/js/app/packages/service-clients/service-search/generated/models/emailSearchResponseItemWithMetadataAllOf.ts b/js/app/packages/service-clients/service-search/generated/models/emailSearchResponseItemWithMetadataAllOf.ts
index 18ab4cd647..50582429eb 100644
--- a/js/app/packages/service-clients/service-search/generated/models/emailSearchResponseItemWithMetadataAllOf.ts
+++ b/js/app/packages/service-clients/service-search/generated/models/emailSearchResponseItemWithMetadataAllOf.ts
@@ -4,6 +4,7 @@
* search_service
* OpenAPI spec version: 0.1.0
*/
+import type { EmailSearchResponseItemWithMetadataAllOfProperties } from './emailSearchResponseItemWithMetadataAllOfProperties';
import type { EmailSearchResponseItemWithMetadataAllOfSnippet } from './emailSearchResponseItemWithMetadataAllOfSnippet';
import type { EmailSearchResponseItemWithMetadataAllOfViewedAt } from './emailSearchResponseItemWithMetadataAllOfViewedAt';
@@ -13,6 +14,8 @@ export type EmailSearchResponseItemWithMetadataAllOf = {
is_draft: boolean;
is_important: boolean;
is_read: boolean;
+ /** Entity properties (e.g. tags) on the thread. */
+ properties?: EmailSearchResponseItemWithMetadataAllOfProperties;
snippet?: EmailSearchResponseItemWithMetadataAllOfSnippet;
updated_at: string;
viewed_at?: EmailSearchResponseItemWithMetadataAllOfViewedAt;
diff --git a/js/app/packages/service-clients/service-search/generated/models/emailSearchResponseItemWithMetadataAllOfProperties.ts b/js/app/packages/service-clients/service-search/generated/models/emailSearchResponseItemWithMetadataAllOfProperties.ts
new file mode 100644
index 0000000000..b7ef31df92
--- /dev/null
+++ b/js/app/packages/service-clients/service-search/generated/models/emailSearchResponseItemWithMetadataAllOfProperties.ts
@@ -0,0 +1,14 @@
+/**
+ * Generated by orval v7.21.0 🍺
+ * Do not edit manually.
+ * search_service
+ * OpenAPI spec version: 0.1.0
+ */
+import type { SoupProperty } from './soupProperty';
+
+/**
+ * Entity properties (e.g. tags) on the thread.
+ */
+export type EmailSearchResponseItemWithMetadataAllOfProperties =
+ | SoupProperty[]
+ | null;
diff --git a/js/app/packages/service-clients/service-search/generated/models/index.ts b/js/app/packages/service-clients/service-search/generated/models/index.ts
index e01a6439ff..f8589274a5 100644
--- a/js/app/packages/service-clients/service-search/generated/models/index.ts
+++ b/js/app/packages/service-clients/service-search/generated/models/index.ts
@@ -74,6 +74,7 @@ export * from './chatSearchResponseItem';
export * from './chatSearchResponseItemWithMetadata';
export * from './chatSearchResponseItemWithMetadataAllOf';
export * from './chatSearchResponseItemWithMetadataAllOfMetadata';
+export * from './chatSearchResponseItemWithMetadataAllOfProperties';
export * from './crmCompanyFilters';
export * from './crmCompanyFiltersHidden';
export * from './crmCompanySearchDomain';
@@ -125,6 +126,7 @@ export * from './emailSearchResponseItemName';
export * from './emailSearchResponseItemSubject';
export * from './emailSearchResponseItemWithMetadata';
export * from './emailSearchResponseItemWithMetadataAllOf';
+export * from './emailSearchResponseItemWithMetadataAllOfProperties';
export * from './emailSearchResponseItemWithMetadataAllOfSnippet';
export * from './emailSearchResponseItemWithMetadataAllOfViewedAt';
export * from './emailSearchResult';
@@ -160,6 +162,7 @@ export * from './projectSearchResponseItem';
export * from './projectSearchResponseItemWithMetadata';
export * from './projectSearchResponseItemWithMetadataAllOf';
export * from './projectSearchResponseItemWithMetadataAllOfMetadata';
+export * from './projectSearchResponseItemWithMetadataAllOfProperties';
export * from './projectSearchResult';
export * from './projectSearchResultScore';
export * from './propertyDefinition';
diff --git a/js/app/packages/service-clients/service-search/generated/models/projectSearchResponseItemWithMetadata.ts b/js/app/packages/service-clients/service-search/generated/models/projectSearchResponseItemWithMetadata.ts
index 9e940d33b5..2145b1c63a 100644
--- a/js/app/packages/service-clients/service-search/generated/models/projectSearchResponseItemWithMetadata.ts
+++ b/js/app/packages/service-clients/service-search/generated/models/projectSearchResponseItemWithMetadata.ts
@@ -8,9 +8,10 @@ import type { ProjectSearchResponseItem } from './projectSearchResponseItem';
import type { ProjectSearchResponseItemWithMetadataAllOf } from './projectSearchResponseItemWithMetadataAllOf';
/**
- * ProjectSearchResponseItem object with project metadata we fetch from macrodb. we don't store these
-timestamps in opensearch as they would require us to update the project record
-every time the project updates (specifically for updated_at and viewed_at)
+ * ProjectSearchResponseItem object with project metadata we fetch from macrodb.
+The index carries created_at/updated_at for ranking, but viewed_at is
+per-user and deleted_at changes without a reindex, so metadata stays
+database-sourced.
*/
export type ProjectSearchResponseItemWithMetadata = ProjectSearchResponseItem &
ProjectSearchResponseItemWithMetadataAllOf;
diff --git a/js/app/packages/service-clients/service-search/generated/models/projectSearchResponseItemWithMetadataAllOf.ts b/js/app/packages/service-clients/service-search/generated/models/projectSearchResponseItemWithMetadataAllOf.ts
index 0a0a2ebf41..d624d266af 100644
--- a/js/app/packages/service-clients/service-search/generated/models/projectSearchResponseItemWithMetadataAllOf.ts
+++ b/js/app/packages/service-clients/service-search/generated/models/projectSearchResponseItemWithMetadataAllOf.ts
@@ -5,7 +5,10 @@
* OpenAPI spec version: 0.1.0
*/
import type { ProjectSearchResponseItemWithMetadataAllOfMetadata } from './projectSearchResponseItemWithMetadataAllOfMetadata';
+import type { ProjectSearchResponseItemWithMetadataAllOfProperties } from './projectSearchResponseItemWithMetadataAllOfProperties';
export type ProjectSearchResponseItemWithMetadataAllOf = {
metadata?: ProjectSearchResponseItemWithMetadataAllOfMetadata;
+ /** Entity properties (e.g. tags) on the project. */
+ properties?: ProjectSearchResponseItemWithMetadataAllOfProperties;
};
diff --git a/js/app/packages/service-clients/service-search/generated/models/projectSearchResponseItemWithMetadataAllOfProperties.ts b/js/app/packages/service-clients/service-search/generated/models/projectSearchResponseItemWithMetadataAllOfProperties.ts
new file mode 100644
index 0000000000..581069ba6a
--- /dev/null
+++ b/js/app/packages/service-clients/service-search/generated/models/projectSearchResponseItemWithMetadataAllOfProperties.ts
@@ -0,0 +1,14 @@
+/**
+ * Generated by orval v7.21.0 🍺
+ * Do not edit manually.
+ * search_service
+ * OpenAPI spec version: 0.1.0
+ */
+import type { SoupProperty } from './soupProperty';
+
+/**
+ * Entity properties (e.g. tags) on the project.
+ */
+export type ProjectSearchResponseItemWithMetadataAllOfProperties =
+ | SoupProperty[]
+ | null;
diff --git a/js/app/packages/service-clients/service-search/openapi.json b/js/app/packages/service-clients/service-search/openapi.json
index 771909fd2f..c5b8b8a702 100644
--- a/js/app/packages/service-clients/service-search/openapi.json
+++ b/js/app/packages/service-clients/service-search/openapi.json
@@ -867,6 +867,13 @@
"description": "Metadata from the database. None if the chat doesn't exist in the database."
}
]
+ },
+ "properties": {
+ "type": ["array", "null"],
+ "items": {
+ "$ref": "#/components/schemas/SoupProperty"
+ },
+ "description": "Entity properties (e.g. tags) on the chat."
}
}
}
@@ -1506,6 +1513,13 @@
"is_read": {
"type": "boolean"
},
+ "properties": {
+ "type": ["array", "null"],
+ "items": {
+ "$ref": "#/components/schemas/SoupProperty"
+ },
+ "description": "Entity properties (e.g. tags) on the thread."
+ },
"snippet": {
"type": ["string", "null"]
},
@@ -2366,11 +2380,18 @@
"description": "Metadata from the database. None if the project doesn't exist in the database."
}
]
+ },
+ "properties": {
+ "type": ["array", "null"],
+ "items": {
+ "$ref": "#/components/schemas/SoupProperty"
+ },
+ "description": "Entity properties (e.g. tags) on the project."
}
}
}
],
- "description": "ProjectSearchResponseItem object with project metadata we fetch from macrodb. we don't store these\ntimestamps in opensearch as they would require us to update the project record\nevery time the project updates (specifically for updated_at and viewed_at)"
+ "description": "ProjectSearchResponseItem object with project metadata we fetch from macrodb.\nThe index carries created_at/updated_at for ranking, but viewed_at is\nper-user and deleted_at changes without a reindex, so metadata stays\ndatabase-sourced."
},
"ProjectSearchResult": {
"type": "object",
diff --git a/rust/cloud-storage/.sqlx/query-2d726fec1915305011d1b97e3baa8b2c37cdd9e6b2ac0fb412705c511021913c.json b/rust/cloud-storage/.sqlx/query-2d726fec1915305011d1b97e3baa8b2c37cdd9e6b2ac0fb412705c511021913c.json
new file mode 100644
index 0000000000..7adcb2f47f
--- /dev/null
+++ b/rust/cloud-storage/.sqlx/query-2d726fec1915305011d1b97e3baa8b2c37cdd9e6b2ac0fb412705c511021913c.json
@@ -0,0 +1,47 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n SELECT\n c.id as entity_id,\n c.name,\n regexp_replace(\n c.name,\n $7,\n '<macro_em>\\1</macro_em>',\n 'gi'\n ) as name_highlighted,\n c.\"updatedAt\" as updated_at\n FROM \"Chat\" c\n WHERE (c.\"userId\" = $1 OR c.id = ANY($2))\n AND c.\"deletedAt\" IS NULL\n AND c.name ILIKE $3\n AND (\n $5::timestamptz IS NULL\n OR (c.\"updatedAt\", c.id) < ($5, $6)\n )\n AND (\n cardinality($8::text[]) = 0\n OR EXISTS (\n SELECT 1 FROM entity_properties ep\n WHERE ep.entity_id = c.id\n AND ep.entity_type = 'CHAT'\n AND ep.values->'value' ?| $8::text[]\n )\n )\n ORDER BY c.\"updatedAt\" DESC, c.id DESC\n LIMIT $4\n ",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "entity_id",
+ "type_info": "Text"
+ },
+ {
+ "ordinal": 1,
+ "name": "name",
+ "type_info": "Text"
+ },
+ {
+ "ordinal": 2,
+ "name": "name_highlighted",
+ "type_info": "Text"
+ },
+ {
+ "ordinal": 3,
+ "name": "updated_at",
+ "type_info": "Timestamp"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Text",
+ "TextArray",
+ "Text",
+ "Int8",
+ "Timestamptz",
+ "Text",
+ "Text",
+ "TextArray"
+ ]
+ },
+ "nullable": [
+ false,
+ false,
+ null,
+ false
+ ]
+ },
+ "hash": "2d726fec1915305011d1b97e3baa8b2c37cdd9e6b2ac0fb412705c511021913c"
+}
diff --git a/rust/cloud-storage/.sqlx/query-3ede9f15e0c607c9497f4e156277057da7eca0000c60d7bccfd7c2e5cef1550c.json b/rust/cloud-storage/.sqlx/query-3ede9f15e0c607c9497f4e156277057da7eca0000c60d7bccfd7c2e5cef1550c.json
new file mode 100644
index 0000000000..40d5fb2f3f
--- /dev/null
+++ b/rust/cloud-storage/.sqlx/query-3ede9f15e0c607c9497f4e156277057da7eca0000c60d7bccfd7c2e5cef1550c.json
@@ -0,0 +1,58 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n SELECT\n p.id,\n p.name,\n p.\"userId\" as user_id,\n p.\"parentId\" as parent_id,\n p.\"createdAt\"::timestamptz as created_at,\n p.\"updatedAt\"::timestamptz as updated_at,\n p.\"deletedAt\"::timestamptz as deleted_at\n FROM \"Project\" p\n WHERE id = $1\n ",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "id",
+ "type_info": "Text"
+ },
+ {
+ "ordinal": 1,
+ "name": "name",
+ "type_info": "Text"
+ },
+ {
+ "ordinal": 2,
+ "name": "user_id",
+ "type_info": "Text"
+ },
+ {
+ "ordinal": 3,
+ "name": "parent_id",
+ "type_info": "Text"
+ },
+ {
+ "ordinal": 4,
+ "name": "created_at",
+ "type_info": "Timestamptz"
+ },
+ {
+ "ordinal": 5,
+ "name": "updated_at",
+ "type_info": "Timestamptz"
+ },
+ {
+ "ordinal": 6,
+ "name": "deleted_at",
+ "type_info": "Timestamptz"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Text"
+ ]
+ },
+ "nullable": [
+ false,
+ false,
+ false,
+ true,
+ null,
+ null,
+ null
+ ]
+ },
+ "hash": "3ede9f15e0c607c9497f4e156277057da7eca0000c60d7bccfd7c2e5cef1550c"
+}
diff --git a/rust/cloud-storage/.sqlx/query-5e4006d396a3d256162ec62127d1eb492a3189665b15f395d86f2aafa6508fe9.json b/rust/cloud-storage/.sqlx/query-5e4006d396a3d256162ec62127d1eb492a3189665b15f395d86f2aafa6508fe9.json
new file mode 100644
index 0000000000..302b50c815
--- /dev/null
+++ b/rust/cloud-storage/.sqlx/query-5e4006d396a3d256162ec62127d1eb492a3189665b15f395d86f2aafa6508fe9.json
@@ -0,0 +1,46 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n SELECT\n c.id as entity_id,\n c.name,\n regexp_replace(\n c.name,\n $6,\n '<macro_em>\\1</macro_em>',\n 'gi'\n ) as name_highlighted,\n c.\"updatedAt\" as updated_at\n FROM \"Chat\" c\n WHERE c.id = ANY($1)\n AND c.\"deletedAt\" IS NULL\n AND c.name ILIKE $2\n AND (\n $4::timestamptz IS NULL\n OR (c.\"updatedAt\", c.id) < ($4, $5)\n )\n AND (\n cardinality($7::text[]) = 0\n OR EXISTS (\n SELECT 1 FROM entity_properties ep\n WHERE ep.entity_id = c.id\n AND ep.entity_type = 'CHAT'\n AND ep.values->'value' ?| $7::text[]\n )\n )\n ORDER BY c.\"updatedAt\" DESC, c.id DESC\n LIMIT $3\n ",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "entity_id",
+ "type_info": "Text"
+ },
+ {
+ "ordinal": 1,
+ "name": "name",
+ "type_info": "Text"
+ },
+ {
+ "ordinal": 2,
+ "name": "name_highlighted",
+ "type_info": "Text"
+ },
+ {
+ "ordinal": 3,
+ "name": "updated_at",
+ "type_info": "Timestamp"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "TextArray",
+ "Text",
+ "Int8",
+ "Timestamptz",
+ "Text",
+ "Text",
+ "TextArray"
+ ]
+ },
+ "nullable": [
+ false,
+ false,
+ null,
+ false
+ ]
+ },
+ "hash": "5e4006d396a3d256162ec62127d1eb492a3189665b15f395d86f2aafa6508fe9"
+}
diff --git a/rust/cloud-storage/.sqlx/query-ba7fb4ba657dd212eb78237865b7ca0dce914f21f53c83a119ba14a38c903684.json b/rust/cloud-storage/.sqlx/query-ba7fb4ba657dd212eb78237865b7ca0dce914f21f53c83a119ba14a38c903684.json
new file mode 100644
index 0000000000..951856a707
--- /dev/null
+++ b/rust/cloud-storage/.sqlx/query-ba7fb4ba657dd212eb78237865b7ca0dce914f21f53c83a119ba14a38c903684.json
@@ -0,0 +1,32 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n SELECT\n p.id as project_id,\n p.\"updatedAt\"::timestamptz as updated_at\n FROM \"Project\" p\n WHERE p.\"deletedAt\" IS NULL\n AND (\n $2::timestamptz IS NULL\n OR (p.\"updatedAt\", p.id) > ($2, $3)\n )\n AND ($4::timestamptz IS NULL OR p.\"updatedAt\" >= $4)\n AND ($5::timestamptz IS NULL OR p.\"updatedAt\" <= $5)\n ORDER BY p.\"updatedAt\" ASC, p.id ASC\n LIMIT $1\n ",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "project_id",
+ "type_info": "Text"
+ },
+ {
+ "ordinal": 1,
+ "name": "updated_at",
+ "type_info": "Timestamptz"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Int8",
+ "Timestamptz",
+ "Text",
+ "Timestamptz",
+ "Timestamptz"
+ ]
+ },
+ "nullable": [
+ false,
+ null
+ ]
+ },
+ "hash": "ba7fb4ba657dd212eb78237865b7ca0dce914f21f53c83a119ba14a38c903684"
+}
diff --git a/rust/cloud-storage/Cargo.lock b/rust/cloud-storage/Cargo.lock
index 64f163193a..e1b0a6a5b4 100644
--- a/rust/cloud-storage/Cargo.lock
+++ b/rust/cloud-storage/Cargo.lock
@@ -7900,6 +7900,7 @@ version = "0.1.0"
dependencies = [
"anyhow",
"sqlx",
+ "sqs_client",
"tokio",
"tracing",
"workspace-hack",
diff --git a/rust/cloud-storage/deleted_item_poller/src/handler.rs b/rust/cloud-storage/deleted_item_poller/src/handler.rs
index ea02d6eb59..590a2bffa6 100644
--- a/rust/cloud-storage/deleted_item_poller/src/handler.rs
+++ b/rust/cloud-storage/deleted_item_poller/src/handler.rs
@@ -5,7 +5,9 @@ use lambda_runtime::{
Error, LambdaEvent,
tracing::{self},
};
-use sqs_client::search::{SearchQueueMessage, chat::RemoveChatMessage, document::DocumentId};
+use sqs_client::search::{
+ SearchQueueMessage, chat::RemoveChatMessage, document::DocumentId, project::RemoveProject,
+};
#[tracing::instrument(skip(ctx, _event), err)]
pub async fn handler(
@@ -35,6 +37,22 @@ async fn handle_projects(ctx: &context::Context) -> anyhow::Result<()> {
tracing::debug!(projects_to_delete=?projects_to_delete, "projects to delete");
+ // Idempotent safety net — the soft delete already removed these from the
+ // search index.
+ ctx.sqs_client
+ .bulk_send_message_to_search_event_queue(
+ projects_to_delete
+ .iter()
+ .map(|id| {
+ SearchQueueMessage::RemoveProject(RemoveProject {
+ project_id: id.to_string(),
+ index_override: None,
+ })
+ })
+ .collect(),
+ )
+ .await?;
+
// We can actually perform the project deletion here as we will automatically be queuing all
// the items in the project for deletion as well
macro_db_client::projects::delete::delete_projects_bulk(&ctx.db, &projects_to_delete)
diff --git a/rust/cloud-storage/document_storage_service/src/api/documents/initialize_user_documents.rs b/rust/cloud-storage/document_storage_service/src/api/documents/initialize_user_documents.rs
index 6c279bdb41..8c05ccc67f 100644
--- a/rust/cloud-storage/document_storage_service/src/api/documents/initialize_user_documents.rs
+++ b/rust/cloud-storage/document_storage_service/src/api/documents/initialize_user_documents.rs
@@ -13,6 +13,7 @@ use model_user::axum_extractor::MacroUserExtractor;
use models_permissions::share_permission::SharePermissionV2;
use reqwest::StatusCode;
use s3_key::build_cloud_storage_bucket_document_key;
+use sqs_client::search::{SearchQueueMessage, project::UpsertProject};
const ONBOARDING_FOLDER_NAME: &str = "ONBOARDING_DOCUMENTS";
const PROJECT_NAME: &str = "Starter Docs";
@@ -245,6 +246,24 @@ pub async fn handler(
.into_response()
})?;
+ tokio::spawn({
+ let sqs_client = state.sqs_client.clone();
+ let project_id = project.id.clone();
+ async move {
+ let _ = sqs_client
+ .send_message_to_search_event_queue(SearchQueueMessage::UpsertProject(
+ UpsertProject {
+ project_id: project_id.clone(),
+ index_override: None,
+ },
+ ))
+ .await
+ .inspect_err(
+ |e| tracing::error!(error=?e, project_id=?project_id, "unable to enqueue project search upsert"),
+ );
+ }
+ });
+
Ok((StatusCode::OK, Json(GenericSuccessResponse::default())).into_response())
}
diff --git a/rust/cloud-storage/document_storage_service/src/api/projects/create_project.rs b/rust/cloud-storage/document_storage_service/src/api/projects/create_project.rs
index 6952d345a8..7d04653fa5 100644
--- a/rust/cloud-storage/document_storage_service/src/api/projects/create_project.rs
+++ b/rust/cloud-storage/document_storage_service/src/api/projects/create_project.rs
@@ -15,6 +15,7 @@ use model::{
use model_entity::EntityType;
use models_permissions::share_permission::SharePermissionV2;
use models_permissions::share_permission::access_level::EditAccessLevel;
+use sqs_client::search::{SearchQueueMessage, project::UpsertProject};
use unicode_segmentation::UnicodeSegmentation;
/// Creates a new project.
@@ -89,6 +90,24 @@ async fn create_project_v2(
}
};
+ tokio::spawn({
+ let sqs_client = ctx.sqs_client.clone();
+ let project_id = project.id.clone();
+ async move {
+ let _ = sqs_client
+ .send_message_to_search_event_queue(SearchQueueMessage::UpsertProject(
+ UpsertProject {
+ project_id: project_id.clone(),
+ index_override: None,
+ },
+ ))
+ .await
+ .inspect_err(
+ |e| tracing::error!(error=?e, project_id=?project_id, "unable to enqueue project search upsert"),
+ );
+ }
+ });
+
// update project modified if necessary
if let Some(project_id) = req.project_parent_id {
tracing::trace!("updating project modified date");
@@ -105,6 +124,7 @@ async fn create_project_v2(
macro_project_utils::update_project_modified(
&ctx.db,
+ Some(ctx.sqs_client.as_ref()),
macro_project_utils::ProjectModifiedArgs {
project_id: Some(project_id.to_string()),
old_project_id: None,
diff --git a/rust/cloud-storage/document_storage_service/src/api/projects/delete_project.rs b/rust/cloud-storage/document_storage_service/src/api/projects/delete_project.rs
index f11f484b5e..86c4722dbd 100644
--- a/rust/cloud-storage/document_storage_service/src/api/projects/delete_project.rs
+++ b/rust/cloud-storage/document_storage_service/src/api/projects/delete_project.rs
@@ -17,7 +17,9 @@ use model::{
user::UserContext,
};
use models_permissions::share_permission::access_level::OwnerAccessLevel;
-use sqs_client::search::{SearchQueueMessage, chat::RemoveChatMessage, document::DocumentId};
+use sqs_client::search::{
+ SearchQueueMessage, chat::RemoveChatMessage, document::DocumentId, project::RemoveProject,
+};
use utoipa::ToSchema;
#[derive(serde::Deserialize)]
@@ -74,6 +76,31 @@ pub async fn delete_project_handler(
}
};
+ if !project_ids.is_empty() {
+ tokio::spawn({
+ let sqs_client = ctx.sqs_client.clone();
+ let project_ids = project_ids.clone();
+ async move {
+ let _ = sqs_client
+ .bulk_send_message_to_search_event_queue(
+ project_ids
+ .iter()
+ .map(|id| {
+ SearchQueueMessage::RemoveProject(RemoveProject {
+ project_id: id.to_string(),
+ index_override: None,
+ })
+ })
+ .collect(),
+ )
+ .await
+ .inspect_err(
+ |e| tracing::error!(error=?e, "unable to enqueue delete projects for search"),
+ );
+ }
+ });
+ }
+
if let Some(parent_id) = &project.parent_id {
tracing::info!(
parent_id,
@@ -81,6 +108,7 @@ pub async fn delete_project_handler(
);
macro_project_utils::update_project_modified(
&ctx.db,
+ Some(ctx.sqs_client.as_ref()),
macro_project_utils::ProjectModifiedArgs {
project_id: None,
old_project_id: Some(parent_id),
@@ -257,6 +285,31 @@ pub async fn permanently_delete_project_handler(
.send(StatusCode::INTERNAL_SERVER_ERROR));
}
+ if !project_ids.is_empty() {
+ tokio::spawn({
+ let sqs_client = ctx.sqs_client.clone();
+ let project_ids = project_ids.clone();
+ async move {
+ let _ = sqs_client
+ .bulk_send_message_to_search_event_queue(
+ project_ids
+ .iter()
+ .map(|id| {
+ SearchQueueMessage::RemoveProject(RemoveProject {
+ project_id: id.to_string(),
+ index_override: None,
+ })
+ })
+ .collect(),
+ )
+ .await
+ .inspect_err(
+ |e| tracing::error!(error=?e, "unable to enqueue delete projects for search"),
+ );
+ }
+ });
+ }
+
if !chat_ids.is_empty() {
tokio::spawn({
let sqs_client = ctx.sqs_client.clone();
diff --git a/rust/cloud-storage/document_storage_service/src/api/projects/edit_project.rs b/rust/cloud-storage/document_storage_service/src/api/projects/edit_project.rs
index 6f6a34a15c..64b9a3f596 100644
--- a/rust/cloud-storage/document_storage_service/src/api/projects/edit_project.rs
+++ b/rust/cloud-storage/document_storage_service/src/api/projects/edit_project.rs
@@ -186,6 +186,7 @@ async fn edit_project_v2(
// Update the project you are editting
macro_project_utils::update_project_modified(
&ctx.db,
+ Some(ctx.sqs_client.as_ref()),
macro_project_utils::ProjectModifiedArgs {
project_id: Some(id.clone()), // The project you edited
old_project_id: None,
@@ -214,6 +215,7 @@ async fn edit_project_v2(
// Update the project you moved from and moved to
macro_project_utils::update_project_modified(
&ctx.db,
+ Some(ctx.sqs_client.as_ref()),
macro_project_utils::ProjectModifiedArgs {
project_id: req.project_parent_id, // The new project you've placed your item in
old_project_id: project_context.parent_id.clone(), // The old project you've moved your item from
diff --git a/rust/cloud-storage/document_storage_service/src/api/projects/revert_delete_project.rs b/rust/cloud-storage/document_storage_service/src/api/projects/revert_delete_project.rs
index b95e30d246..a9e2912096 100644
--- a/rust/cloud-storage/document_storage_service/src/api/projects/revert_delete_project.rs
+++ b/rust/cloud-storage/document_storage_service/src/api/projects/revert_delete_project.rs
@@ -8,6 +8,7 @@ use model::response::{
GenericErrorResponse, GenericResponse, GenericSuccessResponse, SuccessResponse,
};
use model::user::UserContext;
+use sqs_client::search::{SearchQueueMessage, project::UpsertProject};
#[derive(serde::Deserialize)]
pub struct Params {
@@ -39,18 +40,45 @@ pub async fn handler(
) -> impl IntoResponse {
tracing::info!("revert_delete project");
- if let Err(e) = macro_db_client::projects::revert_delete::revert_delete_project(
+ let project_ids = match macro_db_client::projects::revert_delete::revert_delete_project(
&ctx.db,
&id,
project_context.parent_id.as_deref(),
)
.await
{
- tracing::error!(error=?e, "unable to revert project");
- return GenericResponse::builder()
- .message("unable to revert project")
- .is_error(true)
- .send(StatusCode::INTERNAL_SERVER_ERROR);
+ Ok(project_ids) => project_ids,
+ Err(e) => {
+ tracing::error!(error=?e, "unable to revert project");
+ return GenericResponse::builder()
+ .message("unable to revert project")
+ .is_error(true)
+ .send(StatusCode::INTERNAL_SERVER_ERROR);
+ }
+ };
+
+ if !project_ids.is_empty() {
+ tokio::spawn({
+ let sqs_client = ctx.sqs_client.clone();
+ async move {
+ let _ = sqs_client
+ .bulk_send_message_to_search_event_queue(
+ project_ids
+ .iter()
+ .map(|id| {
+ SearchQueueMessage::UpsertProject(UpsertProject {
+ project_id: id.to_string(),
+ index_override: None,
+ })
+ })
+ .collect(),
+ )
+ .await
+ .inspect_err(
+ |e| tracing::error!(error=?e, "unable to enqueue restored projects for search"),
+ );
+ }
+ });
}
let data = GenericSuccessResponse::default();
diff --git a/rust/cloud-storage/document_storage_service/src/api/projects/upload_folder.rs b/rust/cloud-storage/document_storage_service/src/api/projects/upload_folder.rs
index d8b5467efa..973ecaebf7 100644
--- a/rust/cloud-storage/document_storage_service/src/api/projects/upload_folder.rs
+++ b/rust/cloud-storage/document_storage_service/src/api/projects/upload_folder.rs
@@ -26,6 +26,7 @@ use models_bulk_upload::{
use models_permissions::share_permission::SharePermissionV2;
use s3_key::{build_cloud_storage_bucket_document_key, build_docx_staging_bucket_document_key};
use sqlx::{Pool, Postgres};
+use sqs_client::search::{SearchQueueMessage, project::UpsertProject};
use std::{str::FromStr, sync::Arc};
use uuid::Uuid;
@@ -128,6 +129,7 @@ pub async fn upload_folder_handler(
upload_folder_handler_inner(
ctx.s3_client.clone(),
ctx.db.clone(),
+ ctx.sqs_client.clone(),
user_context.macro_user_id,
internal,
req,
@@ -141,6 +143,7 @@ pub async fn upload_folder_handler(
async fn upload_folder_handler_inner(
s3_client: Arc<service::s3::S3>,
db: Pool<Postgres>,
+ sqs_client: Arc<sqs_client::SQS>,
user_id: MacroUserIdStr<'static>,
internal: bool,
req: UploadFolderRequest,
@@ -248,6 +251,31 @@ async fn upload_folder_handler_inner(
.send(StatusCode::INTERNAL_SERVER_ERROR);
}
+ if !project_ids.is_empty() {
+ tokio::spawn({
+ let sqs_client = sqs_client.clone();
+ let project_ids = project_ids.clone();
+ async move {
+ let _ = sqs_client
+ .bulk_send_message_to_search_event_queue(
+ project_ids
+ .iter()
+ .map(|id| {
+ SearchQueueMessage::UpsertProject(UpsertProject {
+ project_id: id.to_string(),
+ index_override: None,
+ })
+ })
+ .collect(),
+ )
+ .await
+ .inspect_err(
+ |e| tracing::error!(error=?e, "unable to enqueue uploaded projects for search"),
+ );
+ }
+ });
+ }
+
let data = UploadFolderResponseData {
file_system,
destination_map,
diff --git a/rust/cloud-storage/macro_db_client/src/projects/get_project/get_project_by_id.rs b/rust/cloud-storage/macro_db_client/src/projects/get_project/get_project_by_id.rs
index 324a69ea49..0e8f572931 100644
--- a/rust/cloud-storage/macro_db_client/src/projects/get_project/get_project_by_id.rs
+++ b/rust/cloud-storage/macro_db_client/src/projects/get_project/get_project_by_id.rs
@@ -2,6 +2,36 @@ use sqlx::{Pool, Postgres};
use model::project::Project;
+/// Fetch a project row for search indexing. Unlike [`get_project_by_id`],
+/// soft-deleted rows are returned so the caller can turn the upsert into a
+/// removal, and a missing row maps to `None` instead of an error.
+#[tracing::instrument(skip(db), err)]
+pub async fn get_project_for_search(
+ db: &Pool<Postgres>,
+ project_id: &str,
+) -> anyhow::Result<Option<Project>> {
+ let result = sqlx::query_as!(
+ Project,
+ r#"
+ SELECT
+ p.id,
+ p.name,
+ p."userId" as user_id,
+ p."parentId" as parent_id,
+ p."createdAt"::timestamptz as created_at,
+ p."updatedAt"::timestamptz as updated_at,
+ p."deletedAt"::timestamptz as deleted_at
+ FROM "Project" p
+ WHERE id = $1
+ "#,
+ project_id
+ )
+ .fetch_optional(db)
+ .await?;
+
+ Ok(result)
+}
+
#[tracing::instrument(skip(db))]
pub async fn get_project_by_id(
db: Pool<Postgres>,
diff --git a/rust/cloud-storage/macro_db_client/src/projects/get_project/mod.rs b/rust/cloud-storage/macro_db_client/src/projects/get_project/mod.rs
index 5d96750806..07b7ce0745 100644
--- a/rust/cloud-storage/macro_db_client/src/projects/get_project/mod.rs
+++ b/rust/cloud-storage/macro_db_client/src/projects/get_project/mod.rs
@@ -7,5 +7,5 @@ pub mod get_basic_project;
pub mod get_project_chats;
pub mod get_project_documents;
-pub use get_project_by_id::get_project_by_id;
+pub use get_project_by_id::{get_project_by_id, get_project_for_search};
pub use get_project_content::get_project_content_v2;
diff --git a/rust/cloud-storage/macro_db_client/src/projects/get_projects.rs b/rust/cloud-storage/macro_db_client/src/projects/get_projects.rs
index 97c9a6f8aa..1a9f1ecd35 100644
--- a/rust/cloud-storage/macro_db_client/src/projects/get_projects.rs
+++ b/rust/cloud-storage/macro_db_client/src/projects/get_projects.rs
@@ -1,7 +1,57 @@
+use chrono::{DateTime, Utc};
use sqlx::{Pool, Postgres};
use model::project::{Project, ProjectWithUploadRequest};
+/// One row of the search-backfill scan: just enough to build an upsert
+/// message and the keyset cursor.
+pub struct ProjectSearchBackfillRow {
+ pub project_id: String,
+ pub updated_at: Option<DateTime<Utc>>,
+}
+
+/// Walk non-deleted projects in `("updatedAt" ASC, id ASC)` order for the
+/// search backfill. `cursor` carries the last row of the previous page so
+/// the next page resumes with `WHERE ("updatedAt", id) > cursor`.
+#[tracing::instrument(skip(db), err)]
+pub async fn get_projects_for_search_backfill(
+ db: &Pool<Postgres>,
+ limit: i64,
+ cursor: Option<(DateTime<Utc>, String)>,
+ updated_after: Option<DateTime<Utc>>,
+ updated_before: Option<DateTime<Utc>>,
+) -> anyhow::Result<Vec<ProjectSearchBackfillRow>> {
+ let (cursor_updated_at, cursor_project_id) = cursor.unzip();
+
+ let rows = sqlx::query_as!(
+ ProjectSearchBackfillRow,
+ r#"
+ SELECT
+ p.id as project_id,
+ p."updatedAt"::timestamptz as updated_at
+ FROM "Project" p
+ WHERE p."deletedAt" IS NULL
+ AND (
+ $2::timestamptz IS NULL
+ OR (p."updatedAt", p.id) > ($2, $3)
+ )
+ AND ($4::timestamptz IS NULL OR p."updatedAt" >= $4)
+ AND ($5::timestamptz IS NULL OR p."updatedAt" <= $5)
+ ORDER BY p."updatedAt" ASC, p.id ASC
+ LIMIT $1
+ "#,
+ limit,
+ cursor_updated_at,
+ cursor_project_id.unwrap_or_default(),
+ updated_after,
+ updated_before,
+ )
+ .fetch_all(db)
+ .await?;
+
+ Ok(rows)
+}
+
/// Gets all sub-projects of the provided project IDs
pub async fn get_sub_project_ids(
db: &Pool<Postgres>,
diff --git a/rust/cloud-storage/macro_db_client/src/projects/mod.rs b/rust/cloud-storage/macro_db_client/src/projects/mod.rs
index 297f65c7b9..63bc1cc0b0 100644
--- a/rust/cloud-storage/macro_db_client/src/projects/mod.rs
+++ b/rust/cloud-storage/macro_db_client/src/projects/mod.rs
@@ -9,8 +9,8 @@ pub mod upload_folder;
pub use create_project::create_project_v2;
pub use edit_project::{edit_project_v2, update_project_modified_date};
pub use get_projects::{
- get_all_project_ids_with_users_paginated, get_pending_root_projects, get_projects,
- get_projects_to_delete, get_sub_project_ids,
+ ProjectSearchBackfillRow, get_all_project_ids_with_users_paginated, get_pending_root_projects,
+ get_projects, get_projects_for_search_backfill, get_projects_to_delete, get_sub_project_ids,
};
pub use preview::batch_get_project_preview_v2;
pub mod delete;
diff --git a/rust/cloud-storage/macro_db_client/src/projects/revert_delete.rs b/rust/cloud-storage/macro_db_client/src/projects/revert_delete.rs
index faffc6ac8b..5508f924d2 100644
--- a/rust/cloud-storage/macro_db_client/src/projects/revert_delete.rs
+++ b/rust/cloud-storage/macro_db_client/src/projects/revert_delete.rs
@@ -10,12 +10,13 @@ use super::get_project::get_sub_items::get_all_deleted_sub_project_ids;
/// Reverts a project deletion
/// Reverts all sub-items of the project as well
/// Adds all items back to the users history as well
+/// Returns the ids of the projects that were restored
#[tracing::instrument(skip(db))]
pub async fn revert_delete_project(
db: &sqlx::Pool<sqlx::Postgres>,
project_id: &str,
project_parent_id: Option<&str>,
-) -> anyhow::Result<()> {
+) -> anyhow::Result<Vec<String>> {
let mut transaction = db.begin().await.context("unable to begin transaction")?;
let project_ids = get_all_deleted_sub_project_ids(&mut transaction, project_id)
@@ -148,7 +149,7 @@ pub async fn revert_delete_project(
.await
.context("unable to commit transaction")?;
- Ok(())
+ Ok(project_ids_vec)
}
#[cfg(test)]
diff --git a/rust/cloud-storage/macro_project_utils/Cargo.toml b/rust/cloud-storage/macro_project_utils/Cargo.toml
index 6651b26cd9..971ad5567d 100644
--- a/rust/cloud-storage/macro_project_utils/Cargo.toml
+++ b/rust/cloud-storage/macro_project_utils/Cargo.toml
@@ -7,6 +7,9 @@ version = "0.1.0"
[dependencies]
anyhow = { workspace = true }
sqlx = { workspace = true }
+sqs_client = { path = "../sqs_client", default-features = false, features = [
+ "search",
+] }
tokio = { workspace = true }
tracing = { workspace = true }
workspace-hack = { version = "0.1", path = "../workspace-hack" }
diff --git a/rust/cloud-storage/macro_project_utils/src/lib.rs b/rust/cloud-storage/macro_project_utils/src/lib.rs
index c6a835f944..ddfa74c46a 100644
--- a/rust/cloud-storage/macro_project_utils/src/lib.rs
+++ b/rust/cloud-storage/macro_project_utils/src/lib.rs
@@ -1,3 +1,8 @@
+use sqs_client::{
+ SQS,
+ search::{SearchQueueMessage, project::UpsertProject},
+};
+
#[derive(Clone, Debug)]
pub struct ProjectModifiedArgs<T>
where
@@ -11,9 +16,10 @@ where
pub user_id: String,
}
-#[tracing::instrument(skip(db))]
+#[tracing::instrument(skip(db, sqs))]
pub async fn update_project_modified<T>(
db: &sqlx::Pool<sqlx::Postgres>,
+ sqs: Option<&SQS>,
project_modified_args: ProjectModifiedArgs<T>,
) where
T: Clone + std::fmt::Debug + ToString + std::marker::Send + Sync,
@@ -31,12 +37,13 @@ pub async fn update_project_modified<T>(
tokio::spawn({
let db = db.clone();
+ let sqs = sqs.cloned();
async move {
if let Some(old_project_id) = old_project_id
&& !old_project_id.is_empty()
{
tracing::trace!(project_id=?old_project_id, "updating project modified date");
- let _ = update_project_modified_date(&db, &old_project_id).await.inspect_err(|e| {
+ let _ = update_project_modified_date(&db, sqs.as_ref(), &old_project_id).await.inspect_err(|e| {
tracing::error!(error=?e, project_id=?old_project_id, "unable to update project modified date");
});
}
@@ -46,7 +53,7 @@ pub async fn update_project_modified<T>(
{
tracing::trace!(project_id=?project_id, "updating project modified date");
- let _ = update_project_modified_date(&db, &project_id).await.inspect_err(|e| {
+ let _ = update_project_modified_date(&db, sqs.as_ref(), &project_id).await.inspect_err(|e| {
tracing::error!(error=?e, project_id=?project_id, "unable to update project modified date");
});
}
@@ -56,6 +63,7 @@ pub async fn update_project_modified<T>(
async fn update_project_modified_date(
db: &sqlx::Pool<sqlx::Postgres>,
+ sqs: Option<&SQS>,
project_id: &str,
) -> anyhow::Result<()> {
sqlx::query!(
@@ -65,5 +73,19 @@ async fn update_project_modified_date(
.execute(db)
.await?;
+ // Refresh the project's search doc so ranking follows the bumped
+ // updatedAt. Best-effort like the bump itself.
+ if let Some(sqs) = sqs {
+ let _ = sqs
+ .send_message_to_search_event_queue(SearchQueueMessage::UpsertProject(UpsertProject {
+ project_id: project_id.to_string(),
+ index_override: None,
+ }))
+ .await
+ .inspect_err(|e| {
+ tracing::error!(error=?e, project_id=?project_id, "unable to enqueue project search upsert");
+ });
+ }
+
Ok(())
}
diff --git a/rust/cloud-storage/models_opensearch/src/lib.rs b/rust/cloud-storage/models_opensearch/src/lib.rs
index b9f64cb2b5..932ecb9b58 100644
--- a/rust/cloud-storage/models_opensearch/src/lib.rs
+++ b/rust/cloud-storage/models_opensearch/src/lib.rs
@@ -21,12 +21,14 @@ pub enum SearchIndex {
Emails,
/// The call records alias
CallRecords,
+ /// The projects alias
+ Projects,
}
/// All searchable entity types — the tag on a unified `SearchHit`,
/// independent of where the hit came from. Most are backed by an
-/// OpenSearch index, but some (Projects, CrmCompanies) are Postgres-only
-/// and synthesized by name searches; those never appear in OpenSearch
+/// OpenSearch index, but some (CrmCompanies) are Postgres-only and
+/// synthesized by name searches; those never appear in OpenSearch
/// responses. The OpenSearch-backed subset is [`OpenSearchEntityType`].
#[derive(
Debug,
@@ -51,7 +53,7 @@ pub enum SearchEntityType {
Documents,
/// The email entity type (has OpenSearch index)
Emails,
- /// The project entity type (Postgres-only)
+ /// The project entity type (has OpenSearch index)
Projects,
/// The call records entity type (has OpenSearch index)
CallRecords,
@@ -85,6 +87,8 @@ pub enum OpenSearchEntityType {
Emails,
/// The call records index
CallRecords,
+ /// The projects index
+ Projects,
}
impl OpenSearchEntityType {
@@ -98,6 +102,7 @@ impl OpenSearchEntityType {
Self::Documents => "documents",
Self::Emails => "emails",
Self::CallRecords => "call_records",
+ Self::Projects => "projects",
}
}
}
@@ -110,6 +115,7 @@ impl From<OpenSearchEntityType> for SearchEntityType {
OpenSearchEntityType::Documents => SearchEntityType::Documents,
OpenSearchEntityType::Emails => SearchEntityType::Emails,
OpenSearchEntityType::CallRecords => SearchEntityType::CallRecords,
+ OpenSearchEntityType::Projects => SearchEntityType::Projects,
}
}
}
@@ -122,6 +128,7 @@ impl From<OpenSearchEntityType> for SearchIndex {
OpenSearchEntityType::Documents => SearchIndex::Documents,
OpenSearchEntityType::Emails => SearchIndex::Emails,
OpenSearchEntityType::CallRecords => SearchIndex::CallRecords,
+ OpenSearchEntityType::Projects => SearchIndex::Projects,
}
}
}
@@ -138,6 +145,7 @@ mod test {
OpenSearchEntityType::Documents,
OpenSearchEntityType::Emails,
OpenSearchEntityType::CallRecords,
+ OpenSearchEntityType::Projects,
] {
let from_index: SearchIndex = variant.clone().into();
assert_eq!(variant.index_name(), from_index.as_ref());
diff --git a/rust/cloud-storage/models_search/src/chat.rs b/rust/cloud-storage/models_search/src/chat.rs
index 170bcb218d..39ff415464 100644
--- a/rust/cloud-storage/models_search/src/chat.rs
+++ b/rust/cloud-storage/models_search/src/chat.rs
@@ -1,5 +1,6 @@
use chrono::{DateTime, Utc};
use item_filters::ChatFilters;
+use models_soup::SoupProperty;
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
use utoipa::ToSchema;
@@ -58,6 +59,10 @@ pub struct ChatMetadata {
pub struct ChatSearchResponseItemWithMetadata {
/// Metadata from the database. None if the chat doesn't exist in the database.
pub metadata: Option<ChatMetadata>,
+ /// Entity properties (e.g. tags) on the chat.
+ #[serde(skip_serializing_if = "Option::is_none")]
+ #[schemars(skip)]
+ pub properties: Option<Vec<SoupProperty>>,
#[serde(flatten)]
pub extra: ChatSearchResponseItem,
}
diff --git a/rust/cloud-storage/models_search/src/email.rs b/rust/cloud-storage/models_search/src/email.rs
index 88e60b1bcc..39a98ef8ea 100644
--- a/rust/cloud-storage/models_search/src/email.rs
+++ b/rust/cloud-storage/models_search/src/email.rs
@@ -1,6 +1,7 @@
use crate::{MatchType, SearchHighlight, SearchOn};
use chrono::{DateTime, Utc};
use item_filters::EmailFilters;
+use models_soup::SoupProperty;
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
use utoipa::ToSchema;
@@ -85,6 +86,10 @@ pub struct EmailSearchResponseItemWithMetadata {
pub inbox_visible: bool,
pub is_draft: bool,
pub is_important: bool,
+ /// Entity properties (e.g. tags) on the thread.
+ #[serde(skip_serializing_if = "Option::is_none")]
+ #[schemars(skip)]
+ pub properties: Option<Vec<SoupProperty>>,
#[serde(flatten)]
pub extra: EmailSearchResponseItem,
}
diff --git a/rust/cloud-storage/models_search/src/project.rs b/rust/cloud-storage/models_search/src/project.rs
index f2e1b08b02..201b4d9ffc 100644
--- a/rust/cloud-storage/models_search/src/project.rs
+++ b/rust/cloud-storage/models_search/src/project.rs
@@ -1,6 +1,7 @@
use crate::{MatchType, SearchHighlight, SearchOn, SearchResponse, SearchResponseItem};
use chrono::{DateTime, Utc};
use item_filters::ProjectFilters;
+use models_soup::SoupProperty;
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
use utoipa::ToSchema;
@@ -48,13 +49,18 @@ pub struct ProjectMetadata {
pub deleted_at: Option<DateTime<Utc>>,
}
-/// ProjectSearchResponseItem object with project metadata we fetch from macrodb. we don't store these
-/// timestamps in opensearch as they would require us to update the project record
-/// every time the project updates (specifically for updated_at and viewed_at)
+/// ProjectSearchResponseItem object with project metadata we fetch from macrodb.
+/// The index carries created_at/updated_at for ranking, but viewed_at is
+/// per-user and deleted_at changes without a reindex, so metadata stays
+/// database-sourced.
#[derive(Debug, Serialize, Deserialize, ToSchema, JsonSchema)]
pub struct ProjectSearchResponseItemWithMetadata {
/// Metadata from the database. None if the project doesn't exist in the database.
pub metadata: Option<ProjectMetadata>,
+ /// Entity properties (e.g. tags) on the project.
+ #[serde(skip_serializing_if = "Option::is_none")]
+ #[schemars(skip)]
+ pub properties: Option<Vec<SoupProperty>>,
#[serde(flatten)]
pub extra: ProjectSearchResponseItem,
}
diff --git a/rust/cloud-storage/name_search/fixtures/chat_tags.sql b/rust/cloud-storage/name_search/fixtures/chat_tags.sql
new file mode 100644
index 0000000000..ed69e4f69f
--- /dev/null
+++ b/rust/cloud-storage/name_search/fixtures/chat_tags.sql
@@ -0,0 +1,11 @@
+-- Tag values on chats, keyed to the system Status definition seeded by
+-- migrations. Only the option ids inside `values` matter to the tag filter.
+INSERT INTO entity_properties (id, entity_id, entity_type, property_definition_id, values)
+VALUES
+ (
+ 'eeeeeeee-0000-0000-0000-000000000001',
+ '22222222-2222-2222-2222-222222222222',
+ 'CHAT',
+ '00000001-0000-0000-0000-000000000002',
+ '{"type": "SelectOption", "value": ["cccccccc-0000-0000-0000-000000000001", "cccccccc-0000-0000-0000-000000000002"]}'
+ );
diff --git a/rust/cloud-storage/name_search/fixtures/project.sql b/rust/cloud-storage/name_search/fixtures/project.sql
deleted file mode 100644
index 5b7762ca20..0000000000
--- a/rust/cloud-storage/name_search/fixtures/project.sql
+++ /dev/null
@@ -1,43 +0,0 @@
--- Test macro_users
-INSERT INTO "macro_user" (id, username, email, stripe_customer_id)
-VALUES
- ('a1111111-1111-1111-1111-111111111111', '[email protected]', '[email protected]', 'stripe_user1'),
- ('a2222222-2222-2222-2222-222222222222', '[email protected]', '[email protected]', 'stripe_user2'),
- ('a3333333-3333-3333-3333-333333333333', '[email protected]', '[email protected]', 'stripe_user3');
-
--- Test users
-INSERT INTO "User" (id, email, name, macro_user_id)
-VALUES
- ('macro|[email protected]', '[email protected]', 'Test User 1', 'a1111111-1111-1111-1111-111111111111'),
- ('macro|[email protected]', '[email protected]', 'Test User 2', 'a2222222-2222-2222-2222-222222222222'),
- ('macro|[email protected]', '[email protected]', 'Test User 3', 'a3333333-3333-3333-3333-333333333333');
-
--- Test projects for user1 (userId)
-INSERT INTO "Project" (id, name, "userId", "createdAt", "updatedAt", "deletedAt")
-VALUES
- -- Projects matching "mobile"
- ('11111111-1111-1111-1111-111111111111', 'Mobile Development', 'macro|[email protected]', '2024-01-01 10:00:00', '2024-12-01 10:00:00', NULL),
- ('22222222-2222-2222-2222-222222222222', 'Mobile App Redesign', 'macro|[email protected]', '2024-02-01 10:00:00', '2024-12-02 10:00:00', NULL),
-
- -- Projects matching "web"
- ('33333333-3333-3333-3333-333333333333', 'Web Platform Upgrade', 'macro|[email protected]', '2024-03-01 10:00:00', '2024-12-03 10:00:00', NULL),
- ('44444444-4444-4444-4444-444444444444', 'Website Optimization', 'macro|[email protected]', '2024-04-01 10:00:00', '2024-12-04 10:00:00', NULL),
-
- -- Projects not matching common search terms
- ('55555555-5555-5555-5555-555555555555', 'Marketing Campaign', 'macro|[email protected]', '2024-05-01 10:00:00', '2024-12-05 10:00:00', NULL),
-
- -- Test case sensitivity (should match "MOBILE" search)
- ('66666666-6666-6666-6666-666666666666', 'MOBILE PLATFORM', 'macro|[email protected]', '2024-06-01 10:00:00', '2024-12-06 10:00:00', NULL),
-
- -- Soft deleted project (should be excluded from search results)
- ('77777777-7777-7777-7777-777777777777', 'Deleted Mobile Project', 'macro|[email protected]', '2024-07-01 10:00:00', '2024-12-07 10:00:00', '2024-12-08 10:00:00');
-
--- Test projects for user2 (not accessible to user1 unless shared)
-INSERT INTO "Project" (id, name, "userId", "createdAt", "updatedAt", "deletedAt")
-VALUES
- ('88888888-8888-8888-8888-888888888888', 'User2 Mobile Project', 'macro|[email protected]', '2024-08-01 10:00:00', '2024-12-09 10:00:00', NULL);
-
--- Test projects for user3 (used for sharing tests)
-INSERT INTO "Project" (id, name, "userId", "createdAt", "updatedAt", "deletedAt")
-VALUES
- ('99999999-9999-9999-9999-999999999999', 'User3 Shared Mobile', 'macro|[email protected]', '2024-09-01 10:00:00', '2024-12-10 10:00:00', NULL);
diff --git a/rust/cloud-storage/name_search/src/chat.rs b/rust/cloud-storage/name_search/src/chat.rs
index d489293367..cd1f9916b9 100644
--- a/rust/cloud-storage/name_search/src/chat.rs
+++ b/rust/cloud-storage/name_search/src/chat.rs
@@ -1,5 +1,9 @@
//! This module contains logic for searching chats by name
+// The `cached` macro drops fn-level lint attributes from its generated
+// wrapper, so the search functions' argument-count allow lives here.
+#![allow(clippy::too_many_arguments)]
+
#[cfg(not(test))]
use cached::proc_macro::cached;
use chrono::{DateTime, Utc};
@@ -16,6 +20,7 @@ async fn ids_search(
chat_ids: &[Uuid],
search_pattern: String,
highlight_pattern: String,
+ tag_option_ids: &[String],
limit: u32,
cursor: Option<SearchMethodCursor>,
) -> Result<PaginatedResult<NameSearchResult>, NameSearchError> {
@@ -52,6 +57,15 @@ async fn ids_search(
$4::timestamptz IS NULL
OR (c."updatedAt", c.id) < ($4, $5)
)
+ AND (
+ cardinality($7::text[]) = 0
+ OR EXISTS (
+ SELECT 1 FROM entity_properties ep
+ WHERE ep.entity_id = c.id
+ AND ep.entity_type = 'CHAT'
+ AND ep.values->'value' ?| $7::text[]
+ )
+ )
ORDER BY c."updatedAt" DESC, c.id DESC
LIMIT $3
"#,
@@ -64,6 +78,7 @@ async fn ids_search(
cursor_updated_at,
cursor_entity_id,
highlight_pattern,
+ tag_option_ids,
)
.fetch_all(db)
.await
@@ -89,6 +104,7 @@ async fn owner_search<'a>(
chat_ids: &[Uuid],
search_pattern: String,
highlight_pattern: String,
+ tag_option_ids: &[String],
limit: u32,
cursor: Option<SearchMethodCursor>,
) -> Result<PaginatedResult<NameSearchResult>, NameSearchError> {
@@ -121,6 +137,15 @@ async fn owner_search<'a>(
$5::timestamptz IS NULL
OR (c."updatedAt", c.id) < ($5, $6)
)
+ AND (
+ cardinality($8::text[]) = 0
+ OR EXISTS (
+ SELECT 1 FROM entity_properties ep
+ WHERE ep.entity_id = c.id
+ AND ep.entity_type = 'CHAT'
+ AND ep.values->'value' ?| $8::text[]
+ )
+ )
ORDER BY c."updatedAt" DESC, c.id DESC
LIMIT $4
"#,
@@ -134,6 +159,7 @@ async fn owner_search<'a>(
cursor_updated_at,
cursor_entity_id,
highlight_pattern,
+ tag_option_ids,
)
.fetch_all(db)
.await
@@ -152,7 +178,8 @@ async fn owner_search<'a>(
Ok(SearchCursorOption::paginate(results, limit as usize))
}
-/// Searches over the user's chats by name
+/// Searches over the user's chats by name. A non-empty `tag_option_ids`
+/// restricts results to chats holding any of those property option ids.
#[tracing::instrument(skip(db), err)]
#[cfg_attr(
not(test),
@@ -160,7 +187,7 @@ async fn owner_search<'a>(
time = 30,
result = true,
key = "String",
- convert = r#"{ format!("{}-{:?}-{}-{}-{}-{}", macro_user_id.as_ref(), chat_ids, term, ids_only, limit, cursor.as_ref().and_then(|c| c.as_updated_at()).map(|(id, ts)| format!("{}-{}", id, ts)).unwrap_or_default()) }"#
+ convert = r#"{ format!("{}-{:?}-{}-{}-{:?}-{}-{}", macro_user_id.as_ref(), chat_ids, term, ids_only, tag_option_ids, limit, cursor.as_ref().and_then(|c| c.as_updated_at()).map(|(id, ts)| format!("{}-{}", id, ts)).unwrap_or_default()) }"#
)
)]
pub async fn search_chat_names<'a>(
@@ -169,6 +196,7 @@ pub async fn search_chat_names<'a>(
chat_ids: &[Uuid],
term: String,
ids_only: bool,
+ tag_option_ids: &[String],
limit: u32,
cursor: Option<SearchMethodCursor>,
) -> Result<PaginatedResult<NameSearchResult>, NameSearchError> {
@@ -188,6 +216,7 @@ pub async fn search_chat_names<'a>(
chat_ids,
search_pattern,
highlight_pattern,
+ tag_option_ids,
limit,
cursor,
)
@@ -199,6 +228,7 @@ pub async fn search_chat_names<'a>(
chat_ids,
search_pattern,
highlight_pattern,
+ tag_option_ids,
limit,
cursor,
)
diff --git a/rust/cloud-storage/name_search/src/chat/test.rs b/rust/cloud-storage/name_search/src/chat/test.rs
index 7b8f6b94ff..2b86581a65 100644
--- a/rust/cloud-storage/name_search/src/chat/test.rs
+++ b/rust/cloud-storage/name_search/src/chat/test.rs
@@ -16,7 +16,8 @@ async fn test_search_chat_names_empty_term(pool: Pool<Postgres>) -> anyhow::Resu
.map(|l| l.lowercase())
.unwrap();
- let result = search_chat_names(&pool, &user_id, &[], "".to_string(), false, 10, None).await;
+ let result =
+ search_chat_names(&pool, &user_id, &[], "".to_string(), false, &[], 10, None).await;
assert!(result.is_err());
assert!(matches!(
@@ -38,8 +39,17 @@ async fn test_search_chat_names_ids_only_with_empty_ids(
.map(|l| l.lowercase())
.unwrap();
- let result =
- search_chat_names(&pool, &user_id, &[], "project".to_string(), true, 10, None).await;
+ let result = search_chat_names(
+ &pool,
+ &user_id,
+ &[],
+ "project".to_string(),
+ true,
+ &[],
+ 10,
+ None,
+ )
+ .await;
assert!(result.is_err());
assert!(matches!(
@@ -72,6 +82,7 @@ async fn test_search_chat_names_ids_only_mode(pool: Pool<Postgres>) -> anyhow::R
&chat_ids,
"project".to_string(),
true,
+ &[],
10,
None,
)
@@ -115,8 +126,17 @@ async fn test_search_chat_names_normal_mode_owned_chats(
.unwrap();
// Search for "project" across all user1's owned chats
- let response =
- search_chat_names(&pool, &user_id, &[], "project".to_string(), false, 10, None).await?;
+ let response = search_chat_names(
+ &pool,
+ &user_id,
+ &[],
+ "project".to_string(),
+ false,
+ &[],
+ 10,
+ None,
+ )
+ .await?;
// Should return 3 chats matching "project" (2 lowercase + 1 uppercase)
assert_eq!(response.items.len(), 3);
@@ -153,20 +173,47 @@ async fn test_search_chat_names_case_insensitive(pool: Pool<Postgres>) -> anyhow
.unwrap();
// Search with uppercase term should match both lowercase and uppercase names
- let response =
- search_chat_names(&pool, &user_id, &[], "PROJECT".to_string(), false, 10, None).await?;
+ let response = search_chat_names(
+ &pool,
+ &user_id,
+ &[],
+ "PROJECT".to_string(),
+ false,
+ &[],
+ 10,
+ None,
+ )
+ .await?;
assert_eq!(response.items.len(), 3);
// Search with lowercase term should also match both
- let response =
- search_chat_names(&pool, &user_id, &[], "project".to_string(), false, 10, None).await?;
+ let response = search_chat_names(
+ &pool,
+ &user_id,
+ &[],
+ "project".to_string(),
+ false,
+ &[],
+ 10,
+ None,
+ )
+ .await?;
assert_eq!(response.items.len(), 3);
// Search with mixed case
- let response =
- search_chat_names(&pool, &user_id, &[], "PrOjEcT".to_string(), false, 10, None).await?;
+ let response = search_chat_names(
+ &pool,
+ &user_id,
+ &[],
+ "PrOjEcT".to_string(),
+ false,
+ &[],
+ 10,
+ None,
+ )
+ .await?;
assert_eq!(response.items.len(), 3);
@@ -191,6 +238,7 @@ async fn test_search_chat_names_with_shared_chats(pool: Pool<Postgres>) -> anyho
&shared_chat_ids,
"project".to_string(),
false,
+ &[],
10,
None,
)
@@ -224,8 +272,17 @@ async fn test_search_chat_names_pagination_limit(pool: Pool<Postgres>) -> anyhow
.unwrap();
// Search with limit of 2
- let response =
- search_chat_names(&pool, &user_id, &[], "project".to_string(), false, 2, None).await?;
+ let response = search_chat_names(
+ &pool,
+ &user_id,
+ &[],
+ "project".to_string(),
+ false,
+ &[],
+ 2,
+ None,
+ )
+ .await?;
assert_eq!(response.items.len(), 2);
@@ -255,8 +312,17 @@ async fn test_search_chat_names_pagination_cursor(pool: Pool<Postgres>) -> anyho
.unwrap();
// First page with limit of 2
- let first_response =
- search_chat_names(&pool, &user_id, &[], "project".to_string(), false, 2, None).await?;
+ let first_response = search_chat_names(
+ &pool,
+ &user_id,
+ &[],
+ "project".to_string(),
+ false,
+ &[],
+ 2,
+ None,
+ )
+ .await?;
assert_eq!(first_response.items.len(), 2);
assert!(first_response.cursor.has_more());
@@ -274,6 +340,7 @@ async fn test_search_chat_names_pagination_cursor(pool: Pool<Postgres>) -> anyho
&[],
"project".to_string(),
false,
+ &[],
2,
cursor,
)
@@ -329,6 +396,7 @@ async fn test_search_chat_names_no_results(pool: Pool<Postgres>) -> anyhow::Resu
&[],
"nonexistent".to_string(),
false,
+ &[],
10,
None,
)
@@ -350,8 +418,17 @@ async fn test_search_chat_names_partial_match(pool: Pool<Postgres>) -> anyhow::R
.unwrap();
// Search for partial term "meet" should match "meeting"
- let response =
- search_chat_names(&pool, &user_id, &[], "meet".to_string(), false, 10, None).await?;
+ let response = search_chat_names(
+ &pool,
+ &user_id,
+ &[],
+ "meet".to_string(),
+ false,
+ &[],
+ 10,
+ None,
+ )
+ .await?;
assert_eq!(response.items.len(), 2);
assert_eq!(
@@ -385,8 +462,17 @@ async fn test_search_chat_names_user_isolation(pool: Pool<Postgres>) -> anyhow::
.unwrap();
// Search for "User2" - user1 should not see user2's private chats
- let response =
- search_chat_names(&pool, &user_id, &[], "User2".to_string(), false, 10, None).await?;
+ let response = search_chat_names(
+ &pool,
+ &user_id,
+ &[],
+ "User2".to_string(),
+ false,
+ &[],
+ 10,
+ None,
+ )
+ .await?;
// Should return 0 results (user2's chats are not owned by user1 and not shared)
assert_eq!(response.items.len(), 0);
@@ -404,8 +490,17 @@ async fn test_search_chat_names_excludes_soft_deleted(pool: Pool<Postgres>) -> a
.unwrap();
// Search for "project" - should NOT include soft-deleted chat
- let response =
- search_chat_names(&pool, &user_id, &[], "project".to_string(), false, 10, None).await?;
+ let response = search_chat_names(
+ &pool,
+ &user_id,
+ &[],
+ "project".to_string(),
+ false,
+ &[],
+ 10,
+ None,
+ )
+ .await?;
// Should NOT include the deleted chat (id: 77777777-7777-7777-7777-777777777777)
assert!(!response.items.iter().any(|r| {
@@ -444,6 +539,7 @@ async fn test_search_chat_names_rejects_thread_cursor(pool: Pool<Postgres>) -> a
&[],
"test".to_string(),
false,
+ &[],
10,
Some(cursor),
)
@@ -456,3 +552,91 @@ async fn test_search_chat_names_rejects_thread_cursor(pool: Pool<Postgres>) -> a
Ok(())
}
+
+#[sqlx::test(
+ migrator = "MACRO_DB_MIGRATIONS",
+ fixtures(path = "../../fixtures", scripts("chat", "chat_tags"))
+)]
+async fn test_search_chat_names_tag_filter(pool: Pool<Postgres>) -> anyhow::Result<()> {
+ let user_id = MacroUserId::parse_from_str("macro|[email protected]")
+ .map(|l| l.lowercase())
+ .unwrap();
+
+ // Only the tagged chat matches when a held option id is passed.
+ let tagged = vec!["cccccccc-0000-0000-0000-000000000001".to_string()];
+ let response = search_chat_names(
+ &pool,
+ &user_id,
+ &[],
+ "project".to_string(),
+ false,
+ &tagged,
+ 10,
+ None,
+ )
+ .await?;
+
+ assert_eq!(response.items.len(), 1);
+ assert_eq!(
+ response.items[0].entity_id.to_string(),
+ "22222222-2222-2222-2222-222222222222"
+ );
+
+ // An option id no chat holds filters everything out.
+ let unheld = vec!["cccccccc-0000-0000-0000-000000000099".to_string()];
+ let response = search_chat_names(
+ &pool,
+ &user_id,
+ &[],
+ "project".to_string(),
+ false,
+ &unheld,
+ 10,
+ None,
+ )
+ .await?;
+
+ assert!(response.items.is_empty());
+
+ Ok(())
+}
+
+#[sqlx::test(
+ migrator = "MACRO_DB_MIGRATIONS",
+ fixtures(path = "../../fixtures", scripts("chat", "chat_tags"))
+)]
+async fn test_search_chat_names_tag_filter_ids_only(pool: Pool<Postgres>) -> anyhow::Result<()> {
+ let user_id = MacroUserId::parse_from_str("macro|[email protected]")
+ .map(|l| l.lowercase())
+ .unwrap();
+
+ let chat_ids = vec![
+ Uuid::parse_str("11111111-1111-1111-1111-111111111111")?,
+ Uuid::parse_str("22222222-2222-2222-2222-222222222222")?,
+ ];
+
+ // Any of the passed option ids matches (OR semantics).
+ let tagged = vec![
+ "cccccccc-0000-0000-0000-000000000002".to_string(),
+ "cccccccc-0000-0000-0000-000000000099".to_string(),
+ ];
+ let response = search_chat_names(
+ &pool,
+ &user_id,
+ &chat_ids,
+ "project".to_string(),
+ true,
+ &tagged,
+ 10,
+ None,
+ )
+ .await?;
+
+ assert_eq!(response.items.len(), 1);
+ assert_eq!(
+ response.items[0].entity_id.to_string(),
+ "22222222-2222-2222-2222-222222222222"
+ );
+
+ Ok(())
+}
diff --git a/rust/cloud-storage/name_search/src/lib.rs b/rust/cloud-storage/name_search/src/lib.rs
index af00aa1366..0cec3f97e0 100644
--- a/rust/cloud-storage/name_search/src/lib.rs
+++ b/rust/cloud-storage/name_search/src/lib.rs
@@ -4,14 +4,12 @@
mod chat;
mod document;
mod highlight;
-mod project;
pub use chat::search_chat_names;
pub use document::search_document_names;
pub use highlight::highlight_name;
pub use models_opensearch::SearchEntityType;
use models_search_cursor::{PaginatedResult, SearchCursorAttributes};
-pub use project::search_project_names;
/// Escapes special regex characters in a search term
pub fn escape_regex(term: &str) -> String {
let special_chars = [
diff --git a/rust/cloud-storage/name_search/src/project.rs b/rust/cloud-storage/name_search/src/project.rs
deleted file mode 100644
index d8d75ad774..0000000000
--- a/rust/cloud-storage/name_search/src/project.rs
+++ /dev/null
@@ -1,210 +0,0 @@
-//! This module contains logic for searching projects by name
-
-#[cfg(not(test))]
-use cached::proc_macro::cached;
-use chrono::{DateTime, Utc};
-use macro_user_id::{lowercased::Lowercase, user_id::MacroUserId};
-use models_search_cursor::{SearchCursorOption, SearchMethodCursor};
-use sqlx::{Pool, Postgres};
-use uuid::Uuid;
-
-use crate::{NameSearchError, NameSearchResult, PaginatedResult, SearchEntityType, escape_regex};
-
-/// Searches projects by IDs only
-async fn ids_search(
- db: &Pool<Postgres>,
- project_ids: &[Uuid],
- search_pattern: String,
- highlight_pattern: String,
- limit: u32,
- cursor: Option<SearchMethodCursor>,
-) -> Result<PaginatedResult<NameSearchResult>, NameSearchError> {
- if project_ids.is_empty() {
- return Err(NameSearchError::EmptyIdsWithIdsOnly);
- }
-
- let (cursor_updated_at, cursor_entity_id) = cursor
- .as_ref()
- .and_then(|c| c.as_updated_at())
- .map(|(id, ts)| (Some(ts), Some(id.to_string())))
- .unwrap_or((None, None));
-
- // Fetch limit + 1 to determine if there are more results
- let fetch_limit = limit as i64 + 1;
-
- let rows = sqlx::query!(
- r#"
- SELECT
- p.id as entity_id,
- p.name,
- regexp_replace(
- p.name,
- $6,
- '<macro_em>\1</macro_em>',
- 'gi'
- ) as name_highlighted,
- p."updatedAt" as updated_at
- FROM "Project" p
- WHERE p.id = ANY($1)
- AND p."deletedAt" IS NULL
- AND p.name ILIKE $2
- AND (
- $4::timestamptz IS NULL
- OR (p."updatedAt", p.id) < ($4, $5)
- )
- ORDER BY p."updatedAt" DESC, p.id DESC
- LIMIT $3
- "#,
- &project_ids
- .iter()
- .map(|id| id.to_string())
- .collect::<Vec<String>>(),
- search_pattern,
- fetch_limit,
- cursor_updated_at,
- cursor_entity_id,
- highlight_pattern,
- )
- .fetch_all(db)
- .await
- .map_err(NameSearchError::DatabaseError)?;
-
- let results: Vec<NameSearchResult> = rows
- .into_iter()
- .map(|row| NameSearchResult {
- entity_id: row.entity_id.parse().unwrap(),
- entity_type: SearchEntityType::Projects,
- name: row.name_highlighted.unwrap_or(row.name),
- updated_at: DateTime::<Utc>::from_naive_utc_and_offset(row.updated_at, Utc),
- })
- .collect();
-
- Ok(SearchCursorOption::paginate(results, limit as usize))
-}
-
-/// Searches projects by owner or IDs
-async fn owner_search<'a>(
- db: &Pool<Postgres>,
- macro_user_id: &MacroUserId<Lowercase<'a>>,
- project_ids: &[Uuid],
- search_pattern: String,
- highlight_pattern: String,
- limit: u32,
- cursor: Option<SearchMethodCursor>,
-) -> Result<PaginatedResult<NameSearchResult>, NameSearchError> {
- let (cursor_updated_at, cursor_entity_id) = cursor
- .as_ref()
- .and_then(|c| c.as_updated_at())
- .map(|(id, ts)| (Some(ts), Some(id.to_string())))
- .unwrap_or((None, None));
-
- // Fetch limit + 1 to determine if there are more results
- let fetch_limit = limit as i64 + 1;
-
- let rows = sqlx::query!(
- r#"
- SELECT
- p.id as entity_id,
- p.name,
- regexp_replace(
- p.name,
- $7,
- '<macro_em>\1</macro_em>',
- 'gi'
- ) as name_highlighted,
- p."updatedAt" as updated_at
- FROM "Project" p
- WHERE (p."userId" = $1 OR p.id = ANY($2))
- AND p."deletedAt" IS NULL
- AND p.name ILIKE $3
- AND (
- $5::timestamptz IS NULL
- OR (p."updatedAt", p.id) < ($5, $6)
- )
- ORDER BY p."updatedAt" DESC, p.id DESC
- LIMIT $4
- "#,
- macro_user_id.as_ref(),
- &project_ids
- .iter()
- .map(|id| id.to_string())
- .collect::<Vec<String>>(),
- search_pattern,
- fetch_limit,
- cursor_updated_at,
- cursor_entity_id,
- highlight_pattern,
- )
- .fetch_all(db)
- .await
- .map_err(NameSearchError::DatabaseError)?;
-
- let results: Vec<NameSearchResult> = rows
- .into_iter()
- .map(|row| NameSearchResult {
- entity_id: row.entity_id.parse().unwrap(),
- entity_type: SearchEntityType::Projects,
- name: row.name_highlighted.unwrap_or(row.name),
- updated_at: DateTime::<Utc>::from_naive_utc_and_offset(row.updated_at, Utc),
- })
- .collect();
-
- Ok(SearchCursorOption::paginate(results, limit as usize))
-}
-
-/// Searches over the user's projects by name
-#[tracing::instrument(skip(db), err)]
-#[cfg_attr(
- not(test),
- cached(
- time = 30,
- result = true,
- key = "String",
- convert = r#"{ format!("{}-{:?}-{}-{}-{}-{}", macro_user_id.as_ref(), project_ids, term, ids_only, limit, cursor.as_ref().and_then(|c| c.as_updated_at()).map(|(id, ts)| format!("{}-{}", id, ts)).unwrap_or_default()) }"#
- )
-)]
-pub async fn search_project_names<'a>(
- db: &Pool<Postgres>,
- macro_user_id: &MacroUserId<Lowercase<'a>>,
- project_ids: &[Uuid],
- term: String,
- ids_only: bool,
- limit: u32,
- cursor: Option<SearchMethodCursor>,
-) -> Result<PaginatedResult<NameSearchResult>, NameSearchError> {
- if term.is_empty() {
- return Err(NameSearchError::EmptySearchTerm);
- }
- if cursor.as_ref().is_some_and(|c| c.as_updated_at().is_none()) {
- return Err(NameSearchError::IncompatibleCursor);
- }
-
- let search_pattern = format!("%{term}%");
- let highlight_pattern = format!("({})", escape_regex(&term));
-
- if ids_only {
- ids_search(
- db,
- project_ids,
- search_pattern,
- highlight_pattern,
- limit,
- cursor,
- )
- .await
- } else {
- owner_search(
- db,
- macro_user_id,
- project_ids,
- search_pattern,
- highlight_pattern,
- limit,
- cursor,
- )
- .await
- }
-}
-
-#[cfg(test)]
-mod test;
diff --git a/rust/cloud-storage/name_search/src/project/test.rs b/rust/cloud-storage/name_search/src/project/test.rs
deleted file mode 100644
index 844caa6de8..0000000000
--- a/rust/cloud-storage/name_search/src/project/test.rs
+++ /dev/null
@@ -1,456 +0,0 @@
-//! Tests for project module
-
-use macro_db_migrator::MACRO_DB_MIGRATIONS;
-use models_search_cursor::{SearchCursorOption, SearchMethodCursor};
-use sqlx::{Pool, Postgres};
-use uuid::Uuid;
-
-use super::*;
-
-#[sqlx::test(
- migrator = "MACRO_DB_MIGRATIONS",
- fixtures(path = "../../fixtures", scripts("project"))
-)]
-async fn test_search_project_names_empty_term(pool: Pool<Postgres>) -> anyhow::Result<()> {
- let user_id = MacroUserId::parse_from_str("macro|[email protected]")
- .map(|l| l.lowercase())
- .unwrap();
-
- let result = search_project_names(&pool, &user_id, &[], "".to_string(), false, 10, None).await;
-
- assert!(result.is_err());
- assert!(matches!(
- result.unwrap_err(),
- NameSearchError::EmptySearchTerm
- ));
-
- Ok(())
-}
-
-#[sqlx::test(
- migrator = "MACRO_DB_MIGRATIONS",
- fixtures(path = "../../fixtures", scripts("project"))
-)]
-async fn test_search_project_names_ids_only_with_empty_ids(
- pool: Pool<Postgres>,
-) -> anyhow::Result<()> {
- let user_id = MacroUserId::parse_from_str("macro|[email protected]")
- .map(|l| l.lowercase())
- .unwrap();
-
- let result =
- search_project_names(&pool, &user_id, &[], "mobile".to_string(), true, 10, None).await;
-
- assert!(result.is_err());
- assert!(matches!(
- result.unwrap_err(),
- NameSearchError::EmptyIdsWithIdsOnly
- ));
-
- Ok(())
-}
-
-#[sqlx::test(
- migrator = "MACRO_DB_MIGRATIONS",
- fixtures(path = "../../fixtures", scripts("project"))
-)]
-async fn test_search_project_names_ids_only_mode(pool: Pool<Postgres>) -> anyhow::Result<()> {
- let user_id = MacroUserId::parse_from_str("macro|[email protected]")
- .map(|l| l.lowercase())
- .unwrap();
-
- // Search for "mobile" within specific project IDs
- let project_ids = vec![
- Uuid::parse_str("11111111-1111-1111-1111-111111111111")?,
- Uuid::parse_str("22222222-2222-2222-2222-222222222222")?,
- Uuid::parse_str("55555555-5555-5555-5555-555555555555")?, // Marketing Campaign - won't match
- ];
-
- let response = search_project_names(
- &pool,
- &user_id,
- &project_ids,
- "mobile".to_string(),
- true,
- 10,
- None,
- )
- .await?;
-
- // Should only return the 2 projects that match "mobile" from the provided IDs
- assert_eq!(response.items.len(), 2);
-
- // Verify results contain expected projects (ordered by updatedAt DESC)
- assert_eq!(
- response.items[0].entity_id.to_string(),
- "22222222-2222-2222-2222-222222222222"
- );
- assert_eq!(response.items[0].entity_type, SearchEntityType::Projects);
- assert_eq!(
- response.items[0].name,
- "<macro_em>Mobile</macro_em> App Redesign"
- );
-
- assert_eq!(
- response.items[1].entity_id.to_string(),
- "11111111-1111-1111-1111-111111111111"
- );
- assert_eq!(
- response.items[1].name,
- "<macro_em>Mobile</macro_em> Development"
- );
-
- Ok(())
-}
-
-#[sqlx::test(
- migrator = "MACRO_DB_MIGRATIONS",
- fixtures(path = "../../fixtures", scripts("project"))
-)]
-async fn test_search_project_names_normal_mode_owned_projects(
- pool: Pool<Postgres>,
-) -> anyhow::Result<()> {
- let user_id = MacroUserId::parse_from_str("macro|[email protected]")
- .map(|l| l.lowercase())
- .unwrap();
-
- // Search for "mobile" across all user1's owned projects
- let response =
- search_project_names(&pool, &user_id, &[], "mobile".to_string(), false, 10, None).await?;
-
- // Should return 3 projects matching "mobile" (2 lowercase + 1 uppercase)
- assert_eq!(response.items.len(), 3);
-
- // Verify ordering by updatedAt DESC
- assert_eq!(
- response.items[0].entity_id.to_string(),
- "66666666-6666-6666-6666-666666666666"
- );
- assert_eq!(
- response.items[0].name,
- "<macro_em>MOBILE</macro_em> PLATFORM"
- );
-
- assert_eq!(
- response.items[1].entity_id.to_string(),
- "22222222-2222-2222-2222-222222222222"
- );
- assert_eq!(
- response.items[1].name,
- "<macro_em>Mobile</macro_em> App Redesign"
- );
-
- Ok(())
-}
-
-#[sqlx::test(
- migrator = "MACRO_DB_MIGRATIONS",
- fixtures(path = "../../fixtures", scripts("project"))
-)]
-async fn test_search_project_names_case_insensitive(pool: Pool<Postgres>) -> anyhow::Result<()> {
- let user_id = MacroUserId::parse_from_str("macro|[email protected]")
- .map(|l| l.lowercase())
- .unwrap();
-
- // Search with uppercase term should match both lowercase and uppercase names
- let response =
- search_project_names(&pool, &user_id, &[], "MOBILE".to_string(), false, 10, None).await?;
-
- assert_eq!(response.items.len(), 3);
-
- // Search with lowercase term should also match both
- let response =
- search_project_names(&pool, &user_id, &[], "mobile".to_string(), false, 10, None).await?;
-
- assert_eq!(response.items.len(), 3);
-
- // Search with mixed case
- let response =
- search_project_names(&pool, &user_id, &[], "MoBiLe".to_string(), false, 10, None).await?;
-
- assert_eq!(response.items.len(), 3);
-
- Ok(())
-}
-
-#[sqlx::test(
- migrator = "MACRO_DB_MIGRATIONS",
- fixtures(path = "../../fixtures", scripts("project"))
-)]
-async fn test_search_project_names_with_shared_projects(
- pool: Pool<Postgres>,
-) -> anyhow::Result<()> {
- let user_id = MacroUserId::parse_from_str("macro|[email protected]")
- .map(|l| l.lowercase())
- .unwrap();
-
- // Include a project from user3 (shared via project_ids parameter)
- let shared_project_ids = vec![Uuid::parse_str("99999999-9999-9999-9999-999999999999")?];
-
- let response = search_project_names(
- &pool,
- &user_id,
- &shared_project_ids,
- "mobile".to_string(),
- false,
- 10,
- None,
- )
- .await?;
-
- // Should return user1's 3 "mobile" projects + user3's 1 "mobile" project
- assert_eq!(response.items.len(), 4);
-
- // Verify user3's project is included
- let user3_project = response.items.iter().find(|r| {
- r.entity_id
- .to_string()
- .eq("99999999-9999-9999-9999-999999999999")
- });
- assert!(user3_project.is_some());
- assert_eq!(
- user3_project.unwrap().name,
- "User3 Shared <macro_em>Mobile</macro_em>"
- );
-
- Ok(())
-}
-
-#[sqlx::test(
- migrator = "MACRO_DB_MIGRATIONS",
- fixtures(path = "../../fixtures", scripts("project"))
-)]
-async fn test_search_project_names_pagination_limit(pool: Pool<Postgres>) -> anyhow::Result<()> {
- let user_id = MacroUserId::parse_from_str("macro|[email protected]")
- .map(|l| l.lowercase())
- .unwrap();
-
- // Search with limit of 2
- let response =
- search_project_names(&pool, &user_id, &[], "mobile".to_string(), false, 2, None).await?;
-
- assert_eq!(response.items.len(), 2);
-
- // Should get the 2 most recently updated projects with "mobile"
- assert_eq!(
- response.items[0].entity_id.to_string(),
- "66666666-6666-6666-6666-666666666666"
- );
- assert_eq!(
- response.items[1].entity_id.to_string(),
- "22222222-2222-2222-2222-222222222222"
- );
-
- // Should have a next_cursor since there are more results
- assert!(response.cursor.has_more());
-
- Ok(())
-}
-
-#[sqlx::test(
- migrator = "MACRO_DB_MIGRATIONS",
- fixtures(path = "../../fixtures", scripts("project"))
-)]
-async fn test_search_project_names_pagination_cursor(pool: Pool<Postgres>) -> anyhow::Result<()> {
- let user_id = MacroUserId::parse_from_str("macro|[email protected]")
- .map(|l| l.lowercase())
- .unwrap();
-
- // First page with limit of 2
- let first_response =
- search_project_names(&pool, &user_id, &[], "mobile".to_string(), false, 2, None).await?;
-
- assert_eq!(first_response.items.len(), 2);
- assert!(first_response.cursor.has_more());
-
- // Extract cursor for second page
- let cursor = match first_response.cursor {
- SearchCursorOption::NotDone(c) => c,
- SearchCursorOption::Done => panic!("Expected more results"),
- };
-
- // Second page using cursor
- let second_response =
- search_project_names(&pool, &user_id, &[], "mobile".to_string(), false, 2, cursor).await?;
-
- assert_eq!(second_response.items.len(), 1);
-
- // Should get the next project (skipping the first 2)
- assert_eq!(
- second_response.items[0].entity_id.to_string(),
- "11111111-1111-1111-1111-111111111111"
- );
-
- // Should NOT have next_cursor since we've reached the end
- assert!(second_response.cursor.is_done());
-
- // Verify no overlap between pages
- let first_ids: Vec<String> = first_response
- .items
- .iter()
- .map(|r| r.entity_id.to_string())
- .collect();
- let second_ids: Vec<String> = second_response
- .items
- .iter()
- .map(|r| r.entity_id.to_string())
- .collect();
-
- for id in &first_ids {
- assert!(
- !second_ids.contains(id),
- "Found duplicate ID between pages: {}",
- id
- );
- }
-
- Ok(())
-}
-
-#[sqlx::test(
- migrator = "MACRO_DB_MIGRATIONS",
- fixtures(path = "../../fixtures", scripts("project"))
-)]
-async fn test_search_project_names_no_results(pool: Pool<Postgres>) -> anyhow::Result<()> {
- let user_id = MacroUserId::parse_from_str("macro|[email protected]")
- .map(|l| l.lowercase())
- .unwrap();
-
- // Search for a term that doesn't match any projects
- let response = search_project_names(
- &pool,
- &user_id,
- &[],
- "nonexistent".to_string(),
- false,
- 10,
- None,
- )
- .await?;
-
- assert_eq!(response.items.len(), 0);
- assert!(response.cursor.is_done());
-
- Ok(())
-}
-
-#[sqlx::test(
- migrator = "MACRO_DB_MIGRATIONS",
- fixtures(path = "../../fixtures", scripts("project"))
-)]
-async fn test_search_project_names_partial_match(pool: Pool<Postgres>) -> anyhow::Result<()> {
- let user_id = MacroUserId::parse_from_str("macro|[email protected]")
- .map(|l| l.lowercase())
- .unwrap();
-
- // Search for partial term "web" should match "website"
- let response =
- search_project_names(&pool, &user_id, &[], "web".to_string(), false, 10, None).await?;
-
- assert_eq!(response.items.len(), 2);
- assert_eq!(
- response.items[0].entity_id.to_string(),
- "44444444-4444-4444-4444-444444444444"
- );
- assert_eq!(
- response.items[0].name,
- "<macro_em>Web</macro_em>site Optimization"
- );
-
- assert_eq!(
- response.items[1].entity_id.to_string(),
- "33333333-3333-3333-3333-333333333333"
- );
- assert_eq!(
- response.items[1].name,
- "<macro_em>Web</macro_em> Platform Upgrade"
- );
-
- Ok(())
-}
-
-#[sqlx::test(
- migrator = "MACRO_DB_MIGRATIONS",
- fixtures(path = "../../fixtures", scripts("project"))
-)]
-async fn test_search_project_names_user_isolation(pool: Pool<Postgres>) -> anyhow::Result<()> {
- let user_id = MacroUserId::parse_from_str("macro|[email protected]")
- .map(|l| l.lowercase())
- .unwrap();
-
- // Search for "User2" - user1 should not see user2's private projects
- let response =
- search_project_names(&pool, &user_id, &[], "User2".to_string(), false, 10, None).await?;
-
- // Should return 0 results (user2's projects are not owned by user1 and not shared)
- assert_eq!(response.items.len(), 0);
-
- Ok(())
-}
-
-#[sqlx::test(
- migrator = "MACRO_DB_MIGRATIONS",
- fixtures(path = "../../fixtures", scripts("project"))
-)]
-async fn test_search_project_names_excludes_soft_deleted(
- pool: Pool<Postgres>,
-) -> anyhow::Result<()> {
- let user_id = MacroUserId::parse_from_str("macro|[email protected]")
- .map(|l| l.lowercase())
- .unwrap();
-
- // Search for "mobile" - should NOT include soft-deleted project
- let response =
- search_project_names(&pool, &user_id, &[], "mobile".to_string(), false, 10, None).await?;
-
- // Should NOT include the deleted project (id: 77777777-7777-7777-7777-777777777777)
- assert!(!response.items.iter().any(|r| {
- r.entity_id
- .to_string()
- .eq("77777777-7777-7777-7777-777777777777")
- }));
-
- // Should include non-deleted projects
- assert!(response.items.iter().any(|r| {
- r.entity_id
- .to_string()
- .eq("11111111-1111-1111-1111-111111111111")
- }));
-
- Ok(())
-}
-
-#[sqlx::test(
- migrator = "MACRO_DB_MIGRATIONS",
- fixtures(path = "../../fixtures", scripts("project"))
-)]
-async fn test_search_project_names_rejects_thread_cursor(
- pool: Pool<Postgres>,
-) -> anyhow::Result<()> {
- let user_id = MacroUserId::parse_from_str("macro|[email protected]")
- .map(|l| l.lowercase())
- .unwrap();
-
- let cursor = SearchMethodCursor::Thread {
- thread_id: Uuid::new_v4(),
- message_id: Uuid::new_v4(),
- };
-
- let result = search_project_names(
- &pool,
- &user_id,
- &[],
- "test".to_string(),
- false,
- 10,
- Some(cursor),
- )
- .await;
-
- assert!(matches!(
- result.unwrap_err(),
- NameSearchError::IncompatibleCursor
- ));
-
- Ok(())
-}
diff --git a/rust/cloud-storage/opensearch_client/src/chat.rs b/rust/cloud-storage/opensearch_client/src/chat.rs
index 46a0447ba8..682e8ac83e 100644
--- a/rust/cloud-storage/opensearch_client/src/chat.rs
+++ b/rust/cloud-storage/opensearch_client/src/chat.rs
@@ -1,6 +1,6 @@
use crate::{
OpensearchClient, Result, delete,
- upsert::{self, chat_message::UpsertChatMessageArgs},
+ upsert::{self, chat_message::UpsertChatMessageArgs, properties::IndexedProperty},
};
impl OpensearchClient {
@@ -47,6 +47,16 @@ impl OpensearchClient {
upsert::chat_message::update_chat_metadata(&self.inner, chat_id, title).await
}
+ /// Refresh only the denormalized `properties` on an existing parent chat doc.
+ #[tracing::instrument(skip(self, properties), err)]
+ pub async fn update_chat_properties(
+ &self,
+ chat_id: &str,
+ properties: &[IndexedProperty],
+ ) -> Result<()> {
+ upsert::chat_message::update_chat_properties(&self.inner, chat_id, properties).await
+ }
+
pub async fn delete_chat_by_user_id(&self, user_id: &str) -> Result<()> {
delete::chat::delete_chat_by_user_id(&self.inner, user_id).await
}
diff --git a/rust/cloud-storage/opensearch_client/src/delete/mod.rs b/rust/cloud-storage/opensearch_client/src/delete/mod.rs
index ee9171dd5a..01bd3928de 100644
--- a/rust/cloud-storage/opensearch_client/src/delete/mod.rs
+++ b/rust/cloud-storage/opensearch_client/src/delete/mod.rs
@@ -3,3 +3,4 @@ pub mod channel;
pub mod chat;
pub mod document;
pub mod email;
+pub mod project;
diff --git a/rust/cloud-storage/opensearch_client/src/delete/project.rs b/rust/cloud-storage/opensearch_client/src/delete/project.rs
new file mode 100644
index 0000000000..9c3a3f18bc
--- /dev/null
+++ b/rust/cloud-storage/opensearch_client/src/delete/project.rs
@@ -0,0 +1,59 @@
+use models_opensearch::SearchIndex;
+
+use crate::{Result, error::OpensearchClientError};
+
+/// Deletes a project doc by id. The projects index is flat with `_id` =
+/// project id, so this is a direct delete. A missing-doc 404 is a no-op —
+/// the project was never indexed or is already gone. A missing-index 404
+/// (`index_not_found_exception`) is a real outage and must propagate.
+#[tracing::instrument(skip(client), err)]
+pub async fn delete_project_by_id(
+ client: &opensearch::OpenSearch,
+ project_id: &str,
+ index_override: Option<&str>,
+) -> Result<()> {
+ let index = index_override.unwrap_or(SearchIndex::Projects.as_ref());
+ let response = client
+ .delete(opensearch::DeleteParts::IndexId(index, project_id))
+ .send()
+ .await
+ .map_err(|err| OpensearchClientError::Unknown {
+ details: err.to_string(),
+ method: Some("delete_project_by_id".to_string()),
+ })?;
+
+ let status_code = response.status_code();
+ if status_code.is_success() {
+ return Ok(());
+ }
+
+ let body =
+ response
+ .text()
+ .await
+ .map_err(|err| OpensearchClientError::DeserializationFailed {
+ details: err.to_string(),
+ method: Some("delete_project_by_id".to_string()),
+ })?;
+
+ if status_code.as_u16() == 404 {
+ let result = serde_json::from_str::<serde_json::Value>(&body)
+ .ok()
+ .and_then(|value| value["result"].as_str().map(str::to_owned));
+ if result.as_deref() == Some("not_found") {
+ tracing::debug!(project_id=%project_id, "project not indexed; nothing to delete");
+ return Ok(());
+ }
+ }
+
+ tracing::error!(
+ status_code = ?status_code,
+ body = ?body,
+ "error deleting project by id"
+ );
+
+ Err(OpensearchClientError::Unknown {
+ details: body,
+ method: Some("delete_project_by_id".to_string()),
+ })
+}
diff --git a/rust/cloud-storage/opensearch_client/src/document.rs b/rust/cloud-storage/opensearch_client/src/document.rs
index 636624092a..608255f659 100644
--- a/rust/cloud-storage/opensearch_client/src/document.rs
+++ b/rust/cloud-storage/opensearch_client/src/document.rs
@@ -1,9 +1,6 @@
use crate::{
OpensearchClient, Result, delete,
- upsert::{
- self,
- document::{IndexedProperty, UpsertDocumentArgs},
- },
+ upsert::{self, document::UpsertDocumentArgs, properties::IndexedProperty},
};
impl OpensearchClient {
diff --git a/rust/cloud-storage/opensearch_client/src/email.rs b/rust/cloud-storage/opensearch_client/src/email.rs
index e8928b3247..d913d188d7 100644
--- a/rust/cloud-storage/opensearch_client/src/email.rs
+++ b/rust/cloud-storage/opensearch_client/src/email.rs
@@ -1,6 +1,6 @@
use crate::{
OpensearchClient, Result, delete,
- upsert::{self, BulkUpsertResult, email::UpsertEmailArgs},
+ upsert::{self, BulkUpsertResult, email::UpsertEmailArgs, properties::IndexedProperty},
};
impl OpensearchClient {
@@ -20,6 +20,17 @@ impl OpensearchClient {
upsert::email::bulk_upsert_email_messages(&self.inner, messages, index_override).await
}
+ /// Refresh only the denormalized `properties` on every message doc of a
+ /// thread.
+ #[tracing::instrument(skip(self, properties), err)]
+ pub async fn update_email_thread_properties(
+ &self,
+ thread_id: &str,
+ properties: &[IndexedProperty],
+ ) -> Result<()> {
+ upsert::email::update_email_thread_properties(&self.inner, thread_id, properties).await
+ }
+
/// Deletes a particular email message
pub async fn delete_email_message_by_id(&self, message_id: &str) -> Result<()> {
delete::email::delete_email_message_by_id(&self.inner, message_id).await
diff --git a/rust/cloud-storage/opensearch_client/src/lib.rs b/rust/cloud-storage/opensearch_client/src/lib.rs
index 516617dc29..401ff01aba 100644
--- a/rust/cloud-storage/opensearch_client/src/lib.rs
+++ b/rust/cloud-storage/opensearch_client/src/lib.rs
@@ -10,6 +10,7 @@ pub mod channel_message;
pub mod chat;
pub mod document;
pub mod email;
+pub mod project;
pub type Result<T> = std::result::Result<T, error::OpensearchClientError>;
diff --git a/rust/cloud-storage/opensearch_client/src/project.rs b/rust/cloud-storage/opensearch_client/src/project.rs
new file mode 100644
index 0000000000..e157455b4b
--- /dev/null
+++ b/rust/cloud-storage/opensearch_client/src/project.rs
@@ -0,0 +1,35 @@
+use crate::{
+ OpensearchClient, Result, delete,
+ upsert::{self, project::UpsertProjectArgs, properties::IndexedProperty},
+};
+
+impl OpensearchClient {
+ #[tracing::instrument(skip(self, upsert_project_args), fields(project_id=%upsert_project_args.project_id), err)]
+ pub async fn upsert_project(
+ &self,
+ upsert_project_args: &UpsertProjectArgs,
+ index_override: Option<&str>,
+ ) -> Result<()> {
+ upsert::project::upsert_project(&self.inner, upsert_project_args, index_override).await
+ }
+
+ /// Deletes a project from the opensearch projects index
+ #[tracing::instrument(skip(self), err)]
+ pub async fn delete_project(
+ &self,
+ project_id: &str,
+ index_override: Option<&str>,
+ ) -> Result<()> {
+ delete::project::delete_project_by_id(&self.inner, project_id, index_override).await
+ }
+
+ /// Updates only the denormalized `properties` on an indexed project
+ #[tracing::instrument(skip(self, properties), err)]
+ pub async fn update_project_properties(
+ &self,
+ project_id: &str,
+ properties: &[IndexedProperty],
+ ) -> Result<()> {
+ upsert::project::update_project_properties(&self.inner, project_id, properties, None).await
+ }
+}
diff --git a/rust/cloud-storage/opensearch_client/src/search/builder.rs b/rust/cloud-storage/opensearch_client/src/search/builder.rs
index db4e4555a4..eaa19fc07e 100644
--- a/rust/cloud-storage/opensearch_client/src/search/builder.rs
+++ b/rust/cloud-storage/opensearch_client/src/search/builder.rs
@@ -198,7 +198,10 @@ impl<T: SearchQueryConfig> SearchQueryBuilder<T> {
/// access to/has requested.
/// This could either be a single term/terms query for ids_only or just user_id
/// Or a bool query that contains both of these items
- fn build_filter_query<'a>(&'a self, user_id_key: Option<&str>) -> Result<QueryType<'a>> {
+ pub(crate) fn build_filter_query<'a>(
+ &'a self,
+ user_id_key: Option<&str>,
+ ) -> Result<QueryType<'a>> {
if self.ids_only {
// We only need to search over the entity ids provided
if self.ids.is_empty() {
diff --git a/rust/cloud-storage/opensearch_client/src/search/chats.rs b/rust/cloud-storage/opensearch_client/src/search/chats.rs
index 5a0cdf8e74..9db01afef5 100644
--- a/rust/cloud-storage/opensearch_client/src/search/chats.rs
+++ b/rust/cloud-storage/opensearch_client/src/search/chats.rs
@@ -5,6 +5,7 @@ use crate::{
search::{
builder::{SearchQueryBuilder, SearchQueryConfig},
model::{Highlight, SearchGotoChat, SearchGotoContent, SearchHit, parse_highlight_hit},
+ properties::build_tag_filter,
query::Keys,
utils::should_wildcard_field_query_builder,
},
@@ -55,6 +56,7 @@ pub(crate) struct ChatQueryBuilder {
inner: SearchQueryBuilder<ChatSearchConfig>,
/// The role of the chat message
role: Vec<String>,
+ tag_option_ids: Vec<String>,
}
impl ChatQueryBuilder {
@@ -62,6 +64,7 @@ impl ChatQueryBuilder {
Self {
inner: SearchQueryBuilder::new(terms),
role: Vec::new(),
+ tag_option_ids: Vec::new(),
}
}
@@ -81,6 +84,11 @@ impl ChatQueryBuilder {
self
}
+ pub fn tag_option_ids(mut self, tag_option_ids: Vec<String>) -> Self {
+ self.tag_option_ids = tag_option_ids;
+ self
+ }
+
/// Parent/child join query: one `has_child` clause per term, ANDed
/// inside `bool.must` so each term must match some message in the
/// same chat. Parent metadata filters (user, ids) sit on
@@ -110,6 +118,12 @@ impl ChatQueryBuilder {
// Access control on parent fields (user_id and/or entity_id).
bool_query.filter(self.build_parent_filter()?);
+ // Tag filter: a single nested clause matching any of the option ids in
+ // the parent's `properties.values`, with no definition_id constraint.
+ if let Some(nested) = build_tag_filter(&self.tag_option_ids) {
+ bool_query.filter(nested);
+ }
+
// One has_child clause per term, ANDed via bool.must. Each
// carries its own inner_hits so highlights + message-nav data
// come back alongside the parent. Shared `highlight_query`
@@ -226,6 +240,7 @@ pub struct ChatSearchArgs {
pub role: Vec<String>,
pub collapse: bool,
pub ids_only: bool,
+ pub tag_option_ids: Vec<String>,
}
impl From<ChatSearchArgs> for ChatQueryBuilder {
@@ -239,6 +254,7 @@ impl From<ChatSearchArgs> for ChatQueryBuilder {
.role(args.role)
.collapse(args.collapse)
.ids_only(args.ids_only)
+ .tag_option_ids(args.tag_option_ids)
}
}
diff --git a/rust/cloud-storage/opensearch_client/src/search/chats/test.rs b/rust/cloud-storage/opensearch_client/src/search/chats/test.rs
index dd8e7412e0..118ac40666 100644
--- a/rust/cloud-storage/opensearch_client/src/search/chats/test.rs
+++ b/rust/cloud-storage/opensearch_client/src/search/chats/test.rs
@@ -68,3 +68,59 @@ fn test_build_bool_query_join_shape() -> anyhow::Result<()> {
Ok(())
}
+
+#[test]
+fn test_build_bool_query_tag_filter_emits_nested_terms() -> anyhow::Result<()> {
+ let builder = ChatQueryBuilder::new(vec!["foo".to_string()])
+ .match_type("partial")
+ .user_id("alice")
+ .tag_option_ids(vec![
+ "00000001-0000-0000-0003-000000000001".to_string(),
+ "00000001-0000-0000-0003-000000000002".to_string(),
+ ]);
+
+ let json = builder.build_bool_query()?.build().to_json();
+ let filter = json["bool"]["filter"].as_array().expect("filter array");
+
+ let nested: Vec<&serde_json::Value> = filter
+ .iter()
+ .filter(|f| f.get("nested").is_some())
+ .collect();
+ assert_eq!(
+ nested.len(),
+ 1,
+ "tags collapse to one nested clause: {filter:?}"
+ );
+
+ let tags = &nested[0]["nested"];
+ assert_eq!(tags["path"], "properties");
+ assert_eq!(tags["ignore_unmapped"], true);
+ // No definition_id term: match is on the globally-unique option ids alone.
+ assert_eq!(
+ tags["query"],
+ serde_json::json!({
+ "terms": {"properties.values": [
+ "00000001-0000-0000-0003-000000000001",
+ "00000001-0000-0000-0003-000000000002"
+ ]}
+ })
+ );
+
+ Ok(())
+}
+
+#[test]
+fn test_build_bool_query_tag_filter_empty_skipped() -> anyhow::Result<()> {
+ let builder = ChatQueryBuilder::new(vec!["foo".to_string()])
+ .match_type("partial")
+ .user_id("alice")
+ .tag_option_ids(vec![]);
+
+ let json = builder.build_bool_query()?.build().to_json();
+ let filter = json["bool"]["filter"].as_array().expect("filter array");
+ assert!(
+ !filter.iter().any(|f| f.get("nested").is_some()),
+ "empty tag_option_ids should emit no nested clause: {filter:?}"
+ );
+ Ok(())
+}
diff --git a/rust/cloud-storage/opensearch_client/src/search/documents.rs b/rust/cloud-storage/opensearch_client/src/search/documents.rs
index 4fca9f31f5..104fcc8462 100644
--- a/rust/cloud-storage/opensearch_client/src/search/documents.rs
+++ b/rust/cloud-storage/opensearch_client/src/search/documents.rs
@@ -5,6 +5,7 @@ use crate::{
search::{
builder::{SearchQueryBuilder, SearchQueryConfig},
model::{Highlight, SearchGotoContent, SearchGotoDocument, SearchHit, parse_highlight_hit},
+ properties::{PROPERTIES_PATH, build_tag_filter},
query::Keys,
},
};
@@ -54,9 +55,6 @@ pub enum DocumentSearchMode {
NameContent,
}
-/// Nested path holding denormalized entity properties on the parent doc.
-const PROPERTIES_PATH: &str = "properties";
-
/// A property-equality filter applied against the indexed `properties` nested
/// field. Matches parents that have a nested entry whose `definition_id`
/// equals `definition_id` and whose `values` contains any of `values`.
@@ -289,25 +287,6 @@ fn build_property_filter<'a>(filter: &PropertyFilterArg) -> Option<QueryType<'a>
)
}
-/// Build a `nested` query over `properties` matching parents that have any
-/// nested entry whose `values` contains one of `option_ids`. Unlike
-/// [`build_property_filter`] there is no `definition_id` constraint: tag option
-/// ids are globally unique, so this matches a tag regardless of which
-/// definition owns it. Returns `None` when there are no option ids.
-fn build_tag_filter<'a>(option_ids: &[String]) -> Option<QueryType<'a>> {
- if option_ids.is_empty() {
- return None;
- }
- Some(
- NestedQuery::new(
- PROPERTIES_PATH,
- QueryType::terms(format!("{PROPERTIES_PATH}.values"), option_ids.to_vec()),
- )
- .ignore_unmapped(true)
- .into(),
- )
-}
-
/// Highlight config attached to each `has_child` inner_hits block.
/// Matches the top-level documents highlight (plain highlighter,
/// `<macro_em>` tags, single fragment) so chunk hits come back with
diff --git a/rust/cloud-storage/opensearch_client/src/search/emails.rs b/rust/cloud-storage/opensearch_client/src/search/emails.rs
index 31f2d65bb3..d630b397ff 100644
--- a/rust/cloud-storage/opensearch_client/src/search/emails.rs
+++ b/rust/cloud-storage/opensearch_client/src/search/emails.rs
@@ -2,6 +2,7 @@ use crate::{
Result, delegate_methods,
search::{
builder::{SearchQueryBuilder, SearchQueryConfig},
+ properties::build_tag_filter,
utils::should_wildcard_field_query_builder,
},
};
@@ -115,6 +116,8 @@ pub(crate) struct EmailQueryBuilder {
importance: Option<bool>,
/// When true, only search the subject field (for name-only search mode)
subject_only: bool,
+ /// Tag option ids the thread must carry (OR'd, definition-agnostic)
+ tag_option_ids: Vec<String>,
}
impl EmailQueryBuilder {
@@ -130,6 +133,7 @@ impl EmailQueryBuilder {
exclude_labels: Vec::new(),
importance: None,
subject_only: false,
+ tag_option_ids: Vec::new(),
}
}
@@ -190,6 +194,11 @@ impl EmailQueryBuilder {
self
}
+ pub fn tag_option_ids(mut self, tag_option_ids: Vec<String>) -> Self {
+ self.tag_option_ids = tag_option_ids;
+ self
+ }
+
pub fn build_bool_query<'a>(&'a self) -> Result<BoolQueryBuilder<'a>> {
let mut content_bool_query = self.inner.build_content_bool_query()?;
@@ -254,6 +263,13 @@ impl EmailQueryBuilder {
content_bool_query.filter(QueryType::terms("labels", self.include_labels.clone()));
}
+ // Tag filter: a single nested clause matching any of the option ids in
+ // `properties.values`. Thread-level properties are denormalized onto
+ // every message doc, so filtering per doc filters the thread.
+ if let Some(nested) = build_tag_filter(&self.tag_option_ids) {
+ content_bool_query.filter(nested);
+ }
+
for label in &self.exclude_labels {
content_bool_query.must_not(QueryType::term("labels", label.clone()));
}
@@ -366,6 +382,7 @@ pub struct EmailSearchArgs {
pub collapse: bool,
pub ids_only: bool,
pub subject_only: bool,
+ pub tag_option_ids: Vec<String>,
}
impl From<EmailSearchArgs> for EmailQueryBuilder {
@@ -388,6 +405,7 @@ impl From<EmailSearchArgs> for EmailQueryBuilder {
.collapse(args.collapse)
.ids_only(args.ids_only)
.subject_only(args.subject_only)
+ .tag_option_ids(args.tag_option_ids)
}
}
diff --git a/rust/cloud-storage/opensearch_client/src/search/emails/test.rs b/rust/cloud-storage/opensearch_client/src/search/emails/test.rs
index 0f91d86eaf..150200866e 100644
--- a/rust/cloud-storage/opensearch_client/src/search/emails/test.rs
+++ b/rust/cloud-storage/opensearch_client/src/search/emails/test.rs
@@ -119,6 +119,7 @@ fn test_email_search_args_quoted_phrase_uses_phrase_query_in_sqs() -> anyhow::Re
collapse: true,
ids_only: false,
subject_only: false,
+ tag_option_ids: vec![],
}
.into();
@@ -156,6 +157,7 @@ fn test_email_search_args_build_injects_simple_query_string() -> anyhow::Result<
collapse: true,
ids_only: false,
subject_only: false,
+ tag_option_ids: vec![],
}
.into();
@@ -440,3 +442,59 @@ fn test_importance_false_with_exclude_labels_both_apply() -> anyhow::Result<()>
Ok(())
}
+
+#[test]
+fn test_build_bool_query_tag_filter_emits_flat_terms() -> anyhow::Result<()> {
+ let builder = EmailQueryBuilder::new(vec!["foo".to_string()])
+ .match_type("partial")
+ .user_id("user123")
+ .tag_option_ids(vec![
+ "00000001-0000-0000-0003-000000000001".to_string(),
+ "00000001-0000-0000-0003-000000000002".to_string(),
+ ]);
+
+ let json = builder.build_bool_query()?.build().to_json();
+ let filter = json["bool"]["filter"].as_array().expect("filter array");
+
+ let nested: Vec<&serde_json::Value> = filter
+ .iter()
+ .filter(|f| f.get("nested").is_some())
+ .collect();
+ assert_eq!(
+ nested.len(),
+ 1,
+ "tags collapse to one nested clause: {filter:?}"
+ );
+
+ let tags = &nested[0]["nested"];
+ assert_eq!(tags["path"], "properties");
+ assert_eq!(tags["ignore_unmapped"], true);
+ // No definition_id term: match is on the globally-unique option ids alone.
+ assert_eq!(
+ tags["query"],
+ serde_json::json!({
+ "terms": {"properties.values": [
+ "00000001-0000-0000-0003-000000000001",
+ "00000001-0000-0000-0003-000000000002"
+ ]}
+ })
+ );
+
+ Ok(())
+}
+
+#[test]
+fn test_build_bool_query_tag_filter_empty_skipped() -> anyhow::Result<()> {
+ let builder = EmailQueryBuilder::new(vec!["foo".to_string()])
+ .match_type("partial")
+ .user_id("user123")
+ .tag_option_ids(vec![]);
+
+ let json = builder.build_bool_query()?.build().to_json();
+ let filter = json["bool"]["filter"].as_array().expect("filter array");
+ assert!(
+ !filter.iter().any(|f| f.get("nested").is_some()),
+ "empty tag_option_ids should emit no nested clause: {filter:?}"
+ );
+ Ok(())
+}
diff --git a/rust/cloud-storage/opensearch_client/src/search/mod.rs b/rust/cloud-storage/opensearch_client/src/search/mod.rs
index 9a6220d929..b5015a3227 100644
--- a/rust/cloud-storage/opensearch_client/src/search/mod.rs
+++ b/rust/cloud-storage/opensearch_client/src/search/mod.rs
@@ -5,6 +5,8 @@ pub mod chats;
pub mod documents;
pub mod emails;
pub mod model;
+pub mod projects;
+mod properties;
mod query;
pub mod unified;
mod utils;
diff --git a/rust/cloud-storage/opensearch_client/src/search/projects.rs b/rust/cloud-storage/opensearch_client/src/search/projects.rs
new file mode 100644
index 0000000000..fb5af112be
--- /dev/null
+++ b/rust/cloud-storage/opensearch_client/src/search/projects.rs
@@ -0,0 +1,118 @@
+use crate::{
+ Result, delegate_methods,
+ search::{
+ builder::{SearchQueryBuilder, SearchQueryConfig},
+ properties::build_tag_filter,
+ },
+};
+
+use models_opensearch::OpenSearchEntityType;
+use opensearch_query_builder::{BoolQueryBuilder, QueryType};
+
+#[derive(Clone)]
+pub(crate) struct ProjectSearchConfig;
+
+impl SearchQueryConfig for ProjectSearchConfig {
+ const USER_ID_KEY: Option<&'static str> = Some("owner_id");
+ const TITLE_KEY: &'static str = "name";
+ const ENTITY_INDEX: OpenSearchEntityType = OpenSearchEntityType::Projects;
+}
+
+/// Query builder for the flat projects index. Projects carry no content, so
+/// every mode matches terms against the parent `name` only. Access control
+/// mirrors documents: `owner_id == caller` and/or `entity_id ∈ ids` resolved
+/// from Postgres at query time.
+pub(crate) struct ProjectQueryBuilder {
+ inner: SearchQueryBuilder<ProjectSearchConfig>,
+ tag_option_ids: Vec<String>,
+}
+
+impl ProjectQueryBuilder {
+ pub fn new(terms: Vec<String>) -> Self {
+ Self {
+ inner: SearchQueryBuilder::new(terms),
+ tag_option_ids: Vec::new(),
+ }
+ }
+
+ pub fn tag_option_ids(mut self, tag_option_ids: Vec<String>) -> Self {
+ self.tag_option_ids = tag_option_ids;
+ self
+ }
+
+ // Copy function signature from SearchQueryBuilder
+ delegate_methods! {
+ fn match_type(match_type: &str) -> Self;
+ fn page(page: u32) -> Self;
+ fn page_size(page_size: u32) -> Self;
+ fn user_id(user_id: &str) -> Self;
+ fn collapse(collapse: bool) -> Self;
+ fn ids(ids: Vec<String>) -> Self;
+ fn ids_only(ids_only: bool) -> Self;
+ }
+
+ pub fn build_bool_query<'a>(&'a self) -> Result<BoolQueryBuilder<'a>> {
+ let mut bool_query = BoolQueryBuilder::new();
+
+ // Only search on the projects alias.
+ bool_query.filter(QueryType::term(
+ "_index",
+ ProjectSearchConfig::ENTITY_INDEX.index_name().to_string(),
+ ));
+
+ // Access control: owner and/or accessible entity ids.
+ bool_query.filter(
+ self.inner
+ .build_filter_query(ProjectSearchConfig::USER_ID_KEY)?,
+ );
+
+ // Tag filter: a single nested clause matching any of the option ids in
+ // `properties.values`, with no definition_id constraint.
+ if let Some(nested) = build_tag_filter(&self.tag_option_ids) {
+ bool_query.filter(nested);
+ }
+
+ // Name match: every term must match the project name.
+ bool_query.must(self.inner.build_title_term_query()?);
+
+ Ok(bool_query)
+ }
+}
+
+/// The `_source` fields of a project doc as returned by search.
+#[derive(Debug, serde::Serialize, serde::Deserialize)]
+pub(crate) struct ProjectIndex {
+ pub entity_id: uuid::Uuid,
+ pub name: String,
+ pub owner_id: String,
+ #[serde(default)]
+ pub parent_project_id: Option<String>,
+ pub updated_at_seconds: Option<i64>,
+}
+
+#[derive(Debug)]
+pub struct ProjectSearchArgs {
+ pub terms: Vec<String>,
+ pub user_id: String,
+ pub project_ids: Vec<String>,
+ pub page: u32,
+ pub page_size: u32,
+ pub match_type: String,
+ pub collapse: bool,
+ pub ids_only: bool,
+ pub tag_option_ids: Vec<String>,
+}
+
+impl From<ProjectSearchArgs> for ProjectQueryBuilder {
+ fn from(args: ProjectSearchArgs) -> Self {
+ ProjectQueryBuilder::new(args.terms)
+ .match_type(&args.match_type)
+ .page_size(args.page_size)
+ .page(args.page)
+ .user_id(&args.user_id)
+ .ids(args.project_ids)
+ .collapse(args.collapse)
+ .ids_only(args.ids_only)
+ .tag_option_ids(args.tag_option_ids)
+ }
+}
diff --git a/rust/cloud-storage/opensearch_client/src/search/properties.rs b/rust/cloud-storage/opensearch_client/src/search/properties.rs
new file mode 100644
index 0000000000..58ae87fe20
--- /dev/null
+++ b/rust/cloud-storage/opensearch_client/src/search/properties.rs
@@ -0,0 +1,23 @@
+use opensearch_query_builder::{NestedQuery, QueryType};
+
+/// Nested path holding denormalized entity properties on a search doc.
+pub(crate) const PROPERTIES_PATH: &str = "properties";
+
+/// Build a `nested` query over `properties` matching docs that have any
+/// nested entry whose `values` contains one of `option_ids`. There is no
+/// `definition_id` constraint: tag option ids are globally unique, so this
+/// matches a tag regardless of which definition owns it. Returns `None` when
+/// there are no option ids.
+pub(crate) fn build_tag_filter<'a>(option_ids: &[String]) -> Option<QueryType<'a>> {
+ if option_ids.is_empty() {
+ return None;
+ }
+ Some(
+ NestedQuery::new(
+ PROPERTIES_PATH,
+ QueryType::terms(format!("{PROPERTIES_PATH}.values"), option_ids.to_vec()),
+ )
+ .ignore_unmapped(true)
+ .into(),
+ )
+}
diff --git a/rust/cloud-storage/opensearch_client/src/search/unified.rs b/rust/cloud-storage/opensearch_client/src/search/unified.rs
index 49b2f54e68..74f9b5fc35 100644
--- a/rust/cloud-storage/opensearch_client/src/search/unified.rs
+++ b/rust/cloud-storage/opensearch_client/src/search/unified.rs
@@ -23,6 +23,7 @@ use crate::{
SearchGotoContent, SearchGotoEmail, SearchHit, exclude_source_content,
inject_fragment_size, parse_highlight_hit,
},
+ projects::{ProjectIndex, ProjectQueryBuilder, ProjectSearchArgs, ProjectSearchConfig},
query::Keys,
},
};
@@ -64,6 +65,8 @@ pub struct UnifiedSearchArgs {
pub chat_search_args: UnifiedChatSearchArgs,
/// The call record search args. If None, we do not search call records
pub call_record_search_args: UnifiedCallRecordSearchArgs,
+ /// The project search args. If None, we do not search projects
+ pub project_search_args: UnifiedProjectSearchArgs,
}
impl From<UnifiedSearchArgs> for DocumentSearchArgs {
@@ -106,6 +109,7 @@ impl From<UnifiedSearchArgs> for EmailSearchArgs {
exclude_labels: args.email_search_args.exclude_labels,
importance: args.email_search_args.importance,
subject_only: args.email_search_args.subject_only,
+ tag_option_ids: args.email_search_args.tag_option_ids,
}
}
}
@@ -140,6 +144,7 @@ impl From<UnifiedSearchArgs> for ChatSearchArgs {
ids_only: args.chat_search_args.ids_only,
chat_ids: args.chat_search_args.chat_ids,
role: args.chat_search_args.role,
+ tag_option_ids: args.chat_search_args.tag_option_ids,
}
}
}
@@ -161,12 +166,29 @@ impl From<UnifiedSearchArgs> for CallRecordSearchArgs {
}
}
+impl From<UnifiedSearchArgs> for ProjectSearchArgs {
+ fn from(args: UnifiedSearchArgs) -> Self {
+ ProjectSearchArgs {
+ terms: args.project_search_args.terms,
+ user_id: args.user_id,
+ page: args.page,
+ page_size: args.page_size,
+ match_type: args.match_type,
+ collapse: args.collapse,
+ ids_only: args.project_search_args.ids_only,
+ project_ids: args.project_search_args.project_ids,
+ tag_option_ids: args.project_search_args.tag_option_ids,
+ }
+ }
+}
+
#[derive(Debug, Default, Clone)]
pub struct UnifiedChatSearchArgs {
pub terms: Vec<String>,
pub chat_ids: Vec<String>,
pub role: Vec<String>,
pub ids_only: bool,
+ pub tag_option_ids: Vec<String>,
}
#[derive(Debug, Default, Clone)]
@@ -194,6 +216,7 @@ pub struct UnifiedEmailSearchArgs {
pub exclude_labels: Vec<String>,
pub importance: Option<bool>,
pub subject_only: bool,
+ pub tag_option_ids: Vec<String>,
}
#[derive(Debug, Default, Clone)]
@@ -214,6 +237,14 @@ pub struct UnifiedCallRecordSearchArgs {
pub ids_only: bool,
}
+#[derive(Debug, Default, Clone)]
+pub struct UnifiedProjectSearchArgs {
+ pub terms: Vec<String>,
+ pub project_ids: Vec<String>,
+ pub ids_only: bool,
+ pub tag_option_ids: Vec<String>,
+}
+
/// Possible search result indices for unified search
#[derive(Debug, serde::Serialize, serde::Deserialize)]
#[serde(untagged)]
@@ -223,6 +254,10 @@ pub(crate) enum UnifiedSearchIndex {
Chat(ChatIndex),
Email(Box<EmailIndex>),
CallRecord(CallRecordIndex),
+ // Keep last: with `untagged`, earlier variants win and every other doc
+ // shape carries required fields (document_name, title, message_id, …)
+ // a project doc lacks.
+ Project(ProjectIndex),
}
pub struct SplitUnifiedSearchResponseValues {
@@ -494,6 +529,27 @@ impl From<Hit<UnifiedSearchIndex>> for SearchHit {
.updated_at_seconds
.and_then(|s| DateTime::from_timestamp(s, 0)),
},
+ UnifiedSearchIndex::Project(a) => SearchHit {
+ entity_id: a.entity_id,
+ entity_type: SearchEntityType::Projects,
+ score: index.score,
+ highlight: index
+ .highlight
+ .map(|h| {
+ parse_highlight_hit(
+ h,
+ Keys {
+ title_key: ProjectSearchConfig::TITLE_KEY,
+ content_key: ProjectSearchConfig::CONTENT_KEY,
+ },
+ )
+ })
+ .unwrap_or_default(),
+ goto: None,
+ updated_at: a
+ .updated_at_seconds
+ .and_then(|s| DateTime::from_timestamp(s, 0)),
+ },
UnifiedSearchIndex::CallRecord(a) => SearchHit {
entity_id: a.entity_id,
entity_type: SearchEntityType::CallRecords,
@@ -599,6 +655,17 @@ fn build_unified_search_request(args: &UnifiedSearchArgs) -> Result<SearchReques
bool_query.should(query_type.to_owned());
}
+ if args
+ .search_indices
+ .contains(&OpenSearchEntityType::Projects)
+ {
+ let project_search_args: ProjectSearchArgs = args.clone().into();
+ let project_query_builder: ProjectQueryBuilder = project_search_args.into();
+ let project_bool_query = project_query_builder.build_bool_query()?;
+ let query_type: QueryType = project_bool_query.build().into();
+ bool_query.should(query_type.to_owned());
+ }
+
// create the search request
let mut search_request_builder = SearchRequestBuilder::new();
@@ -626,6 +693,7 @@ fn build_unified_search_request(args: &UnifiedSearchArgs) -> Result<SearchReques
.require_field_match(true)
.field("content", em_field().number_of_fragments(1))
.field("document_name", em_field().number_of_fragments(0))
+ .field("name", em_field().number_of_fragments(0))
.field("subject", em_field().number_of_fragments(0))
.field("sender", em_field().number_of_fragments(0))
.field("sender_name", em_field().number_of_fragments(0))
diff --git a/rust/cloud-storage/opensearch_client/src/search/unified/test.rs b/rust/cloud-storage/opensearch_client/src/search/unified/test.rs
index 7af9e20860..c08178803b 100644
--- a/rust/cloud-storage/opensearch_client/src/search/unified/test.rs
+++ b/rust/cloud-storage/opensearch_client/src/search/unified/test.rs
@@ -488,8 +488,10 @@ fn test_build_unified_search_request_content() -> anyhow::Result<()> {
chat_ids: vec!["id1".to_string(), "id2".to_string()],
role: vec!["id1".to_string(), "id2".to_string()],
ids_only: false,
+ ..Default::default()
},
call_record_search_args: UnifiedCallRecordSearchArgs::default(),
+ project_search_args: UnifiedProjectSearchArgs::default(),
cursor: SearchCursorOption::NotDone(Some(SearchMethodCursor::UpdatedAt {
entity_id,
updated_at: time,
@@ -522,6 +524,7 @@ fn test_build_unified_search_request_content() -> anyhow::Result<()> {
"fields": {
"content": { "number_of_fragments": 1, "post_tags": ["</macro_em>"], "pre_tags": ["<macro_em>"], "type": "plain" },
"document_name": { "number_of_fragments": 0, "post_tags": ["</macro_em>"], "pre_tags": ["<macro_em>"], "type": "plain" },
+ "name": { "number_of_fragments": 0, "post_tags": ["</macro_em>"], "pre_tags": ["<macro_em>"], "type": "plain" },
"subject": { "number_of_fragments": 0, "post_tags": ["</macro_em>"], "pre_tags": ["<macro_em>"], "type": "plain" },
"sender": { "number_of_fragments": 0, "post_tags": ["</macro_em>"], "pre_tags": ["<macro_em>"], "type": "plain" },
"sender_name": { "number_of_fragments": 0, "post_tags": ["</macro_em>"], "pre_tags": ["<macro_em>"], "type": "plain" },
@@ -786,8 +789,10 @@ fn test_build_unified_search_request_content() -> anyhow::Result<()> {
chat_ids: vec!["id1".to_string(), "id2".to_string()],
role: vec!["id1".to_string(), "id2".to_string()],
ids_only: false,
+ ..Default::default()
},
call_record_search_args: UnifiedCallRecordSearchArgs::default(),
+ project_search_args: UnifiedProjectSearchArgs::default(),
cursor: SearchCursorOption::NotDone(Some(SearchMethodCursor::UpdatedAt {
entity_id,
updated_at: time,
@@ -840,6 +845,7 @@ fn test_build_unified_search_request_single_index() -> anyhow::Result<()> {
"fields": {
"content": { "number_of_fragments": 1, "post_tags": ["</macro_em>"], "pre_tags": ["<macro_em>"], "type": "plain" },
"document_name": { "number_of_fragments": 0, "post_tags": ["</macro_em>"], "pre_tags": ["<macro_em>"], "type": "plain" },
+ "name": { "number_of_fragments": 0, "post_tags": ["</macro_em>"], "pre_tags": ["<macro_em>"], "type": "plain" },
"subject": { "number_of_fragments": 0, "post_tags": ["</macro_em>"], "pre_tags": ["<macro_em>"], "type": "plain" },
"sender": { "number_of_fragments": 0, "post_tags": ["</macro_em>"], "pre_tags": ["<macro_em>"], "type": "plain" },
"sender_name": { "number_of_fragments": 0, "post_tags": ["</macro_em>"], "pre_tags": ["<macro_em>"], "type": "plain" },
@@ -1016,3 +1022,69 @@ fn paginate_page_size_zero_terminates() {
"no anchor to resume from must terminate, not loop, got {cursor:?}"
);
}
+
+#[test]
+fn project_hit_deserializes_and_converts() -> anyhow::Result<()> {
+ // A project doc as the flat projects index returns it. With the untagged
+ // UnifiedSearchIndex enum the Project variant must win: no other variant's
+ // required fields (document_name, title, message_id, channel_id, …) are
+ // present in a project _source.
+ let json = serde_json::json!({
+ "took": 3,
+ "timed_out": false,
+ "_shards": { "total": 3, "successful": 3, "skipped": 0, "failed": 0 },
+ "hits": {
+ "total": { "value": 1, "relation": "eq" },
+ "max_score": 4.2,
+ "hits": [
+ {
+ "_index": "projects_v1",
+ "_id": "0197a863-0000-7000-8000-000000000001",
+ "_score": 4.2,
+ "_source": {
+ "entity_id": "0197a863-0000-7000-8000-000000000001",
+ "name": "Mobile Redesign",
+ "owner_id": "macro|[email protected]",
+ "parent_project_id": "0197a863-0000-7000-8000-000000000002",
+ "created_at_seconds": 1770000000,
+ "updated_at_seconds": 1783000000,
+ "properties": [
+ { "definition_id": "0197a863-0000-7000-8000-00000000000a", "values": ["opt-1"] }
+ ]
+ },
+ "highlight": {
+ "name": ["<macro_em>Mobile</macro_em> Redesign"]
+ }
+ }
+ ]
+ }
+ });
+
+ let result: DefaultSearchResponse<UnifiedSearchIndex> = serde_json::from_value(json)?;
+ assert_eq!(result.hits.hits.len(), 1);
+ assert!(matches!(
+ result.hits.hits[0].source,
+ UnifiedSearchIndex::Project(_)
+ ));
+
+ let (hits, cursor) = paginate_unified_hits(result.hits.hits, 10);
+ assert_eq!(hits.len(), 1);
+ let hit = &hits[0];
+ assert_eq!(hit.entity_type, SearchEntityType::Projects);
+ assert_eq!(
+ hit.entity_id,
+ "0197a863-0000-7000-8000-000000000001".parse::<uuid::Uuid>()?
+ );
+ assert_eq!(
+ hit.highlight.name.as_deref(),
+ Some("<macro_em>Mobile</macro_em> Redesign")
+ );
+ assert!(hit.goto.is_none());
+ assert_eq!(
+ hit.updated_at,
+ chrono::DateTime::from_timestamp(1_783_000_000, 0)
+ );
+ assert!(cursor.is_done());
+
+ Ok(())
+}
diff --git a/rust/cloud-storage/opensearch_client/src/upsert/chat_message.rs b/rust/cloud-storage/opensearch_client/src/upsert/chat_message.rs
index 372e0c46c8..1c651fa54a 100644
--- a/rust/cloud-storage/opensearch_client/src/upsert/chat_message.rs
+++ b/rust/cloud-storage/opensearch_client/src/upsert/chat_message.rs
@@ -1,7 +1,11 @@
use models_opensearch::SearchIndex;
+use crate::upsert::properties::IndexedProperty;
use crate::{Result, date_format::EpochSeconds, error::OpensearchClientError};
+#[cfg(test)]
+mod test;
+
/// Relation name for parent docs in the chats join field.
const PARENT_RELATION: &str = "chat";
@@ -28,6 +32,9 @@ pub struct UpsertChatMessageArgs {
pub title: String,
/// The content of the chat message
pub content: String,
+ /// Denormalized parent entity properties (tags, custom) used for search
+ /// filtering. Empty for chats without properties.
+ pub properties: Vec<IndexedProperty>,
}
/// Resolve `index_override` to the physical/alias name we'll write to.
@@ -35,15 +42,23 @@ fn resolve_destination(index_override: Option<&str>) -> &str {
index_override.unwrap_or(SearchIndex::Chats.as_ref())
}
-/// Builds the JSON document body for the parent chat doc.
+/// Builds the JSON document body for the parent chat doc. Properties ride
+/// every parent write because the write is a full overwrite — omitting them
+/// would wipe values set by `update_chat_properties`.
fn parent_doc_body(args: &UpsertChatMessageArgs) -> serde_json::Value {
- serde_json::json!({
+ let mut doc = serde_json::json!({
"entity_id": &args.chat_id,
"title": &args.title,
"user_id": &args.user_id,
"updated_at_seconds": args.updated_at_seconds,
"chat_relation": PARENT_RELATION,
- })
+ });
+ if !args.properties.is_empty()
+ && let Ok(properties) = serde_json::to_value(&args.properties)
+ {
+ doc["properties"] = properties;
+ }
+ doc
}
/// Builds the JSON document body for a child (message) doc.
@@ -203,3 +218,78 @@ pub(crate) async fn update_chat_metadata(
method: Some("update_chat_metadata".to_string()),
})
}
+
+/// Update only the denormalized `properties` on an existing parent chat doc,
+/// without touching messages. Used when a chat's properties change
+/// independently of its content. A missing doc (404) is treated as a no-op —
+/// the next message upsert will include the properties.
+pub(crate) async fn update_chat_properties(
+ client: &opensearch::OpenSearch,
+ chat_id: &str,
+ properties: &[IndexedProperty],
+) -> Result<()> {
+ use serde_json::json;
+
+ let properties_value =
+ serde_json::to_value(properties).map_err(|err| OpensearchClientError::Unknown {
+ details: err.to_string(),
+ method: Some("update_chat_properties".to_string()),
+ })?;
+ let body = json!({ "doc": { "properties": properties_value } });
+
+ let response = client
+ .update(opensearch::UpdateParts::IndexId(
+ SearchIndex::Chats.as_ref(),
+ chat_id,
+ ))
+ .routing(chat_id)
+ .body(body)
+ .send()
+ .await
+ .map_err(|err| OpensearchClientError::DeserializationFailed {
+ details: err.to_string(),
+ method: Some("update_chat_properties".to_string()),
+ })?;
+
+ let status_code = response.status_code();
+ if status_code.is_success() {
+ tracing::trace!(chat_id=%chat_id, "chat properties updated");
+ return Ok(());
+ }
+ let body =
+ response
+ .text()
+ .await
+ .map_err(|err| OpensearchClientError::DeserializationFailed {
+ details: err.to_string(),
+ method: Some("update_chat_properties".to_string()),
+ })?;
+
+ // A *missing document* 404 is a no-op: the chat isn't indexed yet, so the
+ // next message upsert will include its properties. A *missing index* 404
+ // (`index_not_found_exception`) is a real outage and must propagate.
+ if status_code.as_u16() == 404 {
+ let error_type = serde_json::from_str::<serde_json::Value>(&body)
+ .ok()
+ .and_then(|value| value["error"]["type"].as_str().map(str::to_owned));
+ if error_type.as_deref() == Some("document_missing_exception") {
+ tracing::debug!(
+ chat_id=%chat_id,
+ "chat not indexed yet; skipping property update"
+ );
+ return Ok(());
+ }
+ }
+
+ tracing::error!(
+ status_code=?status_code,
+ body=?body,
+ chat_id=%chat_id,
+ "error updating chat properties",
+ );
+
+ Err(OpensearchClientError::Unknown {
+ details: body,
+ method: Some("update_chat_properties".to_string()),
+ })
+}
diff --git a/rust/cloud-storage/opensearch_client/src/upsert/chat_message/test.rs b/rust/cloud-storage/opensearch_client/src/upsert/chat_message/test.rs
new file mode 100644
index 0000000000..72120d35d5
--- /dev/null
+++ b/rust/cloud-storage/opensearch_client/src/upsert/chat_message/test.rs
@@ -0,0 +1,64 @@
+use super::*;
+
+fn args() -> UpsertChatMessageArgs {
+ UpsertChatMessageArgs {
+ chat_id: "chat1".to_string(),
+ chat_message_id: "msg1".to_string(),
+ user_id: "macro|[email protected]".to_string(),
+ role: "user".to_string(),
+ created_at_seconds: EpochSeconds::new(1_700_000_000).unwrap(),
+ updated_at_seconds: EpochSeconds::new(1_700_000_000).unwrap(),
+ title: "Chat title".to_string(),
+ content: "message content".to_string(),
+ properties: vec![],
+ }
+}
+
+#[test]
+fn parent_doc_body_has_metadata_no_child_fields() {
+ let doc = parent_doc_body(&args());
+
+ assert_eq!(doc["entity_id"], "chat1");
+ assert_eq!(doc["title"], "Chat title");
+ assert_eq!(doc["user_id"], "macro|[email protected]");
+ assert_eq!(doc["chat_relation"], "chat");
+ // Child-only fields must not be present on the parent.
+ assert!(doc.get("content").is_none());
+ assert!(doc.get("chat_message_id").is_none());
+ assert!(doc.get("role").is_none());
+ // No properties key when the chat has none.
+ assert!(doc.get("properties").is_none());
+}
+
+#[test]
+fn parent_doc_body_includes_properties_when_present() {
+ let mut a = args();
+ a.properties = vec![IndexedProperty {
+ definition_id: "tag-def".to_string(),
+ values: vec!["option-1".to_string(), "option-2".to_string()],
+ ..Default::default()
+ }];
+
+ let doc = parent_doc_body(&a);
+ let props = doc["properties"].as_array().expect("properties array");
+ assert_eq!(props.len(), 1);
+ assert_eq!(props[0]["definition_id"], "tag-def");
+ assert_eq!(
+ props[0]["values"],
+ serde_json::json!(["option-1", "option-2"])
+ );
+}
+
+#[test]
+fn child_doc_body_has_no_properties() {
+ let mut a = args();
+ a.properties = vec![IndexedProperty {
+ definition_id: "tag-def".to_string(),
+ values: vec!["option-1".to_string()],
+ ..Default::default()
+ }];
+
+ let doc = child_doc_body(&a);
+ assert!(doc.get("properties").is_none());
+ assert_eq!(doc["chat_relation"]["parent"], "chat1");
+}
diff --git a/rust/cloud-storage/opensearch_client/src/upsert/document.rs b/rust/cloud-storage/opensearch_client/src/upsert/document.rs
index 5fce7d877f..1fbd92c331 100644
--- a/rust/cloud-storage/opensearch_client/src/upsert/document.rs
+++ b/rust/cloud-storage/opensearch_client/src/upsert/document.rs
@@ -3,6 +3,7 @@ use std::collections::HashSet;
use models_opensearch::SearchIndex;
use super::BulkUpsertResult;
+use super::properties::IndexedProperty;
use crate::{Result, date_format::EpochSeconds, error::OpensearchClientError};
/// Relation name for parent docs in the join field.
@@ -11,27 +12,6 @@ const PARENT_RELATION: &str = "document";
/// Relation name for child (chunk) docs in the join field.
const CHILD_RELATION: &str = "chunk";
-/// A denormalized entity property indexed on the parent doc so search can
-/// filter by it. `values` holds every equality-filterable value (select
-/// options, entity refs, links, text, bool); `number_value`/`date_value`
-/// are split out only because they need range + sort semantics that keyword
-/// can't provide. Always queried scoped by `definition_id`.
-#[derive(Debug, Clone, Default, serde::Serialize)]
-pub struct IndexedProperty {
- /// The property definition id this value belongs to.
- pub definition_id: String,
- /// Every equality-filterable value as a keyword: select-option UUIDs,
- /// entity-reference ids, links, text, bool as "true"/"false".
- #[serde(skip_serializing_if = "Vec::is_empty")]
- pub values: Vec<String>,
- /// Numeric value (e.g. story points) — range + sort.
- #[serde(skip_serializing_if = "Option::is_none")]
- pub number_value: Option<f64>,
- /// Date value as epoch milliseconds (e.g. due date) — range + sort.
- #[serde(skip_serializing_if = "Option::is_none")]
- pub date_value: Option<i64>,
-}
-
/// The arguments for upserting a document into the opensearch index
#[derive(Debug, serde::Serialize)]
pub struct UpsertDocumentArgs {
diff --git a/rust/cloud-storage/opensearch_client/src/upsert/email.rs b/rust/cloud-storage/opensearch_client/src/upsert/email.rs
index 417915f74c..15db8f7531 100644
--- a/rust/cloud-storage/opensearch_client/src/upsert/email.rs
+++ b/rust/cloud-storage/opensearch_client/src/upsert/email.rs
@@ -1,6 +1,7 @@
use models_opensearch::SearchIndex;
use super::BulkUpsertResult;
+use super::properties::IndexedProperty;
use crate::{Result, date_format::EpochSeconds, error::OpensearchClientError};
/// The arguments for upserting an email message into the opensearch index
@@ -45,6 +46,10 @@ pub struct UpsertEmailArgs {
pub sent_at_seconds: Option<EpochSeconds>,
/// The content of the email message
pub content: String,
+ /// Denormalized thread properties (e.g. tags) used for search filtering.
+ /// Thread-level, so identical across every message doc of the thread.
+ #[serde(skip_serializing_if = "Vec::is_empty")]
+ pub properties: Vec<IndexedProperty>,
}
#[tracing::instrument(skip(client))]
@@ -127,3 +132,92 @@ pub(crate) async fn bulk_upsert_email_messages(
super::bulk_upsert_to_index(client, index, bulk_body, "bulk_upsert_email_messages").await
}
+
+/// Update only the denormalized `properties` on every message doc of a
+/// thread, without touching content. The emails index is flat (one doc per
+/// message sharing the thread's `entity_id`), so this is an update-by-query
+/// rather than a single-doc update. Zero matched docs is a no-op — the
+/// thread isn't indexed yet, so the next full index will include the
+/// properties. Version conflicts proceed: a concurrent full reindex writes
+/// the same freshly-fetched values.
+pub(crate) async fn update_email_thread_properties(
+ client: &opensearch::OpenSearch,
+ thread_id: &str,
+ properties: &[IndexedProperty],
+) -> Result<()> {
+ let properties_value =
+ serde_json::to_value(properties).map_err(|err| OpensearchClientError::Unknown {
+ details: err.to_string(),
+ method: Some("update_email_thread_properties".to_string()),
+ })?;
+ let body = serde_json::json!({
+ "query": { "term": { "entity_id": thread_id } },
+ "script": {
+ "source": "ctx._source.properties = params.properties",
+ "lang": "painless",
+ "params": { "properties": properties_value }
+ }
+ });
+
+ let response = client
+ .update_by_query(opensearch::UpdateByQueryParts::Index(&[
+ SearchIndex::Emails.as_ref(),
+ ]))
+ .conflicts(opensearch::params::Conflicts::Proceed)
+ .refresh(true) // Ensure the index reflects changes immediately
+ .body(body)
+ .send()
+ .await
+ .map_err(|err| OpensearchClientError::DeserializationFailed {
+ details: err.to_string(),
+ method: Some("update_email_thread_properties".to_string()),
+ })?;
+
+ let status_code = response.status_code();
+ if status_code.is_success() {
+ // A 200 can still carry per-document failures in the body.
+ let body: serde_json::Value =
+ response
+ .json()
+ .await
+ .map_err(|err| OpensearchClientError::DeserializationFailed {
+ details: err.to_string(),
+ method: Some("update_email_thread_properties".to_string()),
+ })?;
+ if let Some(failures) = body.get("failures").and_then(|f| f.as_array())
+ && !failures.is_empty()
+ {
+ tracing::error!(
+ thread_id=%thread_id,
+ failures=?failures,
+ "update_by_query reported failures updating email thread properties",
+ );
+ return Err(OpensearchClientError::Unknown {
+ details: format!("update_by_query failures: {failures:?}"),
+ method: Some("update_email_thread_properties".to_string()),
+ });
+ }
+ tracing::trace!(thread_id=%thread_id, "email thread properties updated");
+ return Ok(());
+ }
+ let body =
+ response
+ .text()
+ .await
+ .map_err(|err| OpensearchClientError::DeserializationFailed {
+ details: err.to_string(),
+ method: Some("update_email_thread_properties".to_string()),
+ })?;
+
+ tracing::error!(
+ status_code=?status_code,
+ body=?body,
+ thread_id=%thread_id,
+ "error updating email thread properties",
+ );
+
+ Err(OpensearchClientError::Unknown {
+ details: body,
+ method: Some("update_email_thread_properties".to_string()),
+ })
+}
diff --git a/rust/cloud-storage/opensearch_client/src/upsert/mod.rs b/rust/cloud-storage/opensearch_client/src/upsert/mod.rs
index bfef2f9cbd..c22417da01 100644
--- a/rust/cloud-storage/opensearch_client/src/upsert/mod.rs
+++ b/rust/cloud-storage/opensearch_client/src/upsert/mod.rs
@@ -3,6 +3,8 @@ pub mod channel_message;
pub mod chat_message;
pub mod document;
pub mod email;
+pub mod project;
+pub mod properties;
use crate::error::OpensearchClientError;
diff --git a/rust/cloud-storage/opensearch_client/src/upsert/project.rs b/rust/cloud-storage/opensearch_client/src/upsert/project.rs
new file mode 100644
index 0000000000..e144ed8b7d
--- /dev/null
+++ b/rust/cloud-storage/opensearch_client/src/upsert/project.rs
@@ -0,0 +1,158 @@
+use models_opensearch::SearchIndex;
+
+use super::properties::IndexedProperty;
+use crate::{Result, date_format::EpochSeconds, error::OpensearchClientError};
+
+/// The arguments for upserting a project into the opensearch index.
+///
+/// The projects index is flat: one doc per project, `_id` = project id.
+#[derive(Debug, serde::Serialize)]
+pub struct UpsertProjectArgs {
+ /// The id of the project
+ #[serde(rename = "entity_id")]
+ pub project_id: String,
+ /// The name of the project
+ pub name: String,
+ /// The owner id of the project
+ pub owner_id: String,
+ /// The parent project id, if nested
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub parent_project_id: Option<String>,
+ /// The created at time of the project
+ pub created_at_seconds: EpochSeconds,
+ /// The updated at time of the project
+ pub updated_at_seconds: EpochSeconds,
+ /// Entity properties (tags, custom) used for search filtering.
+ pub properties: Vec<IndexedProperty>,
+}
+
+/// Resolve `index_override` to the physical/alias name we'll write to.
+fn resolve_destination(index_override: Option<&str>) -> &str {
+ index_override.unwrap_or(SearchIndex::Projects.as_ref())
+}
+
+/// Upsert a single project doc. Full-overwrite `index` semantics so omitted
+/// optional fields (e.g. `parent_project_id`) get cleared on Some→None
+/// transitions.
+#[tracing::instrument(skip(client, args), fields(project_id=%args.project_id), err)]
+pub(crate) async fn upsert_project(
+ client: &opensearch::OpenSearch,
+ args: &UpsertProjectArgs,
+ index_override: Option<&str>,
+) -> Result<()> {
+ let index = resolve_destination(index_override);
+ let body = serde_json::to_value(args).map_err(|err| OpensearchClientError::Unknown {
+ details: err.to_string(),
+ method: Some("upsert_project".to_string()),
+ })?;
+
+ let response = client
+ .index(opensearch::IndexParts::IndexId(index, &args.project_id))
+ .body(body)
+ .send()
+ .await
+ .map_err(|err| OpensearchClientError::Unknown {
+ details: err.to_string(),
+ method: Some("upsert_project".to_string()),
+ })?;
+
+ let status_code = response.status_code();
+ if status_code.is_success() {
+ tracing::trace!(project_id=%args.project_id, "project upserted successfully");
+ return Ok(());
+ }
+
+ let body =
+ response
+ .text()
+ .await
+ .map_err(|err| OpensearchClientError::DeserializationFailed {
+ details: err.to_string(),
+ method: Some("upsert_project".to_string()),
+ })?;
+
+ tracing::error!(
+ status_code=?status_code,
+ body=?body,
+ project_id=%args.project_id,
+ "error upserting project",
+ );
+
+ Err(OpensearchClientError::Unknown {
+ details: body,
+ method: Some("upsert_project".to_string()),
+ })
+}
+
+/// Update only the denormalized `properties` on an existing project doc.
+/// Used when an entity's properties change independently of its metadata.
+/// A missing doc (404) is treated as a no-op — the next full upsert will
+/// include the properties.
+pub(crate) async fn update_project_properties(
+ client: &opensearch::OpenSearch,
+ project_id: &str,
+ properties: &[IndexedProperty],
+ index_override: Option<&str>,
+) -> Result<()> {
+ use serde_json::json;
+
+ let index = resolve_destination(index_override);
+ let properties_value =
+ serde_json::to_value(properties).map_err(|err| OpensearchClientError::Unknown {
+ details: err.to_string(),
+ method: Some("update_project_properties".to_string()),
+ })?;
+ let body = json!({ "doc": { "properties": properties_value } });
+
+ let response = client
+ .update(opensearch::UpdateParts::IndexId(index, project_id))
+ .body(body)
+ .send()
+ .await
+ .map_err(|err| OpensearchClientError::DeserializationFailed {
+ details: err.to_string(),
+ method: Some("update_project_properties".to_string()),
+ })?;
+
+ let status_code = response.status_code();
+ if status_code.is_success() {
+ tracing::trace!(project_id=%project_id, "project properties updated");
+ return Ok(());
+ }
+ let body =
+ response
+ .text()
+ .await
+ .map_err(|err| OpensearchClientError::DeserializationFailed {
+ details: err.to_string(),
+ method: Some("update_project_properties".to_string()),
+ })?;
+
+ // A *missing document* 404 is a no-op: the doc isn't indexed yet, so the
+ // next full upsert will include its properties. A *missing index* 404
+ // (`index_not_found_exception`) is a real outage and must propagate.
+ if status_code.as_u16() == 404 {
+ let error_type = serde_json::from_str::<serde_json::Value>(&body)
+ .ok()
+ .and_then(|value| value["error"]["type"].as_str().map(str::to_owned));
+ if error_type.as_deref() == Some("document_missing_exception") {
+ tracing::debug!(
+ project_id=%project_id,
+ "project not indexed yet; skipping property update"
+ );
+ return Ok(());
+ }
+ }
+
+ tracing::error!(
+ status_code=?status_code,
+ body=?body,
+ project_id=%project_id,
+ "error updating project properties",
+ );
+
+ Err(OpensearchClientError::Unknown {
+ details: body,
+ method: Some("update_project_properties".to_string()),
+ })
+}
diff --git a/rust/cloud-storage/opensearch_client/src/upsert/properties.rs b/rust/cloud-storage/opensearch_client/src/upsert/properties.rs
new file mode 100644
index 0000000000..0cdd1ba826
--- /dev/null
+++ b/rust/cloud-storage/opensearch_client/src/upsert/properties.rs
@@ -0,0 +1,20 @@
+/// A denormalized entity property indexed on the entity's search doc so
+/// search can filter by it. `values` holds every equality-filterable value
+/// (select options, entity refs, links, text, bool); `number_value`/
+/// `date_value` are split out only because they need range + sort semantics
+/// that keyword can't provide. Always queried scoped by `definition_id`.
+#[derive(Debug, Clone, Default, serde::Serialize)]
+pub struct IndexedProperty {
+ /// The property definition id this value belongs to.
+ pub definition_id: String,
+ /// Every equality-filterable value as a keyword: select-option UUIDs,
+ /// entity-reference ids, links, text, bool as "true"/"false".
+ #[serde(skip_serializing_if = "Vec::is_empty")]
+ pub values: Vec<String>,
+ /// Numeric value (e.g. story points) — range + sort.
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub number_value: Option<f64>,
+ /// Date value as epoch milliseconds (e.g. due date) — range + sort.
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub date_value: Option<i64>,
+}
diff --git a/rust/cloud-storage/properties/src/domain/service_impl/mod.rs b/rust/cloud-storage/properties/src/domain/service_impl/mod.rs
index 482f118c71..45ee8a9437 100644
--- a/rust/cloud-storage/properties/src/domain/service_impl/mod.rs
+++ b/rust/cloud-storage/properties/src/domain/service_impl/mod.rs
@@ -22,6 +22,19 @@ use super::service::PropertiesService;
use helpers::{extract_option_ids_from_property_value, is_property_applicable_to};
+/// Entity types whose search index denormalizes property values, i.e. whose
+/// property mutations must enqueue a search reindex.
+fn is_search_indexed(entity_type: EntityType) -> bool {
+ matches!(
+ entity_type,
+ EntityType::Task
+ | EntityType::Document
+ | EntityType::Thread
+ | EntityType::Chat
+ | EntityType::Project
+ )
+}
+
/// Implementation of PropertiesService using a repository and optional permission service.
#[derive(Debug)]
pub struct PropertiesServiceImpl<R, P, N>
@@ -63,14 +76,14 @@ where
self
}
- /// Best-effort publish of a property reindex for entities that live in
- /// the documents search index (tasks/documents). Logs and continues on
+ /// Best-effort publish of a property reindex for entity types whose
+ /// search index denormalizes property values. Logs and continues on
/// failure so a missed reindex never fails the mutation itself.
async fn enqueue_property_upsert(&self, entity_id: &str, entity_type: EntityType) {
let Some(search_indexer) = self.search_indexer.as_ref() else {
return;
};
- if !matches!(entity_type, EntityType::Task | EntityType::Document) {
+ if !is_search_indexed(entity_type) {
return;
}
if let Err(error) = search_indexer
diff --git a/rust/cloud-storage/properties_db_client/.sqlx/query-089ddab77c715460d9e8573812b50ba5b5eb4ba617f613b80c1ac772db727669.json b/rust/cloud-storage/properties_db_client/.sqlx/query-089ddab77c715460d9e8573812b50ba5b5eb4ba617f613b80c1ac772db727669.json
new file mode 100644
index 0000000000..ed4aa7bbf2
--- /dev/null
+++ b/rust/cloud-storage/properties_db_client/.sqlx/query-089ddab77c715460d9e8573812b50ba5b5eb4ba617f613b80c1ac772db727669.json
@@ -0,0 +1,51 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n SELECT\n entity_id,\n property_definition_id,\n values as \"values: serde_json::Value\"\n FROM entity_properties\n WHERE entity_id = ANY($1)\n AND entity_type = $2\n ",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "entity_id",
+ "type_info": "Text"
+ },
+ {
+ "ordinal": 1,
+ "name": "property_definition_id",
+ "type_info": "Uuid"
+ },
+ {
+ "ordinal": 2,
+ "name": "values: serde_json::Value",
+ "type_info": "Jsonb"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "TextArray",
+ {
+ "Custom": {
+ "name": "property_entity_type",
+ "kind": {
+ "Enum": [
+ "CHANNEL",
+ "CHAT",
+ "DOCUMENT",
+ "PROJECT",
+ "THREAD",
+ "USER",
+ "COMPANY",
+ "TASK"
+ ]
+ }
+ }
+ }
+ ]
+ },
+ "nullable": [
+ false,
+ false,
+ true
+ ]
+ },
+ "hash": "089ddab77c715460d9e8573812b50ba5b5eb4ba617f613b80c1ac772db727669"
+}
diff --git a/rust/cloud-storage/properties_db_client/.sqlx/query-4cc83ea0dfe3d47bf30c18701f37f77a52760d279c5dbdeae68b88d329d8c124.json b/rust/cloud-storage/properties_db_client/.sqlx/query-4cc83ea0dfe3d47bf30c18701f37f77a52760d279c5dbdeae68b88d329d8c124.json
new file mode 100644
index 0000000000..146bcb925c
--- /dev/null
+++ b/rust/cloud-storage/properties_db_client/.sqlx/query-4cc83ea0dfe3d47bf30c18701f37f77a52760d279c5dbdeae68b88d329d8c124.json
@@ -0,0 +1,40 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n SELECT DISTINCT entity_id\n FROM entity_properties\n WHERE entity_type = $1\n ORDER BY entity_id\n LIMIT $2 OFFSET $3\n ",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "entity_id",
+ "type_info": "Text"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ {
+ "Custom": {
+ "name": "property_entity_type",
+ "kind": {
+ "Enum": [
+ "CHANNEL",
+ "CHAT",
+ "DOCUMENT",
+ "PROJECT",
+ "THREAD",
+ "USER",
+ "COMPANY",
+ "TASK"
+ ]
+ }
+ }
+ },
+ "Int8",
+ "Int8"
+ ]
+ },
+ "nullable": [
+ false
+ ]
+ },
+ "hash": "4cc83ea0dfe3d47bf30c18701f37f77a52760d279c5dbdeae68b88d329d8c124"
+}
diff --git a/rust/cloud-storage/properties_db_client/src/entity_properties/get.rs b/rust/cloud-storage/properties_db_client/src/entity_properties/get.rs
index 900ac78e50..3884b18d44 100644
--- a/rust/cloud-storage/properties_db_client/src/entity_properties/get.rs
+++ b/rust/cloud-storage/properties_db_client/src/entity_properties/get.rs
@@ -604,6 +604,51 @@ impl IndexedEntityProperty {
}
}
+/// Flatten one entity_properties row into the shape the search index stores.
+/// Null and unparseable values yield None so one bad value can't fail the
+/// whole index.
+fn flatten_property_for_index(
+ property_definition_id: Uuid,
+ values: Option<serde_json::Value>,
+) -> Option<IndexedEntityProperty> {
+ let value = values?;
+ if value.is_null() {
+ return None;
+ }
+ let parsed = match serde_json::from_value::<PropertyValue>(value) {
+ Ok(parsed) => parsed,
+ Err(e) => {
+ tracing::warn!(
+ error = ?e,
+ definition_id = %property_definition_id,
+ "skipping unparseable property value for index"
+ );
+ return None;
+ }
+ };
+ let mut property = IndexedEntityProperty {
+ definition_id: property_definition_id.to_string(),
+ ..Default::default()
+ };
+ match parsed {
+ PropertyValue::SelectOption(ids) => {
+ property.values = ids.into_iter().map(|id| id.to_string()).collect();
+ }
+ PropertyValue::EntityRef(refs) => {
+ property.values = refs.into_iter().map(|r| r.entity_id).collect();
+ }
+ PropertyValue::Link(urls) => property.values = urls,
+ PropertyValue::Str(s) => property.values = vec![s],
+ PropertyValue::Bool(b) => property.values = vec![b.to_string()],
+ PropertyValue::Num(n) => property.number_value = Some(n),
+ PropertyValue::Date(d) => property.date_value = Some(d.timestamp_millis()),
+ }
+ if property.is_empty() {
+ return None;
+ }
+ Some(property)
+}
+
/// Fetch an entity's properties flattened for the search index. Each value
/// lands under the field matching its type (equality-filterable values as
/// strings, numbers, or epoch-millis dates); null and unparseable values are
@@ -629,46 +674,74 @@ pub async fn get_entity_properties_for_index(
.fetch_all(db)
.await?;
- let mut result = Vec::with_capacity(rows.len());
+ Ok(rows
+ .into_iter()
+ .filter_map(|row| flatten_property_for_index(row.property_definition_id, row.values))
+ .collect())
+}
+
+/// Page through the distinct entity ids that hold property rows of one
+/// entity type, ordered by entity id so offset pagination is stable. Used by
+/// the search properties backfill.
+#[tracing::instrument(skip(db), err)]
+pub async fn get_entity_ids_with_properties(
+ db: &Pool<Postgres>,
+ entity_type: EntityType,
+ limit: i64,
+ offset: i64,
+) -> Result<Vec<String>> {
+ let rows = sqlx::query_scalar!(
+ r#"
+ SELECT DISTINCT entity_id
+ FROM entity_properties
+ WHERE entity_type = $1
+ ORDER BY entity_id
+ LIMIT $2 OFFSET $3
+ "#,
+ entity_type as EntityType,
+ limit,
+ offset,
+ )
+ .fetch_all(db)
+ .await?;
+ Ok(rows)
+}
+
+/// Batch variant of [`get_entity_properties_for_index`]: fetch the flattened
+/// properties of many entities of one type in a single query, keyed by
+/// entity id. Entities without properties are absent from the map.
+#[tracing::instrument(skip(db, entity_ids), err)]
+pub async fn get_entity_properties_for_index_batch(
+ db: &Pool<Postgres>,
+ entity_ids: &[String],
+ entity_type: EntityType,
+) -> Result<HashMap<String, Vec<IndexedEntityProperty>>> {
+ if entity_ids.is_empty() {
+ return Ok(HashMap::new());
+ }
+ let rows = sqlx::query!(
+ r#"
+ SELECT
+ entity_id,
+ property_definition_id,
+ values as "values: serde_json::Value"
+ FROM entity_properties
+ WHERE entity_id = ANY($1)
+ AND entity_type = $2
+ "#,
+ entity_ids,
+ entity_type as EntityType,
+ )
+ .fetch_all(db)
+ .await?;
+
+ let mut result: HashMap<String, Vec<IndexedEntityProperty>> = HashMap::new();
for row in rows {
- let Some(value) = row.values else {
+ let Some(property) = flatten_property_for_index(row.property_definition_id, row.values)
+ else {
continue;
};
- if value.is_null() {
- continue;
- }
- let parsed = match serde_json::from_value::<PropertyValue>(value) {
- Ok(parsed) => parsed,
- Err(e) => {
- tracing::warn!(
- error = ?e,
- definition_id = %row.property_definition_id,
- "skipping unparseable property value for index"
- );
- continue;
- }
- };
- let mut property = IndexedEntityProperty {
- definition_id: row.property_definition_id.to_string(),
- ..Default::default()
- };
- match parsed {
- PropertyValue::SelectOption(ids) => {
- property.values = ids.into_iter().map(|id| id.to_string()).collect();
- }
- PropertyValue::EntityRef(refs) => {
- property.values = refs.into_iter().map(|r| r.entity_id).collect();
- }
- PropertyValue::Link(urls) => property.values = urls,
- PropertyValue::Str(s) => property.values = vec![s],
- PropertyValue::Bool(b) => property.values = vec![b.to_string()],
- PropertyValue::Num(n) => property.number_value = Some(n),
- PropertyValue::Date(d) => property.date_value = Some(d.timestamp_millis()),
- }
- if property.is_empty() {
- continue;
- }
- result.push(property);
+ result.entry(row.entity_id).or_default().push(property);
}
Ok(result)
diff --git a/rust/cloud-storage/search_processing_service/src/api/internal/backfill.rs b/rust/cloud-storage/search_processing_service/src/api/internal/backfill.rs
index a76e7968a7..7d8bf59787 100644
--- a/rust/cloud-storage/search_processing_service/src/api/internal/backfill.rs
+++ b/rust/cloud-storage/search_processing_service/src/api/internal/backfill.rs
@@ -28,7 +28,7 @@ use crate::api::context::ApiContext;
use crate::domain::jobs::{BackfillJobs, JobId};
use crate::domain::models::{
CallBackfillRequest, ChannelBackfillRequest, ChatBackfillRequest, DocumentBackfillRequest,
- EmailBackfillRequest,
+ EmailBackfillRequest, ProjectBackfillRequest, PropertiesBackfillRequest,
};
use crate::domain::service::BackfillService;
@@ -39,6 +39,8 @@ pub fn router() -> Router<ApiContext> {
.route("/channels", post(channels))
.route("/documents", post(documents))
.route("/emails", post(emails))
+ .route("/properties", post(properties))
+ .route("/projects", post(projects))
.route("/{job_id}", get(status))
}
@@ -128,6 +130,40 @@ async fn emails(
.await
}
+#[tracing::instrument(skip(service, jobs, req))]
+async fn properties(
+ State(service): State<Arc<BackfillServiceImpl>>,
+ State(jobs): State<BackfillJobs>,
+ extract::Json(req): extract::Json<PropertiesBackfillRequest>,
+) -> Response {
+ spawn_backfill(
+ service,
+ jobs,
+ "properties",
+ move |svc, progress, cancel| async move {
+ svc.backfill_entity_properties(req, progress, cancel).await
+ },
+ )
+ .await
+}
+
+#[tracing::instrument(skip(service, jobs, req))]
+async fn projects(
+ State(service): State<Arc<BackfillServiceImpl>>,
+ State(jobs): State<BackfillJobs>,
+ extract::Json(req): extract::Json<ProjectBackfillRequest>,
+) -> Response {
+ spawn_backfill(
+ service,
+ jobs,
+ "projects",
+ move |svc, progress, cancel| async move {
+ svc.backfill_projects(req, progress, cancel).await
+ },
+ )
+ .await
+}
+
#[tracing::instrument(skip(jobs))]
async fn status(State(jobs): State<BackfillJobs>, Path(job_id): Path<String>) -> Response {
match jobs.snapshot(&JobId::from(job_id)).await {
diff --git a/rust/cloud-storage/search_processing_service/src/config.rs b/rust/cloud-storage/search_processing_service/src/config.rs
index 45cfeb3b27..335a8728d8 100644
--- a/rust/cloud-storage/search_processing_service/src/config.rs
+++ b/rust/cloud-storage/search_processing_service/src/config.rs
@@ -21,6 +21,7 @@ maybe_env_vars! {
pub struct BackfillChannelsPageSize;
pub struct BackfillDocumentsPageSize;
pub struct BackfillEmailsPageSize;
+ pub struct BackfillProjectsPageSize;
pub struct BackfillJobTtlSeconds;
}
@@ -33,6 +34,7 @@ pub struct BackfillPageSizes {
pub channels: usize,
pub documents: usize,
pub emails: usize,
+ pub projects: usize,
}
const DEFAULT_CALLS_PAGE: usize = 2000;
@@ -40,6 +42,7 @@ const DEFAULT_CHATS_PAGE: usize = 5000;
const DEFAULT_CHANNELS_PAGE: usize = 5000;
const DEFAULT_DOCUMENTS_PAGE: usize = 1000;
const DEFAULT_EMAILS_PAGE: usize = 1000;
+const DEFAULT_PROJECTS_PAGE: usize = 2000;
const DEFAULT_BACKFILL_JOB_TTL_SECONDS: u64 = 24 * 60 * 60;
fn parse_page_size(name: &str, raw_value: Option<&str>, default: usize) -> anyhow::Result<usize> {
@@ -126,6 +129,8 @@ pub struct Config {
pub backfill_documents_page_size: BackfillDocumentsPageSize,
/// DB page size used when backfilling emails.
pub backfill_emails_page_size: BackfillEmailsPageSize,
+ /// DB page size used when backfilling projects.
+ pub backfill_projects_page_size: BackfillProjectsPageSize,
/// DynamoDB table name backing the backfill job registry. Items carry an
/// `expires_at` epoch attribute that DynamoDB's TTL sweeps in the
@@ -172,6 +177,11 @@ impl Config {
self.backfill_emails_page_size.value(),
DEFAULT_EMAILS_PAGE,
)?,
+ projects: parse_page_size(
+ "BACKFILL_PROJECTS_PAGE_SIZE",
+ self.backfill_projects_page_size.value(),
+ DEFAULT_PROJECTS_PAGE,
+ )?,
})
}
diff --git a/rust/cloud-storage/search_processing_service/src/domain/models.rs b/rust/cloud-storage/search_processing_service/src/domain/models.rs
index 6ce294041c..1193133af7 100644
--- a/rust/cloud-storage/search_processing_service/src/domain/models.rs
+++ b/rust/cloud-storage/search_processing_service/src/domain/models.rs
@@ -155,6 +155,32 @@ pub struct DocumentBackfillRequest {
pub index_override: Option<String>,
}
+/// Project backfill filter. All `None` means "every non-deleted project".
+///
+/// `updated_after` / `updated_before` filter on `updatedAt` so incremental
+/// runs (e.g. "anything changed since X") catch projects that existed before
+/// the cutoff but were modified after it.
+#[derive(Debug, Clone, Default, Deserialize)]
+#[serde(default)]
+pub struct ProjectBackfillRequest {
+ pub updated_after: Option<DateTime<Utc>>,
+ pub updated_before: Option<DateTime<Utc>>,
+ /// Override the OpenSearch target index for upserts (e.g. blue/green swap).
+ pub index_override: Option<String>,
+}
+
+/// Keyset (seek-method) pagination cursor for project backfills.
+///
+/// `get_projects_for_search_backfill` walks `"Project"` in
+/// `(updatedAt ASC, id ASC)` order; the cursor carries the last row's
+/// pair so the next page resumes with `WHERE (updatedAt, id) > cursor`.
+/// `None` starts at the beginning.
+#[derive(Debug, Clone)]
+pub struct ProjectBackfillCursor {
+ pub updated_at: DateTime<Utc>,
+ pub project_id: String,
+}
+
/// Email-thread backfill filter.
#[derive(Debug, Clone, Default, Deserialize)]
#[serde(default)]
@@ -166,3 +192,13 @@ pub struct EmailBackfillRequest {
/// the adapter's default.
pub batch_size: Option<usize>,
}
+
+/// Property-only backfill: re-enqueue a property update message for every
+/// entity of one type that has property rows, refreshing the denormalized
+/// `properties` field without re-extracting content. Used after adding the
+/// field to an index's mapping.
+#[derive(Debug, Clone, Deserialize)]
+pub struct PropertiesBackfillRequest {
+ /// The property entity type to backfill (e.g. "thread").
+ pub entity_type: String,
+}
diff --git a/rust/cloud-storage/search_processing_service/src/domain/ports.rs b/rust/cloud-storage/search_processing_service/src/domain/ports.rs
index d32496ba48..ed4eedee0c 100644
--- a/rust/cloud-storage/search_processing_service/src/domain/ports.rs
+++ b/rust/cloud-storage/search_processing_service/src/domain/ports.rs
@@ -19,7 +19,8 @@ use sqs_client::search::SearchQueueMessage;
use super::models::{
BackfillError, CallBackfillCursor, CallBackfillRequest, ChannelBackfillRequest,
ChatBackfillCursor, ChatBackfillRequest, DocumentBackfillCursor, DocumentBackfillRequest,
- EmailBackfillRequest, SourcePage,
+ EmailBackfillRequest, ProjectBackfillCursor, ProjectBackfillRequest, PropertiesBackfillRequest,
+ SourcePage,
};
/// Publishes batches of search-event messages.
@@ -84,4 +85,22 @@ pub trait BackfillSource: Send + Sync + 'static {
req: &EmailBackfillRequest,
offset: usize,
) -> impl Future<Output = Result<SourcePage, BackfillError>> + Send;
+
+ /// Distinct entity ids holding property rows of the requested type,
+ /// paginated by plain offset over `entity_properties`.
+ fn fetch_entity_properties(
+ &self,
+ req: &PropertiesBackfillRequest,
+ offset: usize,
+ ) -> impl Future<Output = Result<SourcePage, BackfillError>> + Send;
+ /// Projects paginate by keyset cursor (mirroring documents): each call
+ /// passes the cursor of the last row from the previous page (or `None`
+ /// for the first page), and the implementation returns the page plus
+ /// the cursor to feed back into the next call. An empty page signals
+ /// end-of-source.
+ fn fetch_projects(
+ &self,
+ req: &ProjectBackfillRequest,
+ cursor: Option<ProjectBackfillCursor>,
+ ) -> impl Future<Output = Result<(SourcePage, Option<ProjectBackfillCursor>), BackfillError>> + Send;
}
diff --git a/rust/cloud-storage/search_processing_service/src/domain/service.rs b/rust/cloud-storage/search_processing_service/src/domain/service.rs
index 4aca6e7b16..52cabb640d 100644
--- a/rust/cloud-storage/search_processing_service/src/domain/service.rs
+++ b/rust/cloud-storage/search_processing_service/src/domain/service.rs
@@ -15,7 +15,8 @@ use tokio_util::sync::CancellationToken;
use super::jobs::JobProgress;
use super::models::{
BackfillError, BackfillReceipt, CallBackfillRequest, ChannelBackfillRequest,
- ChatBackfillRequest, DocumentBackfillRequest, EmailBackfillRequest, SourcePage,
+ ChatBackfillRequest, DocumentBackfillRequest, EmailBackfillRequest, ProjectBackfillRequest,
+ PropertiesBackfillRequest, SourcePage,
};
use super::ports::{BackfillSource, SearchEventPublisher};
@@ -97,6 +98,18 @@ pub trait BackfillService: Send + Sync + 'static {
progress: Arc<JobProgress>,
cancel: CancellationToken,
) -> impl Future<Output = Result<BackfillReceipt, BackfillError>> + Send;
+ fn backfill_entity_properties(
+ &self,
+ req: PropertiesBackfillRequest,
+ progress: Arc<JobProgress>,
+ cancel: CancellationToken,
+ ) -> impl Future<Output = Result<BackfillReceipt, BackfillError>> + Send;
+ fn backfill_projects(
+ &self,
+ req: ProjectBackfillRequest,
+ progress: Arc<JobProgress>,
+ cancel: CancellationToken,
+ ) -> impl Future<Output = Result<BackfillReceipt, BackfillError>> + Send;
}
pub struct BackfillOrchestrator<S, P> {
@@ -218,6 +231,30 @@ where
})
.await
}
+
+ async fn backfill_entity_properties(
+ &self,
+ req: PropertiesBackfillRequest,
+ progress: Arc<JobProgress>,
+ cancel: CancellationToken,
+ ) -> Result<BackfillReceipt, BackfillError> {
+ drain_source(&self.publisher, &progress, &cancel, |offset| {
+ self.source.fetch_entity_properties(&req, offset)
+ })
+ .await
+ }
+
+ async fn backfill_projects(
+ &self,
+ req: ProjectBackfillRequest,
+ progress: Arc<JobProgress>,
+ cancel: CancellationToken,
+ ) -> Result<BackfillReceipt, BackfillError> {
+ drain_source_with_cursor(&self.publisher, &progress, &cancel, |cursor| {
+ self.source.fetch_projects(&req, cursor)
+ })
+ .await
+ }
}
#[cfg(test)]
diff --git a/rust/cloud-storage/search_processing_service/src/outbound/source.rs b/rust/cloud-storage/search_processing_service/src/outbound/source.rs
index 18ce9aeefa..981d3b43dc 100644
--- a/rust/cloud-storage/search_processing_service/src/outbound/source.rs
+++ b/rust/cloud-storage/search_processing_service/src/outbound/source.rs
@@ -1,22 +1,30 @@
use std::collections::HashMap;
+use std::str::FromStr;
use model::document::FileType;
+use models_properties::EntityType;
use sqlx::PgPool;
use sqs_client::search::{
SearchQueueMessage, call::CallRecordMessage, channel::ChannelMessageUpdate, chat::ChatMessage,
- email::EmailThreadBatchMessage,
+ document::DocumentPropertiesUpdate, email::EmailThreadBatchMessage, project::UpsertProject,
};
use crate::config::BackfillPageSizes;
use crate::domain::models::{
BackfillError, CallBackfillCursor, CallBackfillRequest, ChannelBackfillRequest,
ChatBackfillCursor, ChatBackfillRequest, DocumentBackfillCursor, DocumentBackfillRequest,
- EmailBackfillRequest, SourcePage,
+ EmailBackfillRequest, ProjectBackfillCursor, ProjectBackfillRequest, PropertiesBackfillRequest,
+ SourcePage,
};
use crate::domain::ports::BackfillSource;
const DEFAULT_EMAIL_BATCH_SIZE: usize = 50;
+/// Page size for the properties backfill's distinct-entity-id scan. A fixed
+/// value rather than a config knob: property rows are few and one message is
+/// enqueued per entity.
+const PROPERTIES_PAGE_SIZE: usize = 5000;
+
/// Postgres-backed [`BackfillSource`] for every search-indexed entity. One
/// struct, one DB pool, per-entity page sizes — collapses what used to be
/// five parallel adapters. New entity types just add a method here.
@@ -326,4 +334,84 @@ impl BackfillSource for PgBackfillSource {
rows_consumed,
})
}
+
+ async fn fetch_entity_properties(
+ &self,
+ req: &PropertiesBackfillRequest,
+ offset: usize,
+ ) -> Result<SourcePage, BackfillError> {
+ let entity_type = EntityType::from_str(&req.entity_type)
+ .map_err(|e| BackfillError::Source(anyhow::Error::new(e)))?;
+
+ let entity_ids =
+ properties_db_client::entity_properties::get::get_entity_ids_with_properties(
+ &self.db,
+ entity_type,
+ PROPERTIES_PAGE_SIZE as i64,
+ offset as i64,
+ )
+ .await
+ .map_err(|e| BackfillError::Source(e.into()))?;
+
+ let rows_consumed = entity_ids.len();
+ let messages: Vec<SearchQueueMessage> = entity_ids
+ .into_iter()
+ .map(|entity_id| {
+ SearchQueueMessage::UpdateDocumentProperties(DocumentPropertiesUpdate {
+ document_id: entity_id,
+ entity_type: entity_type.to_string(),
+ })
+ })
+ .collect();
+
+ Ok(SourcePage {
+ messages,
+ rows_consumed,
+ })
+ }
+
+ async fn fetch_projects(
+ &self,
+ req: &ProjectBackfillRequest,
+ cursor: Option<ProjectBackfillCursor>,
+ ) -> Result<(SourcePage, Option<ProjectBackfillCursor>), BackfillError> {
+ let db_cursor = cursor.map(|c| (c.updated_at, c.project_id));
+ let batch = macro_db_client::projects::get_projects_for_search_backfill(
+ &self.db,
+ self.page_sizes.projects as i64,
+ db_cursor,
+ req.updated_after,
+ req.updated_before,
+ )
+ .await
+ .map_err(BackfillError::Source)?;
+
+ // `updated_at` is NOT NULL in the schema but sqlx types it as Option
+ // because of the timestamptz cast; if it ever came back None we'd
+ // rather stop pagination than build a bogus cursor.
+ let next_cursor = batch.last().and_then(|p| {
+ p.updated_at.map(|updated_at| ProjectBackfillCursor {
+ updated_at,
+ project_id: p.project_id.clone(),
+ })
+ });
+ let rows_consumed = batch.len();
+ let messages: Vec<SearchQueueMessage> = batch
+ .into_iter()
+ .map(|p| {
+ SearchQueueMessage::UpsertProject(UpsertProject {
+ project_id: p.project_id,
+ index_override: req.index_override.clone(),
+ })
+ })
+ .collect();
+
+ Ok((
+ SourcePage {
+ messages,
+ rows_consumed,
+ },
+ next_cursor,
+ ))
+ }
}
diff --git a/rust/cloud-storage/search_processing_service/src/process/chat.rs b/rust/cloud-storage/search_processing_service/src/process/chat.rs
index 52105379db..797d5c26c8 100644
--- a/rust/cloud-storage/search_processing_service/src/process/chat.rs
+++ b/rust/cloud-storage/search_processing_service/src/process/chat.rs
@@ -1,9 +1,13 @@
use anyhow::Context;
+use models_properties::EntityType;
use opensearch_client::{
OpensearchClient, date_format::EpochSeconds, upsert::chat_message::UpsertChatMessageArgs,
};
+use properties_db_client::entity_properties::get::get_entity_properties_for_index;
use sqs_client::search::chat::{ChatMessage, RemoveChatMessage};
+use crate::process::properties::to_indexed_properties;
+
/// Handles the processing of chat messages
#[tracing::instrument(skip(opensearch_client, db))]
pub async fn insert_chat_message(
@@ -37,6 +41,15 @@ pub async fn insert_chat_message(
return Ok(());
}
+ // The parent doc is a full overwrite, so its properties must ride every
+ // write or values set by the property-update path get wiped. A fetch
+ // failure propagates (retry) rather than being mistaken for "empty".
+ let properties = to_indexed_properties(
+ get_entity_properties_for_index(db, chat_message.chat_id.as_str(), EntityType::Chat)
+ .await
+ .context("failed to fetch chat properties for search index")?,
+ );
+
opensearch_client
.upsert_chat_message(
&UpsertChatMessageArgs {
@@ -48,6 +61,7 @@ pub async fn insert_chat_message(
title: info.name,
content: info.content,
role: info.role,
+ properties,
},
index_override,
)
diff --git a/rust/cloud-storage/search_processing_service/src/process/document/mod.rs b/rust/cloud-storage/search_processing_service/src/process/document/mod.rs
index 58662ac8aa..7bd7e6da5a 100644
--- a/rust/cloud-storage/search_processing_service/src/process/document/mod.rs
+++ b/rust/cloud-storage/search_processing_service/src/process/document/mod.rs
@@ -1,7 +1,7 @@
use anyhow::Context;
use model::document::FileTypeExt;
use opensearch_client::OpensearchClient;
-use sqs_client::search::document::{DocumentId, DocumentPropertiesUpdate, SearchExtractorMessage};
+use sqs_client::search::document::{DocumentId, SearchExtractorMessage};
mod document_info;
mod raw_document;
@@ -69,18 +69,3 @@ pub async fn process_extract_sync_message(
Ok(())
}
-
-pub async fn process_property_update(
- opensearch_client: &OpensearchClient,
- db: &sqlx::Pool<sqlx::Postgres>,
- message: &DocumentPropertiesUpdate,
-) -> anyhow::Result<()> {
- raw_document::update_search_with_property_update(opensearch_client, db, message)
- .await
- .context(format!(
- "{} unable to update document properties in search",
- message.document_id
- ))?;
-
- Ok(())
-}
diff --git a/rust/cloud-storage/search_processing_service/src/process/document/raw_document.rs b/rust/cloud-storage/search_processing_service/src/process/document/raw_document.rs
index bcc8879611..a84f3abc2c 100644
--- a/rust/cloud-storage/search_processing_service/src/process/document/raw_document.rs
+++ b/rust/cloud-storage/search_processing_service/src/process/document/raw_document.rs
@@ -8,9 +8,7 @@ use models_properties::EntityType;
use models_search::document::MarkdownParseResult;
use models_search::unified::is_searchable_association;
use opensearch_client::{
- OpensearchClient,
- date_format::EpochSeconds,
- upsert::document::{IndexedProperty, UpsertDocumentArgs},
+ OpensearchClient, date_format::EpochSeconds, upsert::document::UpsertDocumentArgs,
};
use properties_db_client::entity_properties::get::get_entity_properties_for_index;
use s3_key::{
@@ -23,9 +21,10 @@ use crate::parsers::pdf::parse_pdf_pages;
use crate::{
parsers::{canvas::parse_canvas, markdown::parse_markdown_legacy},
process::document::document_info::{DocumentInfo, get_document_info},
+ process::properties::to_indexed_properties,
};
-use super::{DocumentPropertiesUpdate, SearchExtractorMessage};
+use super::SearchExtractorMessage;
async fn upsert_document(
opensearch_client: &OpensearchClient,
@@ -102,15 +101,7 @@ async fn attach_indexed_properties(
if properties.is_empty() {
return Ok(());
}
- let indexed: Vec<IndexedProperty> = properties
- .into_iter()
- .map(|p| IndexedProperty {
- definition_id: p.definition_id,
- values: p.values,
- number_value: p.number_value,
- date_value: p.date_value,
- })
- .collect();
+ let indexed = to_indexed_properties(properties);
for upsert in upserts.iter_mut() {
upsert.properties = indexed.clone();
}
@@ -429,34 +420,6 @@ pub async fn update_search_with_sync_document(
Ok(())
}
-/// Refresh only the indexed `properties` of a document after a property
-/// mutation, without re-extracting its content.
-pub async fn update_search_with_property_update(
- opensearch_client: &OpensearchClient,
- db: &sqlx::Pool<sqlx::Postgres>,
- message: &DocumentPropertiesUpdate,
-) -> anyhow::Result<()> {
- let entity_type = EntityType::from_str(&message.entity_type)
- .with_context(|| format!("invalid entity_type '{}'", message.entity_type))?;
- let properties = get_entity_properties_for_index(db, &message.document_id, entity_type)
- .await
- .context("failed to fetch properties for reindex")?;
- let indexed: Vec<IndexedProperty> = properties
- .into_iter()
- .map(|p| IndexedProperty {
- definition_id: p.definition_id,
- values: p.values,
- number_value: p.number_value,
- date_value: p.date_value,
- })
- .collect();
- opensearch_client
- .update_document_properties(&message.document_id, &indexed)
- .await
- .context("failed to update document properties in search index")?;
- Ok(())
-}
-
#[cfg(test)]
mod tests {
use macro_user_id::user_id::MacroUserIdStr;
diff --git a/rust/cloud-storage/search_processing_service/src/process/email/upsert.rs b/rust/cloud-storage/search_processing_service/src/process/email/upsert.rs
index c7256eb83a..816fe820f1 100644
--- a/rust/cloud-storage/search_processing_service/src/process/email/upsert.rs
+++ b/rust/cloud-storage/search_processing_service/src/process/email/upsert.rs
@@ -1,13 +1,19 @@
use anyhow::Context;
use chrono::Utc;
use models_email::service::label::system_labels;
+use models_properties::EntityType;
use opensearch_client::{
OpensearchClient, date_format::EpochSeconds, upsert::email::UpsertEmailArgs,
};
+use properties_db_client::entity_properties::get::{
+ get_entity_properties_for_index, get_entity_properties_for_index_batch,
+};
use sqlx::PgPool;
use sqs_client::search::email::{EmailMessage, EmailThreadBatchMessage, EmailThreadMessage};
use uuid::Uuid;
+use crate::process::properties::to_indexed_properties;
+
pub async fn process_upsert_message(
opensearch_client: &OpensearchClient,
db: &PgPool,
@@ -54,6 +60,19 @@ pub async fn process_upsert_message(
.transpose()?
.unwrap_or(now);
+ // A full index overwrites the doc, so thread properties must ride along
+ // or a reindex would drop them. A fetch failure propagates (retry)
+ // instead of being mistaken for "no properties".
+ let properties = to_indexed_properties(
+ get_entity_properties_for_index(
+ db,
+ &message_info.thread_db_id.to_string(),
+ EntityType::Thread,
+ )
+ .await
+ .context("failed to fetch thread properties for search index")?,
+ );
+
let upsert_email_message_args: UpsertEmailArgs = UpsertEmailArgs {
message_id: upsert_email_message.message_id.clone(),
link_id: message_info.link_id.to_string(),
@@ -109,6 +128,7 @@ pub async fn process_upsert_message(
.internal_date_ts
.map(|date| EpochSeconds::new(date.timestamp()))
.transpose()?,
+ properties,
};
opensearch_client
@@ -134,6 +154,14 @@ pub async fn process_upsert_thread_message(
let now = EpochSeconds::new(Utc::now().timestamp())?;
+ // A full index overwrites each doc, so thread properties must ride along
+ // or a reindex would drop them.
+ let properties = to_indexed_properties(
+ get_entity_properties_for_index(db, &thread_id.to_string(), EntityType::Thread)
+ .await
+ .context("failed to fetch thread properties for search index")?,
+ );
+
loop {
let messages =
email_db_client::messages::get_parsed_search::get_paginated_parsed_search_messages_by_thread_id(
@@ -216,6 +244,7 @@ pub async fn process_upsert_thread_message(
content,
updated_at_seconds: updated_at,
sent_at_seconds: sent_at,
+ properties: properties.clone(),
});
} else {
tracing::warn!("no content found for email message");
@@ -267,6 +296,17 @@ pub async fn process_upsert_thread_batch_message(
.await
.context("failed to get batch thread messages")?;
+ // A full index overwrites each doc, so thread properties must ride along
+ // or a reindex would drop them.
+ let thread_id_strings: Vec<String> = thread_ids.iter().map(Uuid::to_string).collect();
+ let properties_by_thread =
+ get_entity_properties_for_index_batch(db, &thread_id_strings, EntityType::Thread)
+ .await
+ .context("failed to fetch thread properties for search index")?
+ .into_iter()
+ .map(|(thread_id, properties)| (thread_id, to_indexed_properties(properties)))
+ .collect::<std::collections::HashMap<_, _>>();
+
let mut upsert_email_message_args = Vec::new();
for message in messages {
@@ -331,6 +371,10 @@ pub async fn process_upsert_thread_batch_message(
content,
updated_at_seconds: updated_at,
sent_at_seconds: sent_at,
+ properties: properties_by_thread
+ .get(&message.thread_db_id.to_string())
+ .cloned()
+ .unwrap_or_default(),
});
} else {
tracing::warn!("no content found for email message");
diff --git a/rust/cloud-storage/search_processing_service/src/process/mod.rs b/rust/cloud-storage/search_processing_service/src/process/mod.rs
index 2ed0e7dfa5..8406f309de 100644
--- a/rust/cloud-storage/search_processing_service/src/process/mod.rs
+++ b/rust/cloud-storage/search_processing_service/src/process/mod.rs
@@ -4,6 +4,8 @@ mod chat;
pub mod context;
mod document;
mod email;
+mod project;
+mod properties;
mod user;
pub mod worker;
@@ -91,7 +93,8 @@ pub async fn process_message(
.await?;
}
SearchQueueMessage::UpdateDocumentProperties(message) => {
- document::process_property_update(&ctx.opensearch_client, &ctx.db, &message).await?;
+ properties::process_entity_property_update(&ctx.opensearch_client, &ctx.db, &message)
+ .await?;
}
SearchQueueMessage::ChatMessage(message) => {
chat::insert_chat_message(&ctx.opensearch_client, &ctx.db, &message).await?;
@@ -105,6 +108,12 @@ pub async fn process_message(
SearchQueueMessage::RemoveCallRecord(message) => {
call::process_remove_call_record(&ctx.opensearch_client, &message).await?;
}
+ SearchQueueMessage::UpsertProject(message) => {
+ project::upsert_project(&ctx.opensearch_client, &ctx.db, &message).await?;
+ }
+ SearchQueueMessage::RemoveProject(message) => {
+ project::remove_project(&ctx.opensearch_client, &message).await?;
+ }
}
ctx.worker.cleanup_message(message).await?;
diff --git a/rust/cloud-storage/search_processing_service/src/process/project.rs b/rust/cloud-storage/search_processing_service/src/process/project.rs
new file mode 100644
index 0000000000..708a919caf
--- /dev/null
+++ b/rust/cloud-storage/search_processing_service/src/process/project.rs
@@ -0,0 +1,104 @@
+use anyhow::Context;
+use models_properties::EntityType;
+use opensearch_client::{
+ OpensearchClient,
+ date_format::EpochSeconds,
+ upsert::{project::UpsertProjectArgs, properties::IndexedProperty},
+};
+use properties_db_client::entity_properties::get::get_entity_properties_for_index;
+use sqs_client::search::project::{RemoveProject, UpsertProject};
+
+/// Fetch the project's indexed properties, flattened for the search index.
+async fn fetch_indexed_properties(
+ db: &sqlx::Pool<sqlx::Postgres>,
+ project_id: &str,
+) -> anyhow::Result<Vec<IndexedProperty>> {
+ let properties = get_entity_properties_for_index(db, project_id, EntityType::Project)
+ .await
+ .context("failed to fetch project properties for search index")?;
+ Ok(properties
+ .into_iter()
+ .map(|p| IndexedProperty {
+ definition_id: p.definition_id,
+ values: p.values,
+ number_value: p.number_value,
+ date_value: p.date_value,
+ })
+ .collect())
+}
+
+/// Handles upserting a project into the opensearch index. Re-reads the
+/// project row so the indexed doc always reflects the current state; a
+/// missing or soft-deleted row turns the upsert into a removal.
+#[tracing::instrument(skip(opensearch_client, db), err)]
+pub async fn upsert_project(
+ opensearch_client: &OpensearchClient,
+ db: &sqlx::Pool<sqlx::Postgres>,
+ message: &UpsertProject,
+) -> anyhow::Result<()> {
+ let index_override = message.index_override.as_deref();
+
+ let project = macro_db_client::projects::get_project::get_project_for_search(
+ db,
+ message.project_id.as_str(),
+ )
+ .await
+ .context("failed to get project for search")?;
+
+ let Some(project) = project else {
+ tracing::trace!("project row is gone, removing from search index");
+ opensearch_client
+ .delete_project(message.project_id.as_str(), index_override)
+ .await
+ .context("failed to delete missing project from search")?;
+ return Ok(());
+ };
+
+ if project.deleted_at.is_some() {
+ tracing::trace!("project is deleted, removing from search index");
+ opensearch_client
+ .delete_project(message.project_id.as_str(), index_override)
+ .await
+ .context("failed to delete project from search")?;
+ return Ok(());
+ }
+
+ let properties = fetch_indexed_properties(db, &message.project_id).await?;
+
+ let created_at = project.created_at.context("project missing createdAt")?;
+ let updated_at = project.updated_at.context("project missing updatedAt")?;
+
+ opensearch_client
+ .upsert_project(
+ &UpsertProjectArgs {
+ project_id: project.id,
+ name: project.name,
+ owner_id: project.user_id,
+ parent_project_id: project.parent_id,
+ created_at_seconds: EpochSeconds::new(created_at.timestamp())?,
+ updated_at_seconds: EpochSeconds::new(updated_at.timestamp())?,
+ properties,
+ },
+ index_override,
+ )
+ .await
+ .context("failed to upsert project")?;
+
+ Ok(())
+}
+
+/// Handles the removal of a project from the opensearch index
+#[tracing::instrument(skip(opensearch_client), err)]
+pub async fn remove_project(
+ opensearch_client: &OpensearchClient,
+ message: &RemoveProject,
+) -> anyhow::Result<()> {
+ opensearch_client
+ .delete_project(
+ message.project_id.as_str(),
+ message.index_override.as_deref(),
+ )
+ .await?;
+
+ Ok(())
+}
diff --git a/rust/cloud-storage/search_processing_service/src/process/properties.rs b/rust/cloud-storage/search_processing_service/src/process/properties.rs
new file mode 100644
index 0000000000..424baf16e0
--- /dev/null
+++ b/rust/cloud-storage/search_processing_service/src/process/properties.rs
@@ -0,0 +1,68 @@
+use std::str::FromStr;
+
+use anyhow::Context;
+use models_properties::EntityType;
+use opensearch_client::{OpensearchClient, upsert::properties::IndexedProperty};
+use properties_db_client::entity_properties::get::{
+ IndexedEntityProperty, get_entity_properties_for_index,
+};
+use sqs_client::search::document::DocumentPropertiesUpdate;
+
+/// Refresh only the indexed `properties` of an entity after a property
+/// mutation, without re-extracting its content. Fetches once, then routes to
+/// the writer for the index that holds the entity type. Entity types without
+/// a property-indexing search index are a no-op.
+pub async fn process_entity_property_update(
+ opensearch_client: &OpensearchClient,
+ db: &sqlx::Pool<sqlx::Postgres>,
+ message: &DocumentPropertiesUpdate,
+) -> anyhow::Result<()> {
+ let entity_id = &message.document_id;
+ let entity_type = EntityType::from_str(&message.entity_type)
+ .with_context(|| format!("invalid entity_type '{}'", message.entity_type))?;
+
+ let rows = get_entity_properties_for_index(db, entity_id, entity_type)
+ .await
+ .context("failed to fetch properties for reindex")?;
+ let properties = to_indexed_properties(rows);
+
+ match entity_type {
+ EntityType::Task | EntityType::Document => opensearch_client
+ .update_document_properties(entity_id, &properties)
+ .await
+ .context("failed to update document properties in search index"),
+ EntityType::Thread => opensearch_client
+ .update_email_thread_properties(entity_id, &properties)
+ .await
+ .context("failed to update email thread properties in search index"),
+ EntityType::Chat => opensearch_client
+ .update_chat_properties(entity_id, &properties)
+ .await
+ .context("failed to update chat properties in search index"),
+ EntityType::Project => opensearch_client
+ .update_project_properties(entity_id, &properties)
+ .await
+ .context("failed to update project properties in search index"),
+ other => {
+ tracing::warn!(
+ entity_type = %other,
+ entity_id = %entity_id,
+ "property update for unindexed entity type"
+ );
+ Ok(())
+ }
+ }
+}
+
+/// Convert DB-flattened properties into the search index's document shape.
+pub fn to_indexed_properties(properties: Vec<IndexedEntityProperty>) -> Vec<IndexedProperty> {
+ properties
+ .into_iter()
+ .map(|p| IndexedProperty {
+ definition_id: p.definition_id,
+ values: p.values,
+ number_value: p.number_value,
+ date_value: p.date_value,
+ })
+ .collect()
+}
diff --git a/rust/cloud-storage/search_service/src/api/search/chat.rs b/rust/cloud-storage/search_service/src/api/search/chat.rs
index 82f32c0855..995edf1028 100644
--- a/rust/cloud-storage/search_service/src/api/search/chat.rs
+++ b/rust/cloud-storage/search_service/src/api/search/chat.rs
@@ -1,9 +1,11 @@
use crate::api::context::SearchHandlerState;
use crate::api::search::simple::SearchError;
use indexmap::IndexMap;
+use models_properties::{EntityReference, EntityType};
use models_search::chat::{
ChatMessageSearchResult, ChatSearchResponseItem, ChatSearchResponseItemWithMetadata,
};
+use models_soup::SoupProperty;
use opensearch_client::search::model::SearchGotoContent;
use sqlx::types::Uuid;
use std::collections::HashMap;
@@ -32,9 +34,35 @@ pub(in crate::api::search) async fn enrich_chats(
.await
.map_err(SearchError::InternalError)?;
+ // Fetch chat properties (e.g. tags) so rows can render and re-check
+ // them, mirroring the documents/emails enrichment.
+ let chat_entity_refs: Vec<EntityReference> = chat_ids
+ .iter()
+ .map(|id| EntityReference::new(id.to_string(), EntityType::Chat))
+ .collect();
+ let properties_map: HashMap<String, Vec<SoupProperty>> =
+ properties_db_client::entity_properties::get::get_bulk_entity_properties_values(
+ &ctx.db,
+ &chat_entity_refs,
+ )
+ .await
+ .inspect_err(|e| tracing::error!(error=?e, "failed to fetch chat properties"))
+ .unwrap_or_default()
+ .into_iter()
+ .map(|(id, props)| {
+ (
+ id,
+ props
+ .into_iter()
+ .map(SoupProperty::from)
+ .collect::<Vec<_>>(),
+ )
+ })
+ .collect();
+
// Construct enriched results
- let enriched_results =
- construct_search_result(results, chat_histories).map_err(SearchError::InternalError)?;
+ let enriched_results = construct_search_result(results, chat_histories, properties_map)
+ .map_err(SearchError::InternalError)?;
Ok(enriched_results)
}
@@ -42,6 +70,7 @@ pub(in crate::api::search) async fn enrich_chats(
pub fn construct_search_result(
search_results: Vec<opensearch_client::search::model::SearchHit>,
chat_histories: HashMap<String, macro_db_client::chat::get::ChatHistoryInfo>,
+ properties_map: HashMap<String, Vec<SoupProperty>>,
) -> anyhow::Result<Vec<ChatSearchResponseItemWithMetadata>> {
// construct entity hit map of id -> vec<hits> using IndexMap to preserve insertion order
let entity_id_hit_map: IndexMap<Uuid, Vec<ChatMessageSearchResult>> = search_results
@@ -83,8 +112,13 @@ pub fn construct_search_result(
project_id: info.project_id.clone(),
deleted_at: info.deleted_at,
};
+ let properties = properties_map
+ .get(&entity_id.to_string())
+ .cloned()
+ .filter(|p| !p.is_empty());
Some(ChatSearchResponseItemWithMetadata {
metadata: Some(metadata),
+ properties,
extra: ChatSearchResponseItem {
id: entity_id,
chat_id: entity_id,
diff --git a/rust/cloud-storage/search_service/src/api/search/chat/test.rs b/rust/cloud-storage/search_service/src/api/search/chat/test.rs
index 93fdf05069..d162143a94 100644
--- a/rust/cloud-storage/search_service/src/api/search/chat/test.rs
+++ b/rust/cloud-storage/search_service/src/api/search/chat/test.rs
@@ -45,7 +45,7 @@ fn create_chat_history(chat_id: &str) -> macro_db_client::chat::get::ChatHistory
#[test]
fn test_empty_input() {
let input = vec![];
- let result = construct_search_result(input, HashMap::new()).unwrap();
+ let result = construct_search_result(input, HashMap::new(), HashMap::new()).unwrap();
assert_eq!(result.len(), 0);
}
@@ -73,7 +73,7 @@ fn test_single_chat_with_content() {
},
);
- let result = construct_search_result(input, chat_histories).unwrap();
+ let result = construct_search_result(input, chat_histories, HashMap::new()).unwrap();
assert_eq!(result.len(), 1);
assert_eq!(
@@ -116,7 +116,7 @@ fn test_single_chat_without_content() {
create_chat_history("11111111-1111-1111-1111-111111111111"),
);
- let result = construct_search_result(input, chat_histories).unwrap();
+ let result = construct_search_result(input, chat_histories, HashMap::new()).unwrap();
assert_eq!(result.len(), 1);
assert_eq!(
@@ -154,7 +154,7 @@ fn test_single_chat_multiple_messages() {
create_chat_history("11111111-1111-1111-1111-111111111111"),
);
- let result = construct_search_result(input, chat_histories).unwrap();
+ let result = construct_search_result(input, chat_histories, HashMap::new()).unwrap();
assert_eq!(result.len(), 1);
assert_eq!(
@@ -203,7 +203,7 @@ fn test_multiple_chats() {
create_chat_history("22222222-2222-2222-2222-222222222222"),
);
- let result = construct_search_result(input, chat_histories).unwrap();
+ let result = construct_search_result(input, chat_histories, HashMap::new()).unwrap();
assert_eq!(result.len(), 2);
@@ -244,7 +244,7 @@ fn test_mixed_content_presence() {
create_chat_history("11111111-1111-1111-1111-111111111111"),
);
- let result = construct_search_result(input, chat_histories).unwrap();
+ let result = construct_search_result(input, chat_histories, HashMap::new()).unwrap();
assert_eq!(result.len(), 1);
assert_eq!(
@@ -288,7 +288,7 @@ fn test_user_id_taken_from_first_result() {
create_chat_history("11111111-1111-1111-1111-111111111111"),
);
- let result = construct_search_result(input, chat_histories).unwrap();
+ let result = construct_search_result(input, chat_histories, HashMap::new()).unwrap();
assert_eq!(result.len(), 1);
// user_id should come from the first result (base_search_result)
@@ -321,7 +321,7 @@ fn test_chat_history_timestamps() {
chat_histories.insert("11111111-1111-1111-1111-111111111111".to_string(), history);
// Call the function under test
- let result = construct_search_result(input, chat_histories).unwrap();
+ let result = construct_search_result(input, chat_histories, HashMap::new()).unwrap();
// Verify that timestamps were copied from the chat history
assert_eq!(result.len(), 1);
@@ -355,7 +355,7 @@ fn test_chat_history_missing_entry() {
chat_histories.insert("different_chat".to_string(), history);
// Call the function under test
- let result = construct_search_result(input, chat_histories).unwrap();
+ let result = construct_search_result(input, chat_histories, HashMap::new()).unwrap();
// Chats without history info should not be returned
assert_eq!(result.len(), 0);
@@ -387,7 +387,7 @@ fn test_chat_history_deleted() {
},
);
- let result = construct_search_result(input_deleted, chat_histories).unwrap();
+ let result = construct_search_result(input_deleted, chat_histories, HashMap::new()).unwrap();
// Deleted chat should be returned with metadata including deleted_at
assert_eq!(result.len(), 1);
@@ -406,7 +406,7 @@ fn test_chat_history_deleted() {
let chat_histories_not_found = HashMap::new(); // No entry = not found
let result_not_found =
- construct_search_result(input_not_found, chat_histories_not_found).unwrap();
+ construct_search_result(input_not_found, chat_histories_not_found, HashMap::new()).unwrap();
// Chat not in DB should not be returned
assert_eq!(result_not_found.len(), 0);
@@ -437,7 +437,7 @@ fn test_chat_history_null_viewed_at() {
chat_histories.insert("11111111-1111-1111-1111-111111111111".to_string(), history);
// Call the function under test
- let result = construct_search_result(input, chat_histories).unwrap();
+ let result = construct_search_result(input, chat_histories, HashMap::new()).unwrap();
// Verify that timestamps were copied correctly and viewed_at is None
assert_eq!(result.len(), 1);
@@ -487,9 +487,12 @@ fn test_sort_stability() {
chat_histories.insert(chat_id.to_string(), create_chat_history(chat_id));
}
- let result1 = construct_search_result(input.clone(), chat_histories.clone()).unwrap();
- let result2 = construct_search_result(input.clone(), chat_histories.clone()).unwrap();
- let result3 = construct_search_result(input.clone(), chat_histories.clone()).unwrap();
+ let result1 =
+ construct_search_result(input.clone(), chat_histories.clone(), HashMap::new()).unwrap();
+ let result2 =
+ construct_search_result(input.clone(), chat_histories.clone(), HashMap::new()).unwrap();
+ let result3 =
+ construct_search_result(input.clone(), chat_histories.clone(), HashMap::new()).unwrap();
assert_eq!(result1.len(), 5);
assert_eq!(result2.len(), 5);
diff --git a/rust/cloud-storage/search_service/src/api/search/crm_company.rs b/rust/cloud-storage/search_service/src/api/search/crm_company.rs
index 6613d141e5..a0d7004d31 100644
--- a/rust/cloud-storage/search_service/src/api/search/crm_company.rs
+++ b/rust/cloud-storage/search_service/src/api/search/crm_company.rs
@@ -1,6 +1,6 @@
//! CRM company search source + enrichment for unified search.
//!
-//! CRM companies are Postgres-only (like projects) and gated by a
+//! CRM companies are Postgres-only and gated by a
//! capability-token receipt, so they don't flow through the OpenSearch
//! split/enrich path: [`search_company_names`] synthesizes `SearchHit`s
//! the same way the other name searches do, and [`enrich_crm_companies`]
diff --git a/rust/cloud-storage/search_service/src/api/search/email.rs b/rust/cloud-storage/search_service/src/api/search/email.rs
index 4d2b35ae34..9dfc225bbd 100644
--- a/rust/cloud-storage/search_service/src/api/search/email.rs
+++ b/rust/cloud-storage/search_service/src/api/search/email.rs
@@ -3,10 +3,12 @@ use crate::api::search::simple::SearchError;
use email_db_client::contacts::get::ThreadContactsMap;
use indexmap::IndexMap;
use models_email::service::message::{MessageSenderInfo, ThreadHistoryInfo};
+use models_properties::{EntityReference, EntityType};
use models_search::email::{
EmailSearchParticipant, EmailSearchResponseItem, EmailSearchResponseItemWithMetadata,
EmailSearchResult,
};
+use models_soup::SoupProperty;
use opensearch_client::search::model::SearchGotoContent;
use sqlx::types::Uuid;
use std::collections::HashMap;
@@ -71,10 +73,41 @@ pub(in crate::api::search) async fn enrich_emails(
)
.map_err(SearchError::InternalError)?;
+ // Fetch thread properties (e.g. tags) so rows can render and re-check
+ // them, mirroring the documents enrichment.
+ let thread_entity_refs: Vec<EntityReference> = thread_ids
+ .iter()
+ .map(|id| EntityReference::new(id.to_string(), EntityType::Thread))
+ .collect();
+ let properties_map: HashMap<String, Vec<SoupProperty>> =
+ properties_db_client::entity_properties::get::get_bulk_entity_properties_values(
+ &ctx.db,
+ &thread_entity_refs,
+ )
+ .await
+ .inspect_err(|e| tracing::error!(error=?e, "failed to fetch thread properties"))
+ .unwrap_or_default()
+ .into_iter()
+ .map(|(id, props)| {
+ (
+ id,
+ props
+ .into_iter()
+ .map(SoupProperty::from)
+ .collect::<Vec<_>>(),
+ )
+ })
+ .collect();
+
// Construct enriched results
- let enriched_results =
- construct_search_result(results, thread_histories, message_senders_map, contacts_map)
- .map_err(SearchError::InternalError)?;
+ let enriched_results = construct_search_result(
+ results,
+ thread_histories,
+ message_senders_map,
+ contacts_map,
+ properties_map,
+ )
+ .map_err(SearchError::InternalError)?;
Ok(enriched_results)
}
@@ -84,6 +117,7 @@ pub fn construct_search_result(
thread_histories: HashMap<Uuid, ThreadHistoryInfo>,
message_senders: HashMap<Uuid, MessageSenderInfo>,
contacts_map: ThreadContactsMap,
+ properties_map: HashMap<String, Vec<SoupProperty>>,
) -> anyhow::Result<Vec<EmailSearchResponseItemWithMetadata>> {
// construct entity hit map of id -> vec<hits> using IndexMap to preserve insertion order
let entity_id_hit_map: IndexMap<Uuid, Vec<EmailSearchResult>> = search_results
@@ -159,6 +193,10 @@ pub fn construct_search_result(
})
.unwrap_or_default();
+ let properties = properties_map
+ .get(&entity_id.to_string())
+ .cloned()
+ .filter(|p| !p.is_empty());
Some(EmailSearchResponseItemWithMetadata {
created_at: info.created_at,
updated_at: info.updated_at,
@@ -168,6 +206,7 @@ pub fn construct_search_result(
inbox_visible: info.inbox_visible,
is_draft: info.is_draft,
is_important: info.is_important,
+ properties,
extra: EmailSearchResponseItem {
id: entity_id,
thread_id: entity_id,
diff --git a/rust/cloud-storage/search_service/src/api/search/email/test.rs b/rust/cloud-storage/search_service/src/api/search/email/test.rs
index 1e98088313..4bab2a05f4 100644
--- a/rust/cloud-storage/search_service/src/api/search/email/test.rs
+++ b/rust/cloud-storage/search_service/src/api/search/email/test.rs
@@ -27,7 +27,13 @@ fn create_email_history(thread_id: &str) -> models_email::service::message::Thre
#[test]
fn test_construct_search_result_empty_input() {
- let result = construct_search_result(vec![], HashMap::new(), HashMap::new(), HashMap::new());
+ let result = construct_search_result(
+ vec![],
+ HashMap::new(),
+ HashMap::new(),
+ HashMap::new(),
+ HashMap::new(),
+ );
assert!(result.is_ok());
assert_eq!(result.unwrap().len(), 0);
}
@@ -66,6 +72,7 @@ fn test_construct_search_result_single_thread() {
thread_histories,
HashMap::new(),
HashMap::new(),
+ HashMap::new(),
)
.unwrap();
@@ -223,6 +230,7 @@ fn test_sort_stability() {
thread_histories.clone(),
HashMap::new(),
HashMap::new(),
+ HashMap::new(),
)
.unwrap();
let result2 = construct_search_result(
@@ -230,6 +238,7 @@ fn test_sort_stability() {
thread_histories.clone(),
HashMap::new(),
HashMap::new(),
+ HashMap::new(),
)
.unwrap();
let result3 = construct_search_result(
@@ -237,6 +246,7 @@ fn test_sort_stability() {
thread_histories.clone(),
HashMap::new(),
HashMap::new(),
+ HashMap::new(),
)
.unwrap();
diff --git a/rust/cloud-storage/search_service/src/api/search/project.rs b/rust/cloud-storage/search_service/src/api/search/project.rs
index f59e23c15d..0a8ecdb752 100644
--- a/rust/cloud-storage/search_service/src/api/search/project.rs
+++ b/rust/cloud-storage/search_service/src/api/search/project.rs
@@ -1,8 +1,10 @@
use crate::api::search::simple::SearchError;
use indexmap::IndexMap;
+use models_properties::{EntityReference, EntityType};
use models_search::project::{
ProjectSearchResponseItem, ProjectSearchResponseItemWithMetadata, ProjectSearchResult,
};
+use models_soup::SoupProperty;
use sqlx::types::Uuid;
use std::collections::HashMap;
@@ -36,9 +38,35 @@ pub(in crate::api::search) async fn enrich_projects(
.await
.map_err(SearchError::InternalError)?;
+ // Fetch project properties (e.g. tags) so rows can render and re-check
+ // them, mirroring the documents enrichment.
+ let project_entity_refs: Vec<EntityReference> = project_ids
+ .iter()
+ .map(|id| EntityReference::new(id.to_string(), EntityType::Project))
+ .collect();
+ let properties_map: HashMap<String, Vec<SoupProperty>> =
+ properties_db_client::entity_properties::get::get_bulk_entity_properties_values(
+ &ctx.db,
+ &project_entity_refs,
+ )
+ .await
+ .inspect_err(|e| tracing::error!(error=?e, "failed to fetch project properties"))
+ .unwrap_or_default()
+ .into_iter()
+ .map(|(id, props)| {
+ (
+ id,
+ props
+ .into_iter()
+ .map(SoupProperty::from)
+ .collect::<Vec<_>>(),
+ )
+ })
+ .collect();
+
// Construct enriched results
- let enriched_results =
- construct_search_result(results, project_histories).map_err(SearchError::InternalError)?;
+ let enriched_results = construct_search_result(results, project_histories, properties_map)
+ .map_err(SearchError::InternalError)?;
Ok(enriched_results)
}
@@ -49,6 +77,7 @@ pub fn construct_search_result(
String,
macro_db_client::projects::get_project_history::ProjectHistoryInfo,
>,
+ mut properties_map: HashMap<String, Vec<SoupProperty>>,
) -> anyhow::Result<Vec<ProjectSearchResponseItemWithMetadata>> {
// construct entity hit map of id -> vec<hits> using IndexMap to preserve insertion order
let entity_id_hit_map: IndexMap<Uuid, Vec<ProjectSearchResult>> = search_results
@@ -81,6 +110,9 @@ pub fn construct_search_result(
};
Some(ProjectSearchResponseItemWithMetadata {
metadata: Some(metadata),
+ properties: properties_map
+ .remove(&entity_id.to_string())
+ .filter(|p| !p.is_empty()),
extra: ProjectSearchResponseItem {
id: entity_id,
owner_id: info.user_id.clone(),
diff --git a/rust/cloud-storage/search_service/src/api/search/simple/filter.rs b/rust/cloud-storage/search_service/src/api/search/simple/filter.rs
index 99f811bf36..16e52de1ed 100644
--- a/rust/cloud-storage/search_service/src/api/search/simple/filter.rs
+++ b/rust/cloud-storage/search_service/src/api/search/simple/filter.rs
@@ -4,7 +4,7 @@
use comms_db_client::model::SimpleMention;
use opensearch_client::search::unified::{
UnifiedCallRecordSearchArgs, UnifiedChannelMessageSearchArgs, UnifiedChatSearchArgs,
- UnifiedDocumentSearchArgs, UnifiedEmailSearchArgs,
+ UnifiedDocumentSearchArgs, UnifiedEmailSearchArgs, UnifiedProjectSearchArgs,
};
use crate::api::{
@@ -16,12 +16,6 @@ use crate::api::{
},
};
-#[derive(Default, Debug, Clone)]
-pub struct UnifiedProjectSearchArgs {
- pub project_ids: Vec<String>,
- pub ids_only: bool,
-}
-
/// Trait to convert item_filters into search args
pub(super) trait FilterVariantToSearchArgs {
type Output;
@@ -144,6 +138,7 @@ impl FilterVariantToSearchArgs for item_filters::ProjectFilters {
Ok(UnifiedProjectSearchArgs {
project_ids: filter_project_response.project_ids,
ids_only: filter_project_response.ids_only,
+ ..Default::default()
})
}
}
diff --git a/rust/cloud-storage/search_service/src/api/search/simple/simple_chat.rs b/rust/cloud-storage/search_service/src/api/search/simple/simple_chat.rs
index 1c8ba725cd..0998ef02ac 100644
--- a/rust/cloud-storage/search_service/src/api/search/simple/simple_chat.rs
+++ b/rust/cloud-storage/search_service/src/api/search/simple/simple_chat.rs
@@ -105,6 +105,7 @@ pub(in crate::api::search::simple) async fn search_names<'a>(
user_id: &MacroUserId<macro_user_id::lowercased::Lowercase<'a>>,
filter_chat_response: &FilterChatResponse,
term: String,
+ tag_option_ids: &[String],
limit: u32,
cursor: models_search_cursor::SearchCursorOption,
) -> Result<(Vec<SearchHit>, models_search_cursor::SearchCursorOption), SearchError> {
@@ -128,6 +129,7 @@ pub(in crate::api::search::simple) async fn search_names<'a>(
&chat_uuids,
term,
filter_chat_response.ids_only,
+ tag_option_ids,
limit,
inner_cursor,
)
diff --git a/rust/cloud-storage/search_service/src/api/search/simple/simple_project.rs b/rust/cloud-storage/search_service/src/api/search/simple/simple_project.rs
index 0a03874dee..1ad8b20a15 100644
--- a/rust/cloud-storage/search_service/src/api/search/simple/simple_project.rs
+++ b/rust/cloud-storage/search_service/src/api/search/simple/simple_project.rs
@@ -1,9 +1,6 @@
use crate::api::search::simple::SearchError;
use item_filters::ProjectFilters;
-use macro_user_id::user_id::MacroUserId;
use model::item::{ShareableItem, ShareableItemType};
-use opensearch_client::search::model::{Highlight, SearchHit};
-use sqlx::{Pool, Postgres, types::Uuid};
use crate::api::context::SearchHandlerState;
@@ -95,58 +92,3 @@ pub(in crate::api::search) async fn filter_projects(
ids_only,
})
}
-
-/// Performs the name search over project names
-#[tracing::instrument(skip(db), err)]
-pub(in crate::api::search::simple) async fn search_names<'a>(
- db: &Pool<Postgres>,
- user_id: &MacroUserId<macro_user_id::lowercased::Lowercase<'a>>,
- filter_project_response: &FilterProjectResponse,
- term: String,
- limit: u32,
- cursor: models_search_cursor::SearchCursorOption,
-) -> Result<(Vec<SearchHit>, models_search_cursor::SearchCursorOption), SearchError> {
- // If cursor is Done, no more results to fetch
- let inner_cursor = match cursor {
- models_search_cursor::SearchCursorOption::Done => {
- return Ok((vec![], models_search_cursor::SearchCursorOption::Done));
- }
- models_search_cursor::SearchCursorOption::NotDone(c) => c,
- };
-
- let project_uuids = filter_project_response
- .project_ids
- .iter()
- .map(|p| p.parse().unwrap())
- .collect::<Vec<Uuid>>();
-
- name_search::search_project_names(
- db,
- user_id,
- &project_uuids,
- term,
- filter_project_response.ids_only,
- limit,
- inner_cursor,
- )
- .await
- .map_err(SearchError::NameSearch)
- .map(|response| {
- let hits = response
- .items
- .into_iter()
- .map(|n| SearchHit {
- entity_id: n.entity_id,
- entity_type: n.entity_type,
- score: None,
- highlight: Highlight {
- name: Some(n.name),
- ..Default::default()
- },
- goto: None,
- updated_at: Some(n.updated_at),
- })
- .collect();
- (hits, response.cursor)
- })
-}
diff --git a/rust/cloud-storage/search_service/src/api/search/simple/simple_unified.rs b/rust/cloud-storage/search_service/src/api/search/simple/simple_unified.rs
index 47c85c2447..d20af8b918 100644
--- a/rust/cloud-storage/search_service/src/api/search/simple/simple_unified.rs
+++ b/rust/cloud-storage/search_service/src/api/search/simple/simple_unified.rs
@@ -1,7 +1,7 @@
use crate::api::search::simple::filter::FilterVariantToSearchArgs;
use crate::api::search::crm_company;
-use crate::api::search::simple::{simple_chat, simple_project};
+use crate::api::search::simple::simple_chat;
use crate::api::{
context::SearchHandlerState,
search::{SearchPaginationParams, simple::SearchError},
@@ -29,7 +29,6 @@ use opensearch_client::search::unified::UnifiedSearchArgs;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum SearchSource {
ChatName,
- ProjectName,
Content,
CrmCompany,
}
@@ -233,11 +232,14 @@ pub(in crate::api::search) async fn perform_unified_search(
// Property filters live at the top level of the request and only apply to
// the OpenSearch documents index, so capture them before the conversion
// (which drops them) and attach them to the document search args below.
+ // Tags additionally apply to the emails index (thread properties
+ // denormalized onto every message doc) and the chats index (nested on
+ // the parent chat doc).
let property_filter_args = to_property_filter_args(&req.filters.property_filters);
let tag_option_ids = req.filters.tag_option_ids.clone();
- // Tags are only indexed for the documents index. With a tag filter
- // active every other source is dropped, so response pages contain only
- // rows the filter was actually applied to.
+ // Tags are only indexed for the documents, emails, and chats indexes.
+ // With a tag filter active every other source is dropped, so response
+ // pages contain only rows the filter was actually applied to.
let tags_active = !tag_option_ids.is_empty();
// CRM is opt-in: it only runs when the caller resolved a team receipt
@@ -256,9 +258,11 @@ pub(in crate::api::search) async fn perform_unified_search(
let should_include_documents = search_filters.should_include_documents;
let should_include_channels = search_filters.should_include_channels && !tags_active;
- let should_include_chats = search_filters.should_include_chats && !tags_active;
- let should_include_projects = search_filters.should_include_projects && !tags_active;
- let should_include_emails = search_filters.should_include_emails && !tags_active;
+ // Chats stay live under a tag filter: both chat legs apply it (nested
+ // clause on the chats index, entity_properties EXISTS on the name leg).
+ let should_include_chats = search_filters.should_include_chats;
+ let should_include_projects = search_filters.should_include_projects;
+ let should_include_emails = search_filters.should_include_emails;
let should_include_call_records = search_filters.should_include_call_records && !tags_active;
let email_terms = search_terms.clone();
@@ -268,7 +272,7 @@ pub(in crate::api::search) async fn perform_unified_search(
mut filter_channel_response,
mut filter_chat_response,
mut filter_email_response,
- filter_project_response,
+ mut filter_project_response,
mut filter_call_record_response,
) = tokio::try_join!(
doc_filters.filter_to_search_args(
@@ -317,14 +321,19 @@ pub(in crate::api::search) async fn perform_unified_search(
// channel messages are single-doc-per-message so each token must
// appear in the same message via bool.must. Documents, chats, and
// call records are join-shape, where each term becomes a separate
- // has_child clause ANDed via bool.must.
+ // has_child clause ANDed via bool.must. Projects are flat and match
+ // terms against their name only.
filter_document_response.terms = search_terms.clone();
filter_document_response.property_filters = property_filter_args;
- filter_document_response.tag_option_ids = tag_option_ids;
+ filter_document_response.tag_option_ids = tag_option_ids.clone();
filter_channel_response.terms = search_terms.clone();
filter_chat_response.terms = search_terms.clone();
+ filter_chat_response.tag_option_ids = tag_option_ids.clone();
filter_email_response.terms = email_terms.clone();
+ filter_email_response.tag_option_ids = tag_option_ids.clone();
filter_call_record_response.terms = search_terms.clone();
+ filter_project_response.terms = search_terms.clone();
+ filter_project_response.tag_option_ids = tag_option_ids;
// Widen the email access filter to every inbox the caller can reach (their
// own plus delegated). Connected secondary inboxes share the owner's
@@ -351,10 +360,6 @@ pub(in crate::api::search) async fn perform_unified_search(
.as_ref()
.map(|c| c.chat_name_cursor.clone())
.unwrap_or_default();
- let project_cursor = cursor
- .as_ref()
- .map(|c| c.project_name_cursor.clone())
- .unwrap_or_default();
let content_cursor = cursor
.as_ref()
@@ -368,7 +373,6 @@ pub(in crate::api::search) async fn perform_unified_search(
// Clone cursors for passing to search functions (originals needed for cursor regeneration)
let chat_cursor_for_search = chat_cursor.clone();
- let project_cursor_for_search = project_cursor.clone();
let content_cursor_for_search = content_cursor.clone();
let crm_cursor_for_search = crm_cursor.clone();
@@ -401,6 +405,12 @@ pub(in crate::api::search) async fn perform_unified_search(
if should_include_call_records && !filter_call_record_response.call_ids.is_empty() {
indices.insert(models_opensearch::OpenSearchEntityType::CallRecords);
}
+ if should_include_projects
+ && !(filter_project_response.ids_only
+ && filter_project_response.project_ids.is_empty())
+ {
+ indices.insert(models_opensearch::OpenSearchEntityType::Projects);
+ }
indices
},
document_search_args: filter_document_response.clone(),
@@ -408,11 +418,12 @@ pub(in crate::api::search) async fn perform_unified_search(
channel_message_search_args: filter_channel_response,
chat_search_args: filter_chat_response.clone(),
call_record_search_args: filter_call_record_response.clone(),
+ project_search_args: filter_project_response,
};
// Call search functions in parallel for included entity types
// search_names handles Done cursors internally by returning early
- let (chat_results, project_results, content_results, crm_results) = tokio::join!(
+ let (chat_results, content_results, crm_results) = tokio::join!(
async {
if should_include_chats {
match search_on {
@@ -425,6 +436,7 @@ pub(in crate::api::search) async fn perform_unified_search(
chat_ids: filter_chat_response.chat_ids,
},
name_search_term.clone(),
+ &filter_chat_response.tag_option_ids,
page_size,
chat_cursor_for_search,
)
@@ -437,34 +449,12 @@ pub(in crate::api::search) async fn perform_unified_search(
}
},
async {
- if should_include_projects {
- match search_on {
- SearchOn::Name | SearchOn::NameContent => {
- simple_project::search_names(
- &ctx.db,
- &user_id,
- &simple_project::FilterProjectResponse {
- ids_only: filter_project_response.ids_only,
- project_ids: filter_project_response.project_ids,
- },
- name_search_term.clone(),
- page_size,
- project_cursor_for_search,
- )
- .await
- }
- SearchOn::Content => Ok((vec![], SearchCursorOption::Done)),
- }
- } else {
- Ok((vec![], SearchCursorOption::Done))
- }
- },
- async {
- // Documents and email subjects are name-searched in OpenSearch.
- // In Name mode keep only those two indices: chats and projects
+ // Documents, email subjects, and project names are name-searched
+ // in OpenSearch. In Name mode keep only those indices: chats
// still name-search in Postgres above, and channels/call records
// have no name concept. Documents match `document_name`, emails
- // their subject.
+ // their subject, projects their `name`. Projects have no content,
+ // so a content-only search drops them.
let mut args = unified_search_args;
match search_on {
SearchOn::Name => {
@@ -473,6 +463,7 @@ pub(in crate::api::search) async fn perform_unified_search(
args.search_indices.retain(|i| {
*i == models_opensearch::OpenSearchEntityType::Documents
|| *i == models_opensearch::OpenSearchEntityType::Emails
+ || *i == models_opensearch::OpenSearchEntityType::Projects
});
}
SearchOn::NameContent => {
@@ -480,6 +471,8 @@ pub(in crate::api::search) async fn perform_unified_search(
}
SearchOn::Content => {
args.document_search_args.mode = DocumentSearchMode::Content;
+ args.search_indices
+ .remove(&models_opensearch::OpenSearchEntityType::Projects);
}
}
if args.search_indices.is_empty() {
@@ -512,13 +505,11 @@ pub(in crate::api::search) async fn perform_unified_search(
// Extract results and next cursors
let (chat_hits, chat_next_cursor) = chat_results?;
- let (project_hits, project_next_cursor) = project_results?;
let (content_hits, content_next_cursor) = content_results?;
let (crm_hits, crm_next_cursor) = crm_results?;
// Track original counts before combining
let chat_name_count = chat_hits.len();
- let project_name_count = project_hits.len();
// Content hits are chunk-level (many per document), so the page budget counts
// distinct documents. Name/crm sub-searches return one row per entity, so
// their hit count already equals their entity count.
@@ -526,13 +517,8 @@ pub(in crate::api::search) async fn perform_unified_search(
let crm_count = crm_hits.len();
let final_tagged = {
- let _span = tracing::info_span!(
- "combine_and_sort_results",
- chat_name_count,
- project_name_count,
- content_count
- )
- .entered();
+ let _span = tracing::info_span!("combine_and_sort_results", chat_name_count, content_count)
+ .entered();
// Wrap results with source tags
let mut combined: Vec<TaggedSearchHit> = Vec::new();
@@ -540,10 +526,6 @@ pub(in crate::api::search) async fn perform_unified_search(
hit,
source: SearchSource::ChatName,
}));
- combined.extend(project_hits.into_iter().map(|hit| TaggedSearchHit {
- hit,
- source: SearchSource::ProjectName,
- }));
combined.extend(content_hits.into_iter().map(|hit| TaggedSearchHit {
hit,
source: SearchSource::Content,
@@ -586,10 +568,6 @@ pub(in crate::api::search) async fn perform_unified_search(
.iter()
.filter(|h| h.source == SearchSource::ChatName)
.count();
- let included_project_names = final_tagged
- .iter()
- .filter(|h| h.source == SearchSource::ProjectName)
- .count();
let included_content = distinct_entity_count(
final_tagged
.iter()
@@ -610,14 +588,6 @@ pub(in crate::api::search) async fn perform_unified_search(
&chat_cursor,
);
- let new_project_cursor = compute_next_cursor(
- &project_next_cursor,
- included_project_names,
- project_name_count,
- find_last_of_source(&final_tagged, SearchSource::ProjectName),
- &project_cursor,
- );
-
let new_content_cursor = compute_next_cursor(
&content_next_cursor,
included_content,
@@ -636,7 +606,6 @@ pub(in crate::api::search) async fn perform_unified_search(
// Build next cursor if any source has more results
let has_more = new_chat_cursor.has_more()
- || new_project_cursor.has_more()
|| new_content_cursor.has_more()
|| new_crm_cursor.has_more();
@@ -644,7 +613,9 @@ pub(in crate::api::search) async fn perform_unified_search(
let cursor = SearchCursor {
chat_name_cursor: new_chat_cursor,
content_cursor: new_content_cursor,
- project_name_cursor: new_project_cursor,
+ // Projects ride the OpenSearch content cursor; the field
+ // stays only so older cursors still decode.
+ project_name_cursor: SearchCursorOption::Done,
crm_company_cursor: new_crm_cursor,
};
cursor.encode()
diff --git a/rust/cloud-storage/search_service/src/api/search/simple/simple_unified/test.rs b/rust/cloud-storage/search_service/src/api/search/simple/simple_unified/test.rs
index 9b16488381..5189144cca 100644
--- a/rust/cloud-storage/search_service/src/api/search/simple/simple_unified/test.rs
+++ b/rust/cloud-storage/search_service/src/api/search/simple/simple_unified/test.rs
@@ -38,7 +38,7 @@ fn test_compute_next_cursor_search_done_but_truncated_in_merge_continues() {
// — returning Done here would silently lose the bumped entities.
let entity_id = Uuid::new_v4();
let timestamp = ts(1000);
- let last_hit = make_tagged_hit(entity_id, Some(timestamp), SearchSource::ProjectName);
+ let last_hit = make_tagged_hit(entity_id, Some(timestamp), SearchSource::Content);
let result = compute_next_cursor(
&SearchCursorOption::Done,
@@ -64,7 +64,7 @@ fn test_compute_next_cursor_excluded_results_with_included_generates_cursor() {
// generate cursor from the last included hit
let entity_id = Uuid::new_v4();
let timestamp = ts(1000);
- let last_hit = make_tagged_hit(entity_id, Some(timestamp), SearchSource::ProjectName);
+ let last_hit = make_tagged_hit(entity_id, Some(timestamp), SearchSource::Content);
let result = compute_next_cursor(
&SearchCursorOption::NotDone(None), // search says not done
@@ -266,12 +266,12 @@ fn test_find_last_of_source_finds_correct_source() {
let doc2_id = Uuid::new_v4();
let results = vec![
- make_tagged_hit(doc_id, Some(ts(1000)), SearchSource::ProjectName),
+ make_tagged_hit(doc_id, Some(ts(1000)), SearchSource::Content),
make_tagged_hit(chat_id, Some(ts(2000)), SearchSource::ChatName),
- make_tagged_hit(doc2_id, Some(ts(3000)), SearchSource::ProjectName),
+ make_tagged_hit(doc2_id, Some(ts(3000)), SearchSource::Content),
];
- let last_doc = find_last_of_source(&results, SearchSource::ProjectName);
+ let last_doc = find_last_of_source(&results, SearchSource::Content);
assert!(last_doc.is_some());
assert_eq!(last_doc.unwrap().hit.entity_id, doc2_id);
@@ -283,7 +283,7 @@ fn test_find_last_of_source_finds_correct_source() {
#[test]
fn test_find_last_of_source_returns_none_when_not_found() {
let results = vec![
- make_tagged_hit(Uuid::new_v4(), Some(ts(1000)), SearchSource::ProjectName),
+ make_tagged_hit(Uuid::new_v4(), Some(ts(1000)), SearchSource::ChatName),
make_tagged_hit(Uuid::new_v4(), Some(ts(2000)), SearchSource::ChatName),
];
@@ -294,7 +294,7 @@ fn test_find_last_of_source_returns_none_when_not_found() {
#[test]
fn test_find_last_of_source_empty_results() {
let results: Vec<TaggedSearchHit> = vec![];
- let result = find_last_of_source(&results, SearchSource::ProjectName);
+ let result = find_last_of_source(&results, SearchSource::Content);
assert!(result.is_none());
}
@@ -304,7 +304,7 @@ fn test_find_last_of_source_empty_results() {
fn test_cursor_from_tagged_with_timestamp() {
let entity_id = Uuid::new_v4();
let timestamp = ts(1000);
- let hit = make_tagged_hit(entity_id, Some(timestamp), SearchSource::ProjectName);
+ let hit = make_tagged_hit(entity_id, Some(timestamp), SearchSource::Content);
let cursor = cursor_from_tagged(&hit);
assert!(cursor.is_some());
@@ -330,63 +330,51 @@ fn test_cursor_from_tagged_without_timestamp() {
#[test]
fn test_cursor_logic_pagination_scenario() {
// Simulate a pagination scenario where:
- // - Search returned 10 projects
+ // - Search returned 10 content entities
// - Only 3 made it into the final page (others had newer timestamps from other sources)
- // - The cursor should point to the last included project
+ // - The cursor should point to the last included content entity
- let project_ids: Vec<Uuid> = (0..10).map(|_| Uuid::new_v4()).collect();
+ let content_ids: Vec<Uuid> = (0..10).map(|_| Uuid::new_v4()).collect();
let timestamps: Vec<chrono::DateTime<Utc>> = (0..10).map(|i| ts(1000 + i * 100)).collect();
// Create the final tagged results (simulating merged & sorted output)
let final_tagged: Vec<TaggedSearchHit> = vec![
make_tagged_hit(Uuid::new_v4(), Some(ts(5000)), SearchSource::ChatName),
- make_tagged_hit(
- project_ids[9],
- Some(timestamps[9]),
- SearchSource::ProjectName,
- ),
- make_tagged_hit(Uuid::new_v4(), Some(ts(4000)), SearchSource::Content),
- make_tagged_hit(
- project_ids[8],
- Some(timestamps[8]),
- SearchSource::ProjectName,
- ),
- make_tagged_hit(
- project_ids[7],
- Some(timestamps[7]),
- SearchSource::ProjectName,
- ),
+ make_tagged_hit(content_ids[9], Some(timestamps[9]), SearchSource::Content),
+ make_tagged_hit(Uuid::new_v4(), Some(ts(4000)), SearchSource::CrmCompany),
+ make_tagged_hit(content_ids[8], Some(timestamps[8]), SearchSource::Content),
+ make_tagged_hit(content_ids[7], Some(timestamps[7]), SearchSource::Content),
];
- let project_next_cursor = SearchCursorOption::NotDone(Some(SearchMethodCursor::UpdatedAt {
- entity_id: project_ids[6],
+ let content_next_cursor = SearchCursorOption::NotDone(Some(SearchMethodCursor::UpdatedAt {
+ entity_id: content_ids[6],
updated_at: timestamps[6],
}));
- let included_project_names = final_tagged
+ let included_content = final_tagged
.iter()
- .filter(|h| h.source == SearchSource::ProjectName)
+ .filter(|h| h.source == SearchSource::Content)
.count();
- let project_name_count = 10; // Original fetch returned 10
+ let content_count = 10; // Original fetch returned 10
- let new_project_cursor = compute_next_cursor(
- &project_next_cursor,
- included_project_names,
- project_name_count,
- find_last_of_source(&final_tagged, SearchSource::ProjectName),
+ let new_content_cursor = compute_next_cursor(
+ &content_next_cursor,
+ included_content,
+ content_count,
+ find_last_of_source(&final_tagged, SearchSource::Content),
&SearchCursorOption::NotDone(None),
);
- // Should generate cursor from last included project (project_ids[7])
- match new_project_cursor {
+ // Should generate cursor from last included content entity (content_ids[7])
+ match new_content_cursor {
SearchCursorOption::NotDone(Some(cursor)) => {
let (id, ts) = cursor.as_updated_at().expect("expected UpdatedAt cursor");
- assert_eq!(id, project_ids[7]);
+ assert_eq!(id, content_ids[7]);
assert_eq!(ts, timestamps[7]);
}
_ => panic!(
- "Expected cursor from last included project, got {:?}",
- new_project_cursor
+ "Expected cursor from last included content entity, got {:?}",
+ new_content_cursor
),
}
}
@@ -394,27 +382,27 @@ fn test_cursor_logic_pagination_scenario() {
#[test]
fn test_cursor_logic_source_exhausted_scenario() {
// Simulate when a source is exhausted (returned fewer than requested)
- // - Search returned 3 projects (less than page_size of 10)
+ // - Search returned 3 content entities (less than page_size of 10)
// - All 3 made it into the final page
// - Cursor should be Done since source is exhausted
let final_tagged: Vec<TaggedSearchHit> = vec![
- make_tagged_hit(Uuid::new_v4(), Some(ts(3000)), SearchSource::ProjectName),
- make_tagged_hit(Uuid::new_v4(), Some(ts(2000)), SearchSource::ProjectName),
- make_tagged_hit(Uuid::new_v4(), Some(ts(1000)), SearchSource::ProjectName),
+ make_tagged_hit(Uuid::new_v4(), Some(ts(3000)), SearchSource::Content),
+ make_tagged_hit(Uuid::new_v4(), Some(ts(2000)), SearchSource::Content),
+ make_tagged_hit(Uuid::new_v4(), Some(ts(1000)), SearchSource::Content),
];
- let project_next_cursor = SearchCursorOption::Done; // Search says done
+ let content_next_cursor = SearchCursorOption::Done; // Search says done
- let new_project_cursor = compute_next_cursor(
- &project_next_cursor,
+ let new_content_cursor = compute_next_cursor(
+ &content_next_cursor,
3, // all included
3, // all returned
- find_last_of_source(&final_tagged, SearchSource::ProjectName),
+ find_last_of_source(&final_tagged, SearchSource::Content),
&SearchCursorOption::NotDone(None),
);
- assert!(new_project_cursor.is_done());
+ assert!(new_content_cursor.is_done());
}
// ==================== CRM source cursor scenarios ====================
@@ -436,7 +424,7 @@ fn test_cursor_logic_crm_starved_preserves_cursor() {
// Final page: all non-CRM (CRM starved out by newer doc/chat hits).
let final_tagged: Vec<TaggedSearchHit> = vec![
- make_tagged_hit(Uuid::new_v4(), Some(ts(9000)), SearchSource::ProjectName),
+ make_tagged_hit(Uuid::new_v4(), Some(ts(9000)), SearchSource::Content),
make_tagged_hit(Uuid::new_v4(), Some(ts(8000)), SearchSource::ChatName),
];
@@ -479,7 +467,7 @@ fn test_cursor_logic_crm_included_advances_cursor() {
let crm_id = Uuid::new_v4();
let crm_ts = ts(7000);
let final_tagged: Vec<TaggedSearchHit> = vec![
- make_tagged_hit(Uuid::new_v4(), Some(ts(9000)), SearchSource::ProjectName),
+ make_tagged_hit(Uuid::new_v4(), Some(ts(9000)), SearchSource::Content),
make_tagged_hit(crm_id, Some(crm_ts), SearchSource::CrmCompany),
];
diff --git a/rust/cloud-storage/search_service/src/api/search/unified/test.rs b/rust/cloud-storage/search_service/src/api/search/unified/test.rs
index ab6655011c..7ac24a1ccf 100644
--- a/rust/cloud-storage/search_service/src/api/search/unified/test.rs
+++ b/rust/cloud-storage/search_service/src/api/search/unified/test.rs
@@ -50,6 +50,7 @@ fn test_sort_unified_search_results() {
}),
// Chat with updated_at = 3000 (newest)
UnifiedSearchResponseItem::Chat(ChatSearchResponseItemWithMetadata {
+ properties: None,
metadata: Some(ChatMetadata {
created_at: DateTime::from_timestamp(2900, 0).unwrap(),
updated_at: DateTime::from_timestamp(3000, 0).unwrap(),
@@ -76,6 +77,7 @@ fn test_sort_unified_search_results() {
inbox_visible: true,
is_draft: false,
is_important: false,
+ properties: None,
extra: EmailSearchResponseItem {
id: email_id,
name: Some("Middle Email".to_string()),
@@ -90,6 +92,7 @@ fn test_sort_unified_search_results() {
}),
// Project with updated_at = 2000 (second newest)
UnifiedSearchResponseItem::Project(ProjectSearchResponseItemWithMetadata {
+ properties: None,
metadata: Some(ProjectMetadata {
created_at: DateTime::from_timestamp(1900, 0).unwrap(),
updated_at: DateTime::from_timestamp(2000, 0).unwrap(),
diff --git a/rust/cloud-storage/sqs_client/src/search/mod.rs b/rust/cloud-storage/sqs_client/src/search/mod.rs
index 01799799b6..35bdc374af 100644
--- a/rust/cloud-storage/sqs_client/src/search/mod.rs
+++ b/rust/cloud-storage/sqs_client/src/search/mod.rs
@@ -6,6 +6,7 @@ use crate::{
chat::{ChatMessage, RemoveChatMessage},
document::{DocumentId, DocumentPropertiesUpdate, SearchExtractorMessage},
email::{EmailLinkMessage, EmailMessage, EmailThreadBatchMessage, EmailThreadMessage},
+ project::{RemoveProject, UpsertProject},
},
};
use anyhow::Context;
@@ -18,6 +19,7 @@ pub mod channel;
pub mod chat;
pub mod document;
pub mod email;
+pub mod project;
use crate::{MAX_BATCH_SIZE, PrimaryId};
@@ -86,6 +88,9 @@ pub enum SearchQueueMessage {
// Call
CallRecord(CallRecordMessage),
RemoveCallRecord(RemoveCallRecord),
+ // Project
+ UpsertProject(UpsertProject),
+ RemoveProject(RemoveProject),
// User
RemoveUserProfile(String),
@@ -122,6 +127,8 @@ impl PrimaryId for SearchQueueMessage {
message.channel_id,
message.call_id.clone().unwrap_or_default()
),
+ SearchQueueMessage::UpsertProject(message) => message.project_id.clone(),
+ SearchQueueMessage::RemoveProject(message) => message.project_id.clone(),
SearchQueueMessage::RemoveUserProfile(message) => message.clone(),
}
@@ -151,6 +158,9 @@ impl SearchQueueMessage {
// Calls
SearchQueueMessage::CallRecord(_) => Operation::ExtractText,
SearchQueueMessage::RemoveCallRecord(_) => Operation::Remove,
+ // Projects
+ SearchQueueMessage::UpsertProject(_) => Operation::UpdateMetadata,
+ SearchQueueMessage::RemoveProject(_) => Operation::Remove,
// Users
SearchQueueMessage::RemoveUserProfile(_) => Operation::Remove,
}
diff --git a/rust/cloud-storage/sqs_client/src/search/project.rs b/rust/cloud-storage/sqs_client/src/search/project.rs
new file mode 100644
index 0000000000..5c45b9c420
--- /dev/null
+++ b/rust/cloud-storage/sqs_client/src/search/project.rs
@@ -0,0 +1,17 @@
+#[derive(serde::Serialize, serde::Deserialize, Debug, PartialEq, Eq)]
+pub struct UpsertProject {
+ /// The project id
+ pub project_id: String,
+ /// Optional override for the target OpenSearch index
+ #[serde(skip_serializing_if = "Option::is_none", default)]
+ pub index_override: Option<String>,
+}
+
+#[derive(serde::Serialize, serde::Deserialize, Debug, PartialEq, Eq)]
+pub struct RemoveProject {
+ /// The project id to remove
+ pub project_id: String,
+ /// Optional override for the target OpenSearch index
+ #[serde(skip_serializing_if = "Option::is_none", default)]
+ pub index_override: Option<String>,
+}
diff --git a/rust/cloud-storage/tools/xtask/crates/xtask_local/src/local/opensearch.rs b/rust/cloud-storage/tools/xtask/crates/xtask_local/src/local/opensearch.rs
index 32828f69dd..8131784024 100644
--- a/rust/cloud-storage/tools/xtask/crates/xtask_local/src/local/opensearch.rs
+++ b/rust/cloud-storage/tools/xtask/crates/xtask_local/src/local/opensearch.rs
@@ -47,6 +47,9 @@ pub fn provision_indices(
cmd.current_dir(&dir)
.args(["run", "scripts/create_indices.ts"])
.env("ENVIRONMENT", "local")
+ // The helper scripts are dry-run by default. Local bootstrap always
+ // applies for real.
+ .env("DRY_RUN", "false")
.env("OPENSEARCH_URL", &url)
.env("OPENSEARCH_USERNAME", &username)
.env("OPENSEARCH_PASSWORD", &password);