Skip to content

High performance self-hosted photo and video management solution.

feat: view similar photos (#21108)

Arthur Normand37a79292c00a
diff --git a/i18n/en.json b/i18n/en.json
index 82748c7756..9e7b6fae6f 100644
--- a/i18n/en.json
+++ b/i18n/en.json
@@ -1557,6 +1557,7 @@
"purchase_server_description_2": "Supporter status",
"purchase_server_title": "Server",
"purchase_settings_server_activated": "The server product key is managed by the admin",
+ "query_asset_id": "Query Asset ID",
"queue_status": "Queuing {count}/{total}",
"rating": "Star rating",
"rating_clear": "Clear rating",
@@ -2077,6 +2078,7 @@
"view_next_asset": "View next asset",
"view_previous_asset": "View previous asset",
"view_qr_code": "View QR code",
+ "view_similar_photos": "View similar photos",
"view_stack": "View Stack",
"view_user": "View User",
"viewer_remove_from_stack": "Remove from Stack",
diff --git a/mobile/openapi/lib/model/smart_search_dto.dart b/mobile/openapi/lib/model/smart_search_dto.dart
index 0d16b56d74..90902b9791 100644
--- a/mobile/openapi/lib/model/smart_search_dto.dart
+++ b/mobile/openapi/lib/model/smart_search_dto.dart
@@ -31,7 +31,8 @@ class SmartSearchDto {
this.model,
this.page,
this.personIds = const [],
- required this.query,
+ this.query,
+ this.queryAssetId,
this.rating,
this.size,
this.state,
@@ -151,7 +152,21 @@ class SmartSearchDto {
List<String> personIds;
- String query;
+ ///
+ /// Please note: This property should have been non-nullable! Since the specification file
+ /// does not include a default value (using the "default:" property), however, the generated
+ /// source code must fall back to having a nullable type.
+ /// Consider adding a "default:" property in the specification file to hide this note.
+ ///
+ String? query;
+
+ ///
+ /// Please note: This property should have been non-nullable! Since the specification file
+ /// does not include a default value (using the "default:" property), however, the generated
+ /// source code must fall back to having a nullable type.
+ /// Consider adding a "default:" property in the specification file to hide this note.
+ ///
+ String? queryAssetId;
/// Minimum value: -1
/// Maximum value: 5
@@ -278,6 +293,7 @@ class SmartSearchDto {
other.page == page &&
_deepEquality.equals(other.personIds, personIds) &&
other.query == query &&
+ other.queryAssetId == queryAssetId &&
other.rating == rating &&
other.size == size &&
other.state == state &&
@@ -314,7 +330,8 @@ class SmartSearchDto {
(model == null ? 0 : model!.hashCode) +
(page == null ? 0 : page!.hashCode) +
(personIds.hashCode) +
- (query.hashCode) +
+ (query == null ? 0 : query!.hashCode) +
+ (queryAssetId == null ? 0 : queryAssetId!.hashCode) +
(rating == null ? 0 : rating!.hashCode) +
(size == null ? 0 : size!.hashCode) +
(state == null ? 0 : state!.hashCode) +
@@ -331,7 +348,7 @@ class SmartSearchDto {
(withExif == null ? 0 : withExif!.hashCode);
@override
- String toString() => 'SmartSearchDto[albumIds=$albumIds, city=$city, country=$country, createdAfter=$createdAfter, createdBefore=$createdBefore, deviceId=$deviceId, isEncoded=$isEncoded, isFavorite=$isFavorite, isMotion=$isMotion, isNotInAlbum=$isNotInAlbum, isOffline=$isOffline, language=$language, lensModel=$lensModel, libraryId=$libraryId, make=$make, model=$model, page=$page, personIds=$personIds, query=$query, rating=$rating, size=$size, state=$state, tagIds=$tagIds, takenAfter=$takenAfter, takenBefore=$takenBefore, trashedAfter=$trashedAfter, trashedBefore=$trashedBefore, type=$type, updatedAfter=$updatedAfter, updatedBefore=$updatedBefore, visibility=$visibility, withDeleted=$withDeleted, withExif=$withExif]';
+ String toString() => 'SmartSearchDto[albumIds=$albumIds, city=$city, country=$country, createdAfter=$createdAfter, createdBefore=$createdBefore, deviceId=$deviceId, isEncoded=$isEncoded, isFavorite=$isFavorite, isMotion=$isMotion, isNotInAlbum=$isNotInAlbum, isOffline=$isOffline, language=$language, lensModel=$lensModel, libraryId=$libraryId, make=$make, model=$model, page=$page, personIds=$personIds, query=$query, queryAssetId=$queryAssetId, rating=$rating, size=$size, state=$state, tagIds=$tagIds, takenAfter=$takenAfter, takenBefore=$takenBefore, trashedAfter=$trashedAfter, trashedBefore=$trashedBefore, type=$type, updatedAfter=$updatedAfter, updatedBefore=$updatedBefore, visibility=$visibility, withDeleted=$withDeleted, withExif=$withExif]';
Map<String, dynamic> toJson() {
final json = <String, dynamic>{};
@@ -417,7 +434,16 @@ class SmartSearchDto {
// json[r'page'] = null;
}
json[r'personIds'] = this.personIds;
+ if (this.query != null) {
json[r'query'] = this.query;
+ } else {
+ // json[r'query'] = null;
+ }
+ if (this.queryAssetId != null) {
+ json[r'queryAssetId'] = this.queryAssetId;
+ } else {
+ // json[r'queryAssetId'] = null;
+ }
if (this.rating != null) {
json[r'rating'] = this.rating;
} else {
@@ -522,7 +548,8 @@ class SmartSearchDto {
personIds: json[r'personIds'] is Iterable
? (json[r'personIds'] as Iterable).cast<String>().toList(growable: false)
: const [],
- query: mapValueOfType<String>(json, r'query')!,
+ query: mapValueOfType<String>(json, r'query'),
+ queryAssetId: mapValueOfType<String>(json, r'queryAssetId'),
rating: num.parse('${json[r'rating']}'),
size: num.parse('${json[r'size']}'),
state: mapValueOfType<String>(json, r'state'),
@@ -586,7 +613,6 @@ class SmartSearchDto {
/// The list of required keys that must be present in a JSON.
static const requiredKeys = <String>{
- 'query',
};
}
diff --git a/open-api/immich-openapi-specs.json b/open-api/immich-openapi-specs.json
index a03c2be1e7..81288d7f2e 100644
--- a/open-api/immich-openapi-specs.json
+++ b/open-api/immich-openapi-specs.json
@@ -14571,6 +14571,10 @@
"query": {
"type": "string"
},
+ "queryAssetId": {
+ "format": "uuid",
+ "type": "string"
+ },
"rating": {
"maximum": 5,
"minimum": -1,
@@ -14638,9 +14642,6 @@
"type": "boolean"
}
},
- "required": [
- "query"
- ],
"type": "object"
},
"SourceType": {
diff --git a/open-api/typescript-sdk/src/fetch-client.ts b/open-api/typescript-sdk/src/fetch-client.ts
index d26e2f0524..94044e97fe 100644
--- a/open-api/typescript-sdk/src/fetch-client.ts
+++ b/open-api/typescript-sdk/src/fetch-client.ts
@@ -1014,7 +1014,8 @@ export type SmartSearchDto = {
model?: string | null;
page?: number;
personIds?: string[];
- query: string;
+ query?: string;
+ queryAssetId?: string;
rating?: number;
size?: number;
state?: string | null;
diff --git a/server/src/controllers/search.controller.spec.ts b/server/src/controllers/search.controller.spec.ts
index 39d2cb8fcd..adbc8be0f3 100644
--- a/server/src/controllers/search.controller.spec.ts
+++ b/server/src/controllers/search.controller.spec.ts
@@ -128,12 +128,6 @@ describe(SearchController.name, () => {
await request(ctx.getHttpServer()).post('/search/smart');
expect(ctx.authenticate).toHaveBeenCalled();
});
-
- it('should require a query', async () => {
- const { status, body } = await request(ctx.getHttpServer()).post('/search/smart').send({});
- expect(status).toBe(400);
- expect(body).toEqual(errorDto.badRequest(['query should not be empty', 'query must be a string']));
- });
});
describe('GET /search/explore', () => {
diff --git a/server/src/dtos/search.dto.ts b/server/src/dtos/search.dto.ts
index f709ad94ab..ac7bf4feb2 100644
--- a/server/src/dtos/search.dto.ts
+++ b/server/src/dtos/search.dto.ts
@@ -199,7 +199,12 @@ export class StatisticsSearchDto extends BaseSearchDto {
export class SmartSearchDto extends BaseSearchWithResultsDto {
@IsString()
@IsNotEmpty()
- query!: string;
+ @Optional()
+ query?: string;
+
+ @ValidateUUID({ optional: true })
+ @Optional()
+ queryAssetId?: string;
@IsString()
@IsNotEmpty()
diff --git a/server/src/queries/search.repository.sql b/server/src/queries/search.repository.sql
index be2245a74e..e0aaedfdf3 100644
--- a/server/src/queries/search.repository.sql
+++ b/server/src/queries/search.repository.sql
@@ -123,6 +123,14 @@ offset
$8
commit
+-- SearchRepository.getEmbedding
+select
+ *
+from
+ "smart_search"
+where
+ "assetId" = $1
+
-- SearchRepository.searchFaces
begin
set
diff --git a/server/src/repositories/search.repository.ts b/server/src/repositories/search.repository.ts
index 36ef7a27f1..88de2fb06f 100644
--- a/server/src/repositories/search.repository.ts
+++ b/server/src/repositories/search.repository.ts
@@ -293,6 +293,13 @@ export class SearchRepository {
});
}
+ @GenerateSql({
+ params: [DummyValue.UUID],
+ })
+ async getEmbedding(assetId: string) {
+ return this.db.selectFrom('smart_search').selectAll().where('assetId', '=', assetId).executeTakeFirst();
+ }
+
@GenerateSql({
params: [
{
diff --git a/server/src/services/search.service.ts b/server/src/services/search.service.ts
index b9391fed90..51a2c94338 100644
--- a/server/src/services/search.service.ts
+++ b/server/src/services/search.service.ts
@@ -18,7 +18,7 @@ import {
SmartSearchDto,
StatisticsSearchDto,
} from 'src/dtos/search.dto';
-import { AssetOrder, AssetVisibility } from 'src/enum';
+import { AssetOrder, AssetVisibility, Permission } from 'src/enum';
import { BaseService } from 'src/services/base.service';
import { requireElevatedPermission } from 'src/utils/access';
import { getMyPartnerIds } from 'src/utils/asset.util';
@@ -113,14 +113,27 @@ export class SearchService extends BaseService {
}
const userIds = this.getUserIdsToSearch(auth);
- const key = machineLearning.clip.modelName + dto.query + dto.language;
- let embedding = this.embeddingCache.get(key);
- if (!embedding) {
- embedding = await this.machineLearningRepository.encodeText(machineLearning.urls, dto.query, {
- modelName: machineLearning.clip.modelName,
- language: dto.language,
- });
- this.embeddingCache.set(key, embedding);
+ let embedding;
+ if (dto.query) {
+ const key = machineLearning.clip.modelName + dto.query + dto.language;
+ embedding = this.embeddingCache.get(key);
+ if (!embedding) {
+ embedding = await this.machineLearningRepository.encodeText(machineLearning.urls, dto.query, {
+ modelName: machineLearning.clip.modelName,
+ language: dto.language,
+ });
+ this.embeddingCache.set(key, embedding);
+ }
+ } else if (dto.queryAssetId) {
+ await this.requireAccess({ auth, permission: Permission.AssetRead, ids: [dto.queryAssetId] });
+ const getEmbeddingResponse = await this.searchRepository.getEmbedding(dto.queryAssetId);
+ const assetEmbedding = getEmbeddingResponse?.embedding;
+ if (!assetEmbedding) {
+ throw new BadRequestException(`Asset ${dto.queryAssetId} has no embedding`);
+ }
+ embedding = assetEmbedding;
+ } else {
+ throw new BadRequestException('Either `query` or `queryAssetId` must be set');
}
const page = dto.page ?? 1;
const size = dto.size || 100;
diff --git a/web/src/lib/components/asset-viewer/asset-viewer-nav-bar.svelte b/web/src/lib/components/asset-viewer/asset-viewer-nav-bar.svelte
index 66061ebb01..dd197f3a90 100644
--- a/web/src/lib/components/asset-viewer/asset-viewer-nav-bar.svelte
+++ b/web/src/lib/components/asset-viewer/asset-viewer-nav-bar.svelte
@@ -22,6 +22,7 @@
import ButtonContextMenu from '$lib/components/shared-components/context-menu/button-context-menu.svelte';
import MenuOption from '$lib/components/shared-components/context-menu/menu-option.svelte';
import { AppRoute } from '$lib/constants';
+ import { featureFlags } from '$lib/stores/server-config.store';
import { user } from '$lib/stores/user.store';
import { photoZoomState } from '$lib/stores/zoom-image.store';
import { getAssetJobName, getSharedLink } from '$lib/utils';
@@ -41,6 +42,7 @@
import {
mdiAlertOutline,
mdiCogRefreshOutline,
+ mdiCompare,
mdiContentCopy,
mdiDatabaseRefreshOutline,
mdiDotsVertical,
@@ -98,6 +100,7 @@
let isOwner = $derived($user && asset.ownerId === $user?.id);
let showDownloadButton = $derived(sharedLink ? sharedLink.allowDownload : !asset.isOffline);
let isLocked = $derived(asset.visibility === AssetVisibility.Locked);
+ let smartSearchEnabled = $derived($featureFlags.loaded && $featureFlags.smartSearch);
// $: showEditorButton =
// isOwner &&
@@ -225,6 +228,13 @@
text={$t('view_in_timeline')}
/>
{/if}
+ {#if !asset.isArchived && !asset.isTrashed && smartSearchEnabled}
+ <MenuOption
+ icon={mdiCompare}
+ onClick={() => goto(`${AppRoute.SEARCH}?query={"queryAssetId":"${stack?.primaryAssetId ?? asset.id}"}`)}
+ text={$t('view_similar_photos')}
+ />
+ {/if}
{/if}
{#if !asset.isTrashed}
diff --git a/web/src/lib/modals/SearchFilterModal.svelte b/web/src/lib/modals/SearchFilterModal.svelte
index 0647d0bb71..b9841c311e 100644
--- a/web/src/lib/modals/SearchFilterModal.svelte
+++ b/web/src/lib/modals/SearchFilterModal.svelte
@@ -64,8 +64,16 @@
return validQueryTypes.has(storedQueryType) ? storedQueryType : QueryType.SMART;
}
+ let query = '';
+ if ('query' in searchQuery && searchQuery.query) {
+ query = searchQuery.query;
+ }
+ if ('originalFileName' in searchQuery && searchQuery.originalFileName) {
+ query = searchQuery.originalFileName;
+ }
+
let filter: SearchFilter = $state({
- query: 'query' in searchQuery ? searchQuery.query : searchQuery.originalFileName || '',
+ query,
queryType: defaultQueryType(),
personIds: new SvelteSet('personIds' in searchQuery ? searchQuery.personIds : []),
tagIds:
diff --git a/web/src/routes/(user)/search/[[photos=photos]]/[[assetId=id]]/+page.svelte b/web/src/routes/(user)/search/[[photos=photos]]/[[assetId=id]]/+page.svelte
index 56e58c3633..512d8c548a 100644
--- a/web/src/routes/(user)/search/[[photos=photos]]/[[assetId=id]]/+page.svelte
+++ b/web/src/routes/(user)/search/[[photos=photos]]/[[assetId=id]]/+page.svelte
@@ -68,7 +68,7 @@
const assetInteraction = new AssetInteraction();
- type SearchTerms = MetadataSearchDto & Pick<SmartSearchDto, 'query'>;
+ type SearchTerms = MetadataSearchDto & Pick<SmartSearchDto, 'query' | 'queryAssetId'>;
let searchQuery = $derived(page.url.searchParams.get(QueryParameter.QUERY));
let smartSearchEnabled = $derived($featureFlags.loaded && $featureFlags.smartSearch);
let terms = $derived(searchQuery ? JSON.parse(searchQuery) : {});
@@ -164,7 +164,7 @@
try {
const { albums, assets } =
- 'query' in searchDto && smartSearchEnabled
+ ('query' in searchDto || 'queryAssetId' in searchDto) && smartSearchEnabled
? await searchSmart({ smartSearchDto: searchDto })
: await searchAssets({ metadataSearchDto: searchDto });
@@ -210,6 +210,7 @@
tagIds: $t('tags'),
originalFileName: $t('file_name'),
description: $t('description'),
+ queryAssetId: $t('query_asset_id'),
};
return keyMap[key] || key;
}