diff --git a/i18n/en.json b/i18n/en.json index 82748c7756..9e7b6fae6f 100644 --- a/i18n/en.json +++ b/i18n/en.json @@ -1557,6 +1557,7 @@ "purchase_server_description_2": "Supporter status", "purchase_server_title": "Server", "purchase_settings_server_activated": "The server product key is managed by the admin", + "query_asset_id": "Query Asset ID", "queue_status": "Queuing {count}/{total}", "rating": "Star rating", "rating_clear": "Clear rating", @@ -2077,6 +2078,7 @@ "view_next_asset": "View next asset", "view_previous_asset": "View previous asset", "view_qr_code": "View QR code", + "view_similar_photos": "View similar photos", "view_stack": "View Stack", "view_user": "View User", "viewer_remove_from_stack": "Remove from Stack", diff --git a/mobile/openapi/lib/model/smart_search_dto.dart b/mobile/openapi/lib/model/smart_search_dto.dart index 0d16b56d74..90902b9791 100644 --- a/mobile/openapi/lib/model/smart_search_dto.dart +++ b/mobile/openapi/lib/model/smart_search_dto.dart @@ -31,7 +31,8 @@ class SmartSearchDto { this.model, this.page, this.personIds = const [], - required this.query, + this.query, + this.queryAssetId, this.rating, this.size, this.state, @@ -151,7 +152,21 @@ class SmartSearchDto { List personIds; - String query; + /// + /// Please note: This property should have been non-nullable! Since the specification file + /// does not include a default value (using the "default:" property), however, the generated + /// source code must fall back to having a nullable type. + /// Consider adding a "default:" property in the specification file to hide this note. + /// + String? query; + + /// + /// Please note: This property should have been non-nullable! Since the specification file + /// does not include a default value (using the "default:" property), however, the generated + /// source code must fall back to having a nullable type. + /// Consider adding a "default:" property in the specification file to hide this note. + /// + String? queryAssetId; /// Minimum value: -1 /// Maximum value: 5 @@ -278,6 +293,7 @@ class SmartSearchDto { other.page == page && _deepEquality.equals(other.personIds, personIds) && other.query == query && + other.queryAssetId == queryAssetId && other.rating == rating && other.size == size && other.state == state && @@ -314,7 +330,8 @@ class SmartSearchDto { (model == null ? 0 : model!.hashCode) + (page == null ? 0 : page!.hashCode) + (personIds.hashCode) + - (query.hashCode) + + (query == null ? 0 : query!.hashCode) + + (queryAssetId == null ? 0 : queryAssetId!.hashCode) + (rating == null ? 0 : rating!.hashCode) + (size == null ? 0 : size!.hashCode) + (state == null ? 0 : state!.hashCode) + @@ -331,7 +348,7 @@ class SmartSearchDto { (withExif == null ? 0 : withExif!.hashCode); @override - String toString() => 'SmartSearchDto[albumIds=$albumIds, city=$city, country=$country, createdAfter=$createdAfter, createdBefore=$createdBefore, deviceId=$deviceId, isEncoded=$isEncoded, isFavorite=$isFavorite, isMotion=$isMotion, isNotInAlbum=$isNotInAlbum, isOffline=$isOffline, language=$language, lensModel=$lensModel, libraryId=$libraryId, make=$make, model=$model, page=$page, personIds=$personIds, query=$query, rating=$rating, size=$size, state=$state, tagIds=$tagIds, takenAfter=$takenAfter, takenBefore=$takenBefore, trashedAfter=$trashedAfter, trashedBefore=$trashedBefore, type=$type, updatedAfter=$updatedAfter, updatedBefore=$updatedBefore, visibility=$visibility, withDeleted=$withDeleted, withExif=$withExif]'; + String toString() => 'SmartSearchDto[albumIds=$albumIds, city=$city, country=$country, createdAfter=$createdAfter, createdBefore=$createdBefore, deviceId=$deviceId, isEncoded=$isEncoded, isFavorite=$isFavorite, isMotion=$isMotion, isNotInAlbum=$isNotInAlbum, isOffline=$isOffline, language=$language, lensModel=$lensModel, libraryId=$libraryId, make=$make, model=$model, page=$page, personIds=$personIds, query=$query, queryAssetId=$queryAssetId, rating=$rating, size=$size, state=$state, tagIds=$tagIds, takenAfter=$takenAfter, takenBefore=$takenBefore, trashedAfter=$trashedAfter, trashedBefore=$trashedBefore, type=$type, updatedAfter=$updatedAfter, updatedBefore=$updatedBefore, visibility=$visibility, withDeleted=$withDeleted, withExif=$withExif]'; Map toJson() { final json = {}; @@ -417,7 +434,16 @@ class SmartSearchDto { // json[r'page'] = null; } json[r'personIds'] = this.personIds; + if (this.query != null) { json[r'query'] = this.query; + } else { + // json[r'query'] = null; + } + if (this.queryAssetId != null) { + json[r'queryAssetId'] = this.queryAssetId; + } else { + // json[r'queryAssetId'] = null; + } if (this.rating != null) { json[r'rating'] = this.rating; } else { @@ -522,7 +548,8 @@ class SmartSearchDto { personIds: json[r'personIds'] is Iterable ? (json[r'personIds'] as Iterable).cast().toList(growable: false) : const [], - query: mapValueOfType(json, r'query')!, + query: mapValueOfType(json, r'query'), + queryAssetId: mapValueOfType(json, r'queryAssetId'), rating: num.parse('${json[r'rating']}'), size: num.parse('${json[r'size']}'), state: mapValueOfType(json, r'state'), @@ -586,7 +613,6 @@ class SmartSearchDto { /// The list of required keys that must be present in a JSON. static const requiredKeys = { - 'query', }; } diff --git a/open-api/immich-openapi-specs.json b/open-api/immich-openapi-specs.json index a03c2be1e7..81288d7f2e 100644 --- a/open-api/immich-openapi-specs.json +++ b/open-api/immich-openapi-specs.json @@ -14571,6 +14571,10 @@ "query": { "type": "string" }, + "queryAssetId": { + "format": "uuid", + "type": "string" + }, "rating": { "maximum": 5, "minimum": -1, @@ -14638,9 +14642,6 @@ "type": "boolean" } }, - "required": [ - "query" - ], "type": "object" }, "SourceType": { diff --git a/open-api/typescript-sdk/src/fetch-client.ts b/open-api/typescript-sdk/src/fetch-client.ts index d26e2f0524..94044e97fe 100644 --- a/open-api/typescript-sdk/src/fetch-client.ts +++ b/open-api/typescript-sdk/src/fetch-client.ts @@ -1014,7 +1014,8 @@ export type SmartSearchDto = { model?: string | null; page?: number; personIds?: string[]; - query: string; + query?: string; + queryAssetId?: string; rating?: number; size?: number; state?: string | null; diff --git a/server/src/controllers/search.controller.spec.ts b/server/src/controllers/search.controller.spec.ts index 39d2cb8fcd..adbc8be0f3 100644 --- a/server/src/controllers/search.controller.spec.ts +++ b/server/src/controllers/search.controller.spec.ts @@ -128,12 +128,6 @@ describe(SearchController.name, () => { await request(ctx.getHttpServer()).post('/search/smart'); expect(ctx.authenticate).toHaveBeenCalled(); }); - - it('should require a query', async () => { - const { status, body } = await request(ctx.getHttpServer()).post('/search/smart').send({}); - expect(status).toBe(400); - expect(body).toEqual(errorDto.badRequest(['query should not be empty', 'query must be a string'])); - }); }); describe('GET /search/explore', () => { diff --git a/server/src/dtos/search.dto.ts b/server/src/dtos/search.dto.ts index f709ad94ab..ac7bf4feb2 100644 --- a/server/src/dtos/search.dto.ts +++ b/server/src/dtos/search.dto.ts @@ -199,7 +199,12 @@ export class StatisticsSearchDto extends BaseSearchDto { export class SmartSearchDto extends BaseSearchWithResultsDto { @IsString() @IsNotEmpty() - query!: string; + @Optional() + query?: string; + + @ValidateUUID({ optional: true }) + @Optional() + queryAssetId?: string; @IsString() @IsNotEmpty() diff --git a/server/src/queries/search.repository.sql b/server/src/queries/search.repository.sql index be2245a74e..e0aaedfdf3 100644 --- a/server/src/queries/search.repository.sql +++ b/server/src/queries/search.repository.sql @@ -123,6 +123,14 @@ offset $8 commit +-- SearchRepository.getEmbedding +select + * +from + "smart_search" +where + "assetId" = $1 + -- SearchRepository.searchFaces begin set diff --git a/server/src/repositories/search.repository.ts b/server/src/repositories/search.repository.ts index 36ef7a27f1..88de2fb06f 100644 --- a/server/src/repositories/search.repository.ts +++ b/server/src/repositories/search.repository.ts @@ -293,6 +293,13 @@ export class SearchRepository { }); } + @GenerateSql({ + params: [DummyValue.UUID], + }) + async getEmbedding(assetId: string) { + return this.db.selectFrom('smart_search').selectAll().where('assetId', '=', assetId).executeTakeFirst(); + } + @GenerateSql({ params: [ { diff --git a/server/src/services/search.service.ts b/server/src/services/search.service.ts index b9391fed90..51a2c94338 100644 --- a/server/src/services/search.service.ts +++ b/server/src/services/search.service.ts @@ -18,7 +18,7 @@ import { SmartSearchDto, StatisticsSearchDto, } from 'src/dtos/search.dto'; -import { AssetOrder, AssetVisibility } from 'src/enum'; +import { AssetOrder, AssetVisibility, Permission } from 'src/enum'; import { BaseService } from 'src/services/base.service'; import { requireElevatedPermission } from 'src/utils/access'; import { getMyPartnerIds } from 'src/utils/asset.util'; @@ -113,14 +113,27 @@ export class SearchService extends BaseService { } const userIds = this.getUserIdsToSearch(auth); - const key = machineLearning.clip.modelName + dto.query + dto.language; - let embedding = this.embeddingCache.get(key); - if (!embedding) { - embedding = await this.machineLearningRepository.encodeText(machineLearning.urls, dto.query, { - modelName: machineLearning.clip.modelName, - language: dto.language, - }); - this.embeddingCache.set(key, embedding); + let embedding; + if (dto.query) { + const key = machineLearning.clip.modelName + dto.query + dto.language; + embedding = this.embeddingCache.get(key); + if (!embedding) { + embedding = await this.machineLearningRepository.encodeText(machineLearning.urls, dto.query, { + modelName: machineLearning.clip.modelName, + language: dto.language, + }); + this.embeddingCache.set(key, embedding); + } + } else if (dto.queryAssetId) { + await this.requireAccess({ auth, permission: Permission.AssetRead, ids: [dto.queryAssetId] }); + const getEmbeddingResponse = await this.searchRepository.getEmbedding(dto.queryAssetId); + const assetEmbedding = getEmbeddingResponse?.embedding; + if (!assetEmbedding) { + throw new BadRequestException(`Asset ${dto.queryAssetId} has no embedding`); + } + embedding = assetEmbedding; + } else { + throw new BadRequestException('Either `query` or `queryAssetId` must be set'); } const page = dto.page ?? 1; const size = dto.size || 100; diff --git a/web/src/lib/components/asset-viewer/asset-viewer-nav-bar.svelte b/web/src/lib/components/asset-viewer/asset-viewer-nav-bar.svelte index 66061ebb01..dd197f3a90 100644 --- a/web/src/lib/components/asset-viewer/asset-viewer-nav-bar.svelte +++ b/web/src/lib/components/asset-viewer/asset-viewer-nav-bar.svelte @@ -22,6 +22,7 @@ import ButtonContextMenu from '$lib/components/shared-components/context-menu/button-context-menu.svelte'; import MenuOption from '$lib/components/shared-components/context-menu/menu-option.svelte'; import { AppRoute } from '$lib/constants'; + import { featureFlags } from '$lib/stores/server-config.store'; import { user } from '$lib/stores/user.store'; import { photoZoomState } from '$lib/stores/zoom-image.store'; import { getAssetJobName, getSharedLink } from '$lib/utils'; @@ -41,6 +42,7 @@ import { mdiAlertOutline, mdiCogRefreshOutline, + mdiCompare, mdiContentCopy, mdiDatabaseRefreshOutline, mdiDotsVertical, @@ -98,6 +100,7 @@ let isOwner = $derived($user && asset.ownerId === $user?.id); let showDownloadButton = $derived(sharedLink ? sharedLink.allowDownload : !asset.isOffline); let isLocked = $derived(asset.visibility === AssetVisibility.Locked); + let smartSearchEnabled = $derived($featureFlags.loaded && $featureFlags.smartSearch); // $: showEditorButton = // isOwner && @@ -225,6 +228,13 @@ text={$t('view_in_timeline')} /> {/if} + {#if !asset.isArchived && !asset.isTrashed && smartSearchEnabled} + goto(`${AppRoute.SEARCH}?query={"queryAssetId":"${stack?.primaryAssetId ?? asset.id}"}`)} + text={$t('view_similar_photos')} + /> + {/if} {/if} {#if !asset.isTrashed} diff --git a/web/src/lib/modals/SearchFilterModal.svelte b/web/src/lib/modals/SearchFilterModal.svelte index 0647d0bb71..b9841c311e 100644 --- a/web/src/lib/modals/SearchFilterModal.svelte +++ b/web/src/lib/modals/SearchFilterModal.svelte @@ -64,8 +64,16 @@ return validQueryTypes.has(storedQueryType) ? storedQueryType : QueryType.SMART; } + let query = ''; + if ('query' in searchQuery && searchQuery.query) { + query = searchQuery.query; + } + if ('originalFileName' in searchQuery && searchQuery.originalFileName) { + query = searchQuery.originalFileName; + } + let filter: SearchFilter = $state({ - query: 'query' in searchQuery ? searchQuery.query : searchQuery.originalFileName || '', + query, queryType: defaultQueryType(), personIds: new SvelteSet('personIds' in searchQuery ? searchQuery.personIds : []), tagIds: diff --git a/web/src/routes/(user)/search/[[photos=photos]]/[[assetId=id]]/+page.svelte b/web/src/routes/(user)/search/[[photos=photos]]/[[assetId=id]]/+page.svelte index 56e58c3633..512d8c548a 100644 --- a/web/src/routes/(user)/search/[[photos=photos]]/[[assetId=id]]/+page.svelte +++ b/web/src/routes/(user)/search/[[photos=photos]]/[[assetId=id]]/+page.svelte @@ -68,7 +68,7 @@ const assetInteraction = new AssetInteraction(); - type SearchTerms = MetadataSearchDto & Pick; + type SearchTerms = MetadataSearchDto & Pick; let searchQuery = $derived(page.url.searchParams.get(QueryParameter.QUERY)); let smartSearchEnabled = $derived($featureFlags.loaded && $featureFlags.smartSearch); let terms = $derived(searchQuery ? JSON.parse(searchQuery) : {}); @@ -164,7 +164,7 @@ try { const { albums, assets } = - 'query' in searchDto && smartSearchEnabled + ('query' in searchDto || 'queryAssetId' in searchDto) && smartSearchEnabled ? await searchSmart({ smartSearchDto: searchDto }) : await searchAssets({ metadataSearchDto: searchDto }); @@ -210,6 +210,7 @@ tagIds: $t('tags'), originalFileName: $t('file_name'), description: $t('description'), + queryAssetId: $t('query_asset_id'), }; return keyMap[key] || key; }