feat: view similar photos (#21108)

* Enable filteing by example

* Drop `@GenerateSql` for `getEmbedding`?

* Improve error message

* PR Feedback

* Sort en.json

* Add SQL

* Fix lint

* Drop test that is no longer valid

* Fix i18n file sorting

* Fix TS error

* Add a `requireAccess` before pulling the embedding

* Fix decorators

* Run `make open-api`

---------

Co-authored-by: Alex <alex.tran1502@gmail.com>

i18n/en.json

@@ -1557,6 +1557,7 @@
   "purchase_server_description_2": "Supporter status",
   "purchase_server_title": "Server",
   "purchase_settings_server_activated": "The server product key is managed by the admin",
+  "query_asset_id": "Query Asset ID",
   "queue_status": "Queuing {count}/{total}",
   "rating": "Star rating",
   "rating_clear": "Clear rating",
@@ -2077,6 +2078,7 @@
   "view_next_asset": "View next asset",
   "view_previous_asset": "View previous asset",
   "view_qr_code": "View QR code",
+  "view_similar_photos": "View similar photos",
   "view_stack": "View Stack",
   "view_user": "View User",
   "viewer_remove_from_stack": "Remove from Stack",

mobile/openapi/lib/model/smart_search_dto.dart

@@ -31,7 +31,8 @@ class SmartSearchDto {
     this.model,
     this.page,
     this.personIds = const [],
-    required this.query,
+    this.query,
+    this.queryAssetId,
     this.rating,
     this.size,
     this.state,
@@ -151,7 +152,21 @@ class SmartSearchDto {
 
   List<String> personIds;
 
-  String query;
+  ///
+  /// Please note: This property should have been non-nullable! Since the specification file
+  /// does not include a default value (using the "default:" property), however, the generated
+  /// source code must fall back to having a nullable type.
+  /// Consider adding a "default:" property in the specification file to hide this note.
+  ///
+  String? query;
+
+  ///
+  /// Please note: This property should have been non-nullable! Since the specification file
+  /// does not include a default value (using the "default:" property), however, the generated
+  /// source code must fall back to having a nullable type.
+  /// Consider adding a "default:" property in the specification file to hide this note.
+  ///
+  String? queryAssetId;
 
   /// Minimum value: -1
   /// Maximum value: 5
@@ -278,6 +293,7 @@ class SmartSearchDto {
     other.page == page &&
     _deepEquality.equals(other.personIds, personIds) &&
     other.query == query &&
+    other.queryAssetId == queryAssetId &&
     other.rating == rating &&
     other.size == size &&
     other.state == state &&
@@ -314,7 +330,8 @@ class SmartSearchDto {
     (model == null ? 0 : model!.hashCode) +
     (page == null ? 0 : page!.hashCode) +
     (personIds.hashCode) +
-    (query.hashCode) +
+    (query == null ? 0 : query!.hashCode) +
+    (queryAssetId == null ? 0 : queryAssetId!.hashCode) +
     (rating == null ? 0 : rating!.hashCode) +
     (size == null ? 0 : size!.hashCode) +
     (state == null ? 0 : state!.hashCode) +
@@ -331,7 +348,7 @@ class SmartSearchDto {
     (withExif == null ? 0 : withExif!.hashCode);
 
   @override
-  String toString() => 'SmartSearchDto[albumIds=$albumIds, city=$city, country=$country, createdAfter=$createdAfter, createdBefore=$createdBefore, deviceId=$deviceId, isEncoded=$isEncoded, isFavorite=$isFavorite, isMotion=$isMotion, isNotInAlbum=$isNotInAlbum, isOffline=$isOffline, language=$language, lensModel=$lensModel, libraryId=$libraryId, make=$make, model=$model, page=$page, personIds=$personIds, query=$query, rating=$rating, size=$size, state=$state, tagIds=$tagIds, takenAfter=$takenAfter, takenBefore=$takenBefore, trashedAfter=$trashedAfter, trashedBefore=$trashedBefore, type=$type, updatedAfter=$updatedAfter, updatedBefore=$updatedBefore, visibility=$visibility, withDeleted=$withDeleted, withExif=$withExif]';
+  String toString() => 'SmartSearchDto[albumIds=$albumIds, city=$city, country=$country, createdAfter=$createdAfter, createdBefore=$createdBefore, deviceId=$deviceId, isEncoded=$isEncoded, isFavorite=$isFavorite, isMotion=$isMotion, isNotInAlbum=$isNotInAlbum, isOffline=$isOffline, language=$language, lensModel=$lensModel, libraryId=$libraryId, make=$make, model=$model, page=$page, personIds=$personIds, query=$query, queryAssetId=$queryAssetId, rating=$rating, size=$size, state=$state, tagIds=$tagIds, takenAfter=$takenAfter, takenBefore=$takenBefore, trashedAfter=$trashedAfter, trashedBefore=$trashedBefore, type=$type, updatedAfter=$updatedAfter, updatedBefore=$updatedBefore, visibility=$visibility, withDeleted=$withDeleted, withExif=$withExif]';
 
   Map<String, dynamic> toJson() {
     final json = <String, dynamic>{};
@@ -417,7 +434,16 @@ class SmartSearchDto {
     //  json[r'page'] = null;
     }
       json[r'personIds'] = this.personIds;
+    if (this.query != null) {
       json[r'query'] = this.query;
+    } else {
+    //  json[r'query'] = null;
+    }
+    if (this.queryAssetId != null) {
+      json[r'queryAssetId'] = this.queryAssetId;
+    } else {
+    //  json[r'queryAssetId'] = null;
+    }
     if (this.rating != null) {
       json[r'rating'] = this.rating;
     } else {
@@ -522,7 +548,8 @@ class SmartSearchDto {
         personIds: json[r'personIds'] is Iterable
             ? (json[r'personIds'] as Iterable).cast<String>().toList(growable: false)
             : const [],
-        query: mapValueOfType<String>(json, r'query')!,
+        query: mapValueOfType<String>(json, r'query'),
+        queryAssetId: mapValueOfType<String>(json, r'queryAssetId'),
         rating: num.parse('${json[r'rating']}'),
         size: num.parse('${json[r'size']}'),
         state: mapValueOfType<String>(json, r'state'),
@@ -586,7 +613,6 @@ class SmartSearchDto {
 
   /// The list of required keys that must be present in a JSON.
   static const requiredKeys = <String>{
-    'query',
   };
 }
 

open-api/immich-openapi-specs.json

@@ -14571,6 +14571,10 @@
           "query": {
             "type": "string"
           },
+          "queryAssetId": {
+            "format": "uuid",
+            "type": "string"
+          },
           "rating": {
             "maximum": 5,
             "minimum": -1,
@@ -14638,9 +14642,6 @@
             "type": "boolean"
           }
         },
-        "required": [
-          "query"
-        ],
         "type": "object"
       },
       "SourceType": {

open-api/typescript-sdk/src/fetch-client.ts

@@ -1014,7 +1014,8 @@ export type SmartSearchDto = {
     model?: string | null;
     page?: number;
     personIds?: string[];
-    query: string;
+    query?: string;
+    queryAssetId?: string;
     rating?: number;
     size?: number;
     state?: string | null;

server/src/controllers/search.controller.spec.ts

@@ -128,12 +128,6 @@ describe(SearchController.name, () => {
         await request(ctx.getHttpServer()).post('/search/smart');
         expect(ctx.authenticate).toHaveBeenCalled();
       });
-
-      it('should require a query', async () => {
-        const { status, body } = await request(ctx.getHttpServer()).post('/search/smart').send({});
-        expect(status).toBe(400);
-        expect(body).toEqual(errorDto.badRequest(['query should not be empty', 'query must be a string']));
-      });
     });
 
     describe('GET /search/explore', () => {

server/src/dtos/search.dto.ts

@@ -199,7 +199,12 @@ export class StatisticsSearchDto extends BaseSearchDto {
 export class SmartSearchDto extends BaseSearchWithResultsDto {
   @IsString()
   @IsNotEmpty()
-  query!: string;
+  @Optional()
+  query?: string;
+
+  @ValidateUUID({ optional: true })
+  @Optional()
+  queryAssetId?: string;
 
   @IsString()
   @IsNotEmpty()

server/src/queries/search.repository.sql

@@ -123,6 +123,14 @@ offset
   $8
 commit
 
+-- SearchRepository.getEmbedding
+select
+  *
+from
+  "smart_search"
+where
+  "assetId" = $1
+
 -- SearchRepository.searchFaces
 begin
 set

server/src/repositories/search.repository.ts

@@ -293,6 +293,13 @@ export class SearchRepository {
     });
   }
 
+  @GenerateSql({
+    params: [DummyValue.UUID],
+  })
+  async getEmbedding(assetId: string) {
+    return this.db.selectFrom('smart_search').selectAll().where('assetId', '=', assetId).executeTakeFirst();
+  }
+
   @GenerateSql({
     params: [
       {

server/src/services/search.service.ts

@@ -18,7 +18,7 @@ import {
   SmartSearchDto,
   StatisticsSearchDto,
 } from 'src/dtos/search.dto';
-import { AssetOrder, AssetVisibility } from 'src/enum';
+import { AssetOrder, AssetVisibility, Permission } from 'src/enum';
 import { BaseService } from 'src/services/base.service';
 import { requireElevatedPermission } from 'src/utils/access';
 import { getMyPartnerIds } from 'src/utils/asset.util';
@@ -113,14 +113,27 @@ export class SearchService extends BaseService {
     }
 
     const userIds = this.getUserIdsToSearch(auth);
-    const key = machineLearning.clip.modelName + dto.query + dto.language;
+    let embedding;
-    let embedding = this.embeddingCache.get(key);
+    if (dto.query) {
-    if (!embedding) {
+      const key = machineLearning.clip.modelName + dto.query + dto.language;
-      embedding = await this.machineLearningRepository.encodeText(machineLearning.urls, dto.query, {
+      embedding = this.embeddingCache.get(key);
-        modelName: machineLearning.clip.modelName,
+      if (!embedding) {
-        language: dto.language,
+        embedding = await this.machineLearningRepository.encodeText(machineLearning.urls, dto.query, {
-      });
+          modelName: machineLearning.clip.modelName,
-      this.embeddingCache.set(key, embedding);
+          language: dto.language,
+        });
+        this.embeddingCache.set(key, embedding);
+      }
+    } else if (dto.queryAssetId) {
+      await this.requireAccess({ auth, permission: Permission.AssetRead, ids: [dto.queryAssetId] });
+      const getEmbeddingResponse = await this.searchRepository.getEmbedding(dto.queryAssetId);
+      const assetEmbedding = getEmbeddingResponse?.embedding;
+      if (!assetEmbedding) {
+        throw new BadRequestException(`Asset ${dto.queryAssetId} has no embedding`);
+      }
+      embedding = assetEmbedding;
+    } else {
+      throw new BadRequestException('Either `query` or `queryAssetId` must be set');
     }
     const page = dto.page ?? 1;
     const size = dto.size || 100;

web/src/lib/components/asset-viewer/asset-viewer-nav-bar.svelte

@@ -22,6 +22,7 @@
   import ButtonContextMenu from '$lib/components/shared-components/context-menu/button-context-menu.svelte';
   import MenuOption from '$lib/components/shared-components/context-menu/menu-option.svelte';
   import { AppRoute } from '$lib/constants';
+  import { featureFlags } from '$lib/stores/server-config.store';
   import { user } from '$lib/stores/user.store';
   import { photoZoomState } from '$lib/stores/zoom-image.store';
   import { getAssetJobName, getSharedLink } from '$lib/utils';
@@ -41,6 +42,7 @@
   import {
     mdiAlertOutline,
     mdiCogRefreshOutline,
+    mdiCompare,
     mdiContentCopy,
     mdiDatabaseRefreshOutline,
     mdiDotsVertical,
@@ -98,6 +100,7 @@
   let isOwner = $derived($user && asset.ownerId === $user?.id);
   let showDownloadButton = $derived(sharedLink ? sharedLink.allowDownload : !asset.isOffline);
   let isLocked = $derived(asset.visibility === AssetVisibility.Locked);
+  let smartSearchEnabled = $derived($featureFlags.loaded && $featureFlags.smartSearch);
 
   // $: showEditorButton =
   //   isOwner &&
@@ -225,6 +228,13 @@
                 text={$t('view_in_timeline')}
               />
             {/if}
+            {#if !asset.isArchived && !asset.isTrashed && smartSearchEnabled}
+              <MenuOption
+                icon={mdiCompare}
+                onClick={() => goto(`${AppRoute.SEARCH}?query={"queryAssetId":"${stack?.primaryAssetId ?? asset.id}"}`)}
+                text={$t('view_similar_photos')}
+              />
+            {/if}
           {/if}
 
           {#if !asset.isTrashed}

web/src/lib/modals/SearchFilterModal.svelte

@@ -64,8 +64,16 @@
     return validQueryTypes.has(storedQueryType) ? storedQueryType : QueryType.SMART;
   }
 
+  let query = '';
+  if ('query' in searchQuery && searchQuery.query) {
+    query = searchQuery.query;
+  }
+  if ('originalFileName' in searchQuery && searchQuery.originalFileName) {
+    query = searchQuery.originalFileName;
+  }
+
   let filter: SearchFilter = $state({
-    query: 'query' in searchQuery ? searchQuery.query : searchQuery.originalFileName || '',
+    query,
     queryType: defaultQueryType(),
     personIds: new SvelteSet('personIds' in searchQuery ? searchQuery.personIds : []),
     tagIds:

web/src/routes/(user)/search/[[photos=photos]]/[[assetId=id]]/+page.svelte

@@ -68,7 +68,7 @@
 
   const assetInteraction = new AssetInteraction();
 
-  type SearchTerms = MetadataSearchDto & Pick<SmartSearchDto, 'query'>;
+  type SearchTerms = MetadataSearchDto & Pick<SmartSearchDto, 'query' | 'queryAssetId'>;
   let searchQuery = $derived(page.url.searchParams.get(QueryParameter.QUERY));
   let smartSearchEnabled = $derived($featureFlags.loaded && $featureFlags.smartSearch);
   let terms = $derived(searchQuery ? JSON.parse(searchQuery) : {});
@@ -164,7 +164,7 @@
 
     try {
       const { albums, assets } =
-        'query' in searchDto && smartSearchEnabled
+        ('query' in searchDto || 'queryAssetId' in searchDto) && smartSearchEnabled
           ? await searchSmart({ smartSearchDto: searchDto })
           : await searchAssets({ metadataSearchDto: searchDto });
 
@@ -210,6 +210,7 @@
       tagIds: $t('tags'),
       originalFileName: $t('file_name'),
       description: $t('description'),
+      queryAssetId: $t('query_asset_id'),
     };
     return keyMap[key] || key;
   }