feat(core): Use language fallback on DefaultSearchPlugin search (#1696)

packages/core/e2e/default-search-plugin.e2e-spec.ts

@@ -1541,7 +1541,7 @@ describe('Default search plugin', () => {
                     SEARCH_PRODUCTS,
                     {
                         input: {
-                            take: 1,
+                            take: 100,
                         },
                     },
                     {
@@ -1592,25 +1592,34 @@ describe('Default search plugin', () => {
                 await awaitRunningJobs(adminClient);
             });
 
+            it('fallbacks to default language', async () => {
+                const { search } = await searchInLanguage(LanguageCode.af)
+                // No records for AF language, but we expect > 0
+                // because of fallback to default language (EN)
+                expect(search.totalItems).toBeGreaterThan(0);
+            })
+
             it('indexes product-level languages', async () => {
                 const {search: search1} = await searchInLanguage(LanguageCode.de);
 
-                expect(search1.items[0].productName).toBe('laptop name de');
-                expect(search1.items[0].slug).toBe('laptop-slug-de');
-                expect(search1.items[0].description).toBe('laptop description de');
+                expect(search1.items.map(i => i.productName)).toContain('laptop name de');
+                expect(search1.items.map(i => i.productName)).not.toContain('laptop name zh')
+                expect(search1.items.map(i => i.slug)).toContain('laptop-slug-de');
+                expect(search1.items.map(i => i.description)).toContain('laptop description de');
 
                 const {search: search2} = await searchInLanguage(LanguageCode.zh);
 
-                expect(search2.items[0].productName).toBe('laptop name zh');
-                expect(search2.items[0].slug).toBe('laptop-slug-zh');
-                expect(search2.items[0].description).toBe('laptop description zh');
+                expect(search2.items.map(i => i.productName)).toContain('laptop name zh');
+                expect(search2.items.map(i => i.productName)).not.toContain('laptop name de');
+                expect(search2.items.map(i => i.slug)).toContain('laptop-slug-zh');
+                expect(search2.items.map(i => i.description)).toContain('laptop description zh');
             });
 
             it('indexes product variant-level languages', async () => {
                 const {search: search1} = await searchInLanguage(LanguageCode.fr);
 
-                expect(search1.items[0].productName).toBe('Laptop');
-                expect(search1.items[0].productVariantName).toBe('laptop variant fr');
+                expect(search1.items.map(i => i.productName)).toContain('Laptop');
+                expect(search1.items.map(i => i.productVariantName)).toContain('laptop variant fr');
             });
         });
     });

packages/core/src/plugin/default-search-plugin/search-strategy/mysql-search-strategy.ts

@@ -13,6 +13,7 @@ import { DefaultSearchPluginInitOptions, SearchInput } from '../types';
 import { SearchStrategy } from './search-strategy';
 import { getFieldsToSelect } from './search-strategy-common';
 import {
+    applyLanguageConstraints,
     createCollectionIdCountMap,
     createFacetIdCountMap,
     createPlaceholderFromId,
@@ -40,8 +41,8 @@ export class MysqlSearchStrategy implements SearchStrategy {
         const facetValuesQb = this.connection
             .getRepository(ctx, SearchIndexItem)
             .createQueryBuilder('si')
-            .select(['MIN(productId)', 'MIN(productVariantId)'])
-            .addSelect('GROUP_CONCAT(facetValueIds)', 'facetValues');
+            .select(['MIN(si.productId)', 'MIN(si.productVariantId)'])
+            .addSelect('GROUP_CONCAT(si.facetValueIds)', 'facetValues');
 
         this.applyTermAndFilters(ctx, facetValuesQb, { ...input, groupByProduct: true });
         if (!input.groupByProduct) {
@@ -62,12 +63,12 @@ export class MysqlSearchStrategy implements SearchStrategy {
         const collectionsQb = this.connection
             .getRepository(ctx, SearchIndexItem)
             .createQueryBuilder('si')
-            .select(['MIN(productId)', 'MIN(productVariantId)'])
-            .addSelect('GROUP_CONCAT(collectionIds)', 'collections');
+            .select(['MIN(si.productId)', 'MIN(si.productVariantId)'])
+            .addSelect('GROUP_CONCAT(si.collectionIds)', 'collections');
 
         this.applyTermAndFilters(ctx, collectionsQb, input);
         if (!input.groupByProduct) {
-            collectionsQb.groupBy('productVariantId');
+            collectionsQb.groupBy('si.productVariantId');
         }
         if (enabledOnly) {
             collectionsQb.andWhere('si.enabled = :enabled', { enabled: true });
@@ -89,18 +90,18 @@ export class MysqlSearchStrategy implements SearchStrategy {
             .createQueryBuilder('si')
             .select(this.createMysqlSelect(!!input.groupByProduct));
         if (input.groupByProduct) {
-            qb.addSelect('MIN(price)', 'minPrice')
-                .addSelect('MAX(price)', 'maxPrice')
-                .addSelect('MIN(priceWithTax)', 'minPriceWithTax')
-                .addSelect('MAX(priceWithTax)', 'maxPriceWithTax');
+            qb.addSelect('MIN(si.price)', 'minPrice')
+                .addSelect('MAX(si.price)', 'maxPrice')
+                .addSelect('MIN(si.priceWithTax)', 'minPriceWithTax')
+                .addSelect('MAX(si.priceWithTax)', 'maxPriceWithTax');
         }
         this.applyTermAndFilters(ctx, qb, input);
         if (sort) {
             if (sort.name) {
-                qb.addOrderBy(input.groupByProduct ? 'MIN(productName)' : 'productName', sort.name);
+                qb.addOrderBy(input.groupByProduct ? 'MIN(si.productName)' : 'productName', sort.name);
             }
             if (sort.price) {
-                qb.addOrderBy(input.groupByProduct ? 'MIN(price)' : 'price', sort.price);
+                qb.addOrderBy(input.groupByProduct ? 'MIN(si.price)' : 'price', sort.price);
             }
         } else {
             if (input.term && input.term.length > this.minTermLength) {
@@ -153,23 +154,23 @@ export class MysqlSearchStrategy implements SearchStrategy {
                 .createQueryBuilder('si_inner')
                 .select('si_inner.productId', 'inner_productId')
                 .addSelect('si_inner.productVariantId', 'inner_productVariantId')
-                .addSelect(`IF (sku LIKE :like_term, 10, 0)`, 'sku_score')
+                .addSelect(`IF (si_inner.sku LIKE :like_term, 10, 0)`, 'sku_score')
                 .addSelect(
                     `(SELECT sku_score) +
-                     MATCH (productName) AGAINST (:term IN BOOLEAN MODE) * 2 +
-                     MATCH (productVariantName) AGAINST (:term IN BOOLEAN MODE) * 1.5 +
-                     MATCH (description) AGAINST (:term IN BOOLEAN MODE) * 1`,
+                     MATCH (si_inner.productName) AGAINST (:term IN BOOLEAN MODE) * 2 +
+                     MATCH (si_inner.productVariantName) AGAINST (:term IN BOOLEAN MODE) * 1.5 +
+                     MATCH (si_inner.description) AGAINST (:term IN BOOLEAN MODE) * 1`,
                     'score',
                 )
                 .where(
                     new Brackets(qb1 => {
-                        qb1.where('sku LIKE :like_term')
-                            .orWhere('MATCH (productName) AGAINST (:term IN BOOLEAN MODE)')
-                            .orWhere('MATCH (productVariantName) AGAINST (:term IN BOOLEAN MODE)')
-                            .orWhere('MATCH (description) AGAINST (:term IN BOOLEAN MODE)');
+                        qb1.where('si_inner.sku LIKE :like_term')
+                            .orWhere('MATCH (si_inner.productName) AGAINST (:term IN BOOLEAN MODE)')
+                            .orWhere('MATCH (si_inner.productVariantName) AGAINST (:term IN BOOLEAN MODE)')
+                            .orWhere('MATCH (si_inner.description) AGAINST (:term IN BOOLEAN MODE)');
                     }),
                 )
-                .andWhere('channelId = :channelId')
+                .andWhere('si_inner.channelId = :channelId')
                 .setParameters({ term: `${term}*`, like_term: `%${term}%`, channelId: ctx.channelId });
 
             qb.innerJoin(`(${termScoreQuery.getQuery()})`, 'term_result', 'inner_productId = si.productId')
@@ -181,9 +182,9 @@ export class MysqlSearchStrategy implements SearchStrategy {
         }
         if (input.inStock != null) {
             if (input.groupByProduct) {
-                qb.andWhere('productInStock = :inStock', { inStock: input.inStock });
+                qb.andWhere('si.productInStock = :inStock', { inStock: input.inStock });
             } else {
-                qb.andWhere('inStock = :inStock', { inStock: input.inStock });
+                qb.andWhere('si.inStock = :inStock', { inStock: input.inStock });
             }
         }
         if (facetValueIds?.length) {
@@ -191,7 +192,7 @@ export class MysqlSearchStrategy implements SearchStrategy {
                 new Brackets(qb1 => {
                     for (const id of facetValueIds) {
                         const placeholder = createPlaceholderFromId(id);
-                        const clause = `FIND_IN_SET(:${placeholder}, facetValueIds)`;
+                        const clause = `FIND_IN_SET(:${placeholder}, si.facetValueIds)`;
                         const params = { [placeholder]: id };
                         if (facetValueOperator === LogicalOperator.AND) {
                             qb1.andWhere(clause, params);
@@ -213,14 +214,14 @@ export class MysqlSearchStrategy implements SearchStrategy {
                                 }
                                 if (facetValueFilter.and) {
                                     const placeholder = createPlaceholderFromId(facetValueFilter.and);
-                                    const clause = `FIND_IN_SET(:${placeholder}, facetValueIds)`;
+                                    const clause = `FIND_IN_SET(:${placeholder}, si.facetValueIds)`;
                                     const params = { [placeholder]: facetValueFilter.and };
                                     qb2.where(clause, params);
                                 }
                                 if (facetValueFilter.or?.length) {
                                     for (const id of facetValueFilter.or) {
                                         const placeholder = createPlaceholderFromId(id);
-                                        const clause = `FIND_IN_SET(:${placeholder}, facetValueIds)`;
+                                        const clause = `FIND_IN_SET(:${placeholder}, si.facetValueIds)`;
                                         const params = { [placeholder]: id };
                                         qb2.orWhere(clause, params);
                                     }
@@ -232,16 +233,17 @@ export class MysqlSearchStrategy implements SearchStrategy {
             );
         }
         if (collectionId) {
-            qb.andWhere(`FIND_IN_SET (:collectionId, collectionIds)`, { collectionId });
+            qb.andWhere(`FIND_IN_SET (:collectionId, si.collectionIds)`, { collectionId });
         }
         if (collectionSlug) {
-            qb.andWhere(`FIND_IN_SET (:collectionSlug, collectionSlugs)`, { collectionSlug });
+            qb.andWhere(`FIND_IN_SET (:collectionSlug, si.collectionSlugs)`, { collectionSlug });
         }
-        qb.andWhere('languageCode = :languageCode', { languageCode: ctx.languageCode });
-        qb.andWhere('channelId = :channelId', { channelId: ctx.channelId });
+
+        applyLanguageConstraints(qb, ctx.languageCode, ctx.channel.defaultLanguageCode);
+        qb.andWhere('si.channelId = :channelId', { channelId: ctx.channelId });
         if (input.groupByProduct === true) {
-            qb.groupBy('productId');
-            qb.addSelect('BIT_OR(enabled)', 'productEnabled');
+            qb.groupBy('si.productId');
+            qb.addSelect('BIT_OR(si.enabled)', 'productEnabled');
         }
         return qb;
     }

packages/core/src/plugin/default-search-plugin/search-strategy/postgres-search-strategy.ts

@@ -13,6 +13,7 @@ import { DefaultSearchPluginInitOptions, SearchInput } from '../types';
 import { SearchStrategy } from './search-strategy';
 import { getFieldsToSelect } from './search-strategy-common';
 import {
+    applyLanguageConstraints,
     createCollectionIdCountMap,
     createFacetIdCountMap,
     createPlaceholderFromId,
@@ -89,10 +90,10 @@ export class PostgresSearchStrategy implements SearchStrategy {
             .createQueryBuilder('si')
             .select(this.createPostgresSelect(!!input.groupByProduct));
         if (input.groupByProduct) {
-            qb.addSelect('MIN(price)', 'minPrice')
-                .addSelect('MAX(price)', 'maxPrice')
-                .addSelect('MIN("priceWithTax")', 'minPriceWithTax')
-                .addSelect('MAX("priceWithTax")', 'maxPriceWithTax');
+            qb.addSelect('MIN(si.price)', 'minPrice')
+                .addSelect('MAX(si.price)', 'maxPrice')
+                .addSelect('MIN(si.priceWithTax)', 'minPriceWithTax')
+                .addSelect('MAX(si.priceWithTax)', 'maxPriceWithTax');
         }
         this.applyTermAndFilters(ctx, qb, input);
 
@@ -243,7 +244,8 @@ export class PostgresSearchStrategy implements SearchStrategy {
                 collectionSlug,
             });
         }
-        qb.andWhere('si.languageCode = :languageCode', { languageCode: ctx.languageCode });
+        
+        applyLanguageConstraints(qb, ctx.languageCode, ctx.channel.defaultLanguageCode);
         qb.andWhere('si.channelId = :channelId', { channelId: ctx.channelId });
         if (input.groupByProduct === true) {
             qb.groupBy('si.productId');

packages/core/src/plugin/default-search-plugin/search-strategy/search-strategy-common.ts

@@ -22,6 +22,14 @@ export const fieldsToSelect = [
     'productVariantPreviewFocalPoint',
 ];
 
+export const identifierFields = [
+    'channelId',
+    'productVariantId',
+    'productId',
+    'productAssetId',
+    'productVariantAssetId',
+]
+
 export function getFieldsToSelect(includeStockStatus: boolean = false) {
     return includeStockStatus ? [...fieldsToSelect, 'inStock', 'productInStock'] : fieldsToSelect;
 }

packages/core/src/plugin/default-search-plugin/search-strategy/search-strategy-utils.ts

@@ -1,6 +1,7 @@
 import {
     Coordinate,
     CurrencyCode,
+    LanguageCode,
     PriceRange,
     SearchResult,
     SearchResultAsset,
@@ -8,6 +9,9 @@ import {
 } from '@vendure/common/lib/generated-types';
 import { ID } from '@vendure/common/lib/shared-types';
 import { unique } from '@vendure/common/lib/unique';
+import { QueryBuilder, SelectQueryBuilder } from 'typeorm';
+import { SearchIndexItem } from '../entities/search-index-item.entity';
+import { identifierFields } from './search-strategy-common';
 
 /**
  * Maps a raw database result to a SearchResult.
@@ -110,3 +114,40 @@ function parseFocalPoint(focalPoint: any): Coordinate | undefined {
 export function createPlaceholderFromId(id: ID): string {
     return '_' + id.toString().replace(/-/g, '_');
 }
+
+/**
+ * Applies language constraints for {@link SearchIndexItem} query.
+ * 
+ * @param qb QueryBuilder instance
+ * @param languageCode Preferred language code
+ * @param defaultLanguageCode Default language code that is used if {@link SearchIndexItem} is not available in preferred language
+ */
+export function applyLanguageConstraints(
+    qb: SelectQueryBuilder<SearchIndexItem>,
+    languageCode: LanguageCode,
+    defaultLanguageCode: LanguageCode,
+) {
+    if (languageCode == defaultLanguageCode) {
+        qb.andWhere('si.languageCode = :languageCode', { languageCode: languageCode });
+    } else {
+        qb.andWhere('si.languageCode IN (:...languageCodes)', {
+            languageCodes: [languageCode, defaultLanguageCode],
+        });
+
+        const joinFieldConditions = identifierFields
+            .map(field => `si.${field} = sil.${field}`)
+            .join(' AND ');
+
+        qb.leftJoin(SearchIndexItem, 'sil', `
+            ${joinFieldConditions}
+            AND si.languageCode != sil.languageCode
+            AND sil.languageCode = :languageCode
+        `, {
+            languageCode: languageCode,
+        });
+
+        qb.andWhere('sil.languageCode IS NULL');
+    }
+
+    return qb;
+}

packages/core/src/plugin/default-search-plugin/search-strategy/sqlite-search-strategy.ts

@@ -12,6 +12,7 @@ import { DefaultSearchPluginInitOptions, SearchInput } from '../types';
 
 import { SearchStrategy } from './search-strategy';
 import {
+    applyLanguageConstraints,
     createCollectionIdCountMap,
     createFacetIdCountMap,
     createPlaceholderFromId,
@@ -40,12 +41,12 @@ export class SqliteSearchStrategy implements SearchStrategy {
         const facetValuesQb = this.connection
             .getRepository(ctx, SearchIndexItem)
             .createQueryBuilder('si')
-            .select(['productId', 'productVariantId'])
+            .select(['si.productId', 'si.productVariantId'])
             .addSelect('GROUP_CONCAT(si.facetValueIds)', 'facetValues');
 
         this.applyTermAndFilters(ctx, facetValuesQb, input);
         if (!input.groupByProduct) {
-            facetValuesQb.groupBy('productVariantId');
+            facetValuesQb.groupBy('si.productVariantId');
         }
         if (enabledOnly) {
             facetValuesQb.andWhere('si.enabled = :enabled', { enabled: true });
@@ -62,12 +63,12 @@ export class SqliteSearchStrategy implements SearchStrategy {
         const collectionsQb = this.connection
             .getRepository(ctx, SearchIndexItem)
             .createQueryBuilder('si')
-            .select(['productId', 'productVariantId'])
+            .select(['si.productId', 'si.productVariantId'])
             .addSelect('GROUP_CONCAT(si.collectionIds)', 'collections');
 
         this.applyTermAndFilters(ctx, collectionsQb, input);
         if (!input.groupByProduct) {
-            collectionsQb.groupBy('productVariantId');
+            collectionsQb.groupBy('si.productVariantId');
         }
         if (enabledOnly) {
             collectionsQb.andWhere('si.enabled = :enabled', { enabled: true });
@@ -86,9 +87,9 @@ export class SqliteSearchStrategy implements SearchStrategy {
         const sort = input.sort;
         const qb = this.connection.getRepository(ctx, SearchIndexItem).createQueryBuilder('si');
         if (input.groupByProduct) {
-            qb.addSelect('MIN(price)', 'minPrice').addSelect('MAX(price)', 'maxPrice');
-            qb.addSelect('MIN(priceWithTax)', 'minPriceWithTax').addSelect(
-                'MAX(priceWithTax)',
+            qb.addSelect('MIN(si.price)', 'minPrice').addSelect('MAX(si.price)', 'maxPrice');
+            qb.addSelect('MIN(si.priceWithTax)', 'minPriceWithTax').addSelect(
+                'MAX(si.priceWithTax)',
                 'maxPriceWithTax',
             );
         }
@@ -98,13 +99,13 @@ export class SqliteSearchStrategy implements SearchStrategy {
         }
         if (sort) {
             if (sort.name) {
-                qb.addOrderBy('productName', sort.name);
+                qb.addOrderBy('si.productName', sort.name);
             }
             if (sort.price) {
-                qb.addOrderBy('price', sort.price);
+                qb.addOrderBy('si.price', sort.price);
             }
         } else {
-            qb.addOrderBy('productVariantId', 'ASC');
+            qb.addOrderBy('si.productVariantId', 'ASC');
         }
         if (enabledOnly) {
             qb.andWhere('si.enabled = :enabled', { enabled: true });
@@ -114,7 +115,10 @@ export class SqliteSearchStrategy implements SearchStrategy {
             .take(take)
             .skip(skip)
             .getRawMany()
-            .then(res => res.map(r => mapToSearchResult(r, ctx.channel.currencyCode)));
+            .then(res => {
+                // console.warn(qb.getQueryAndParameters(), "take", take, "skip", skip, "length", res.length)
+                return res.map(r => mapToSearchResult(r, ctx.channel.currencyCode))
+            });
     }
 
     async getTotalCount(ctx: RequestContext, input: SearchInput, enabledOnly: boolean): Promise<number> {
@@ -150,27 +154,27 @@ export class SqliteSearchStrategy implements SearchStrategy {
             // so we just use a weighted LIKE match
             qb.addSelect(
                 `
-                    CASE WHEN sku LIKE :like_term THEN 10 ELSE 0 END +
-                    CASE WHEN productName LIKE :like_term THEN 3 ELSE 0 END +
-                    CASE WHEN productVariantName LIKE :like_term THEN 2 ELSE 0 END +
-                    CASE WHEN description LIKE :like_term THEN 1 ELSE 0 END`,
+                    CASE WHEN si.sku LIKE :like_term THEN 10 ELSE 0 END +
+                    CASE WHEN si.productName LIKE :like_term THEN 3 ELSE 0 END +
+                    CASE WHEN si.productVariantName LIKE :like_term THEN 2 ELSE 0 END +
+                    CASE WHEN si.description LIKE :like_term THEN 1 ELSE 0 END`,
                 'score',
             )
                 .andWhere(
                     new Brackets(qb1 => {
-                        qb1.where('sku LIKE :like_term')
-                            .orWhere('productName LIKE :like_term')
-                            .orWhere('productVariantName LIKE :like_term')
-                            .orWhere('description LIKE :like_term');
+                        qb1.where('si.sku LIKE :like_term')
+                            .orWhere('si.productName LIKE :like_term')
+                            .orWhere('si.productVariantName LIKE :like_term')
+                            .orWhere('si.description LIKE :like_term');
                     }),
                 )
                 .setParameters({ term, like_term: `%${term}%` });
         }
         if (input.inStock != null) {
             if (input.groupByProduct) {
-                qb.andWhere('productInStock = :inStock', { inStock: input.inStock });
+                qb.andWhere('si.productInStock = :inStock', { inStock: input.inStock });
             } else {
-                qb.andWhere('inStock = :inStock', { inStock: input.inStock });
+                qb.andWhere('si.inStock = :inStock', { inStock: input.inStock });
             }
         }
         if (facetValueIds?.length) {
@@ -178,7 +182,7 @@ export class SqliteSearchStrategy implements SearchStrategy {
                 new Brackets(qb1 => {
                     for (const id of facetValueIds) {
                         const placeholder = createPlaceholderFromId(id);
-                        const clause = `(',' || facetValueIds || ',') LIKE :${placeholder}`;
+                        const clause = `(',' || si.facetValueIds || ',') LIKE :${placeholder}`;
                         const params = { [placeholder]: `%,${id},%` };
                         if (facetValueOperator === LogicalOperator.AND) {
                             qb1.andWhere(clause, params);
@@ -200,14 +204,14 @@ export class SqliteSearchStrategy implements SearchStrategy {
                                 }
                                 if (facetValueFilter.and) {
                                     const placeholder = createPlaceholderFromId(facetValueFilter.and);
-                                    const clause = `(',' || facetValueIds || ',') LIKE :${placeholder}`;
+                                    const clause = `(',' || si.facetValueIds || ',') LIKE :${placeholder}`;
                                     const params = { [placeholder]: `%,${facetValueFilter.and},%` };
                                     qb2.where(clause, params);
                                 }
                                 if (facetValueFilter.or?.length) {
                                     for (const id of facetValueFilter.or) {
                                         const placeholder = createPlaceholderFromId(id);
-                                        const clause = `(',' || facetValueIds || ',') LIKE :${placeholder}`;
+                                        const clause = `(',' || si.facetValueIds || ',') LIKE :${placeholder}`;
                                         const params = { [placeholder]: `%,${id},%` };
                                         qb2.orWhere(clause, params);
                                     }
@@ -219,20 +223,23 @@ export class SqliteSearchStrategy implements SearchStrategy {
             );
         }
         if (collectionId) {
-            qb.andWhere(`(',' || collectionIds || ',') LIKE :collectionId`, {
+            qb.andWhere(`(',' || si.collectionIds || ',') LIKE :collectionId`, {
                 collectionId: `%,${collectionId},%`,
             });
         }
         if (collectionSlug) {
-            qb.andWhere(`(',' || collectionSlugs || ',') LIKE :collectionSlug`, {
+            qb.andWhere(`(',' || si.collectionSlugs || ',') LIKE :collectionSlug`, {
                 collectionSlug: `%,${collectionSlug},%`,
             });
         }
-        qb.andWhere('languageCode = :languageCode', { languageCode: ctx.languageCode });
-        qb.andWhere('channelId = :channelId', { channelId: ctx.channelId });
+
+        applyLanguageConstraints(qb, ctx.languageCode, ctx.channel.defaultLanguageCode);
+        qb.andWhere('si.channelId = :channelId', { channelId: ctx.channelId });
+
         if (input.groupByProduct === true) {
-            qb.groupBy('productId');
+            qb.groupBy('si.productId');
         }
+
         return qb;
     }
 }