feat(core): Improve speed of bulk product import

packages/core/src/data-import/data-import.module.ts

@@ -5,6 +5,7 @@ import { PluginModule } from '../plugin/plugin.module';
 import { ServiceModule } from '../service/service.module';
 
 import { ImportParser } from './providers/import-parser/import-parser';
+import { FastImporterService } from './providers/importer/fast-importer.service';
 import { Importer } from './providers/importer/importer';
 import { Populator } from './providers/populator/populator';
 
@@ -13,7 +14,7 @@ import { Populator } from './providers/populator/populator';
     // in order that overrides of Services (e.g. SearchService) are correctly
     // registered with the injector.
     imports: [PluginModule, ServiceModule.forRoot(), ConfigModule],
-    exports: [ImportParser, Importer, Populator],
-    providers: [ImportParser, Importer, Populator],
+    exports: [ImportParser, Importer, Populator, FastImporterService],
+    providers: [ImportParser, Importer, Populator, FastImporterService],
 })
 export class DataImportModule {}

packages/core/src/data-import/providers/importer/fast-importer.service.ts

@@ -0,0 +1,139 @@
+import { Injectable } from '@nestjs/common';
+import { InjectConnection } from '@nestjs/typeorm';
+import {
+    CreateProductInput,
+    CreateProductOptionGroupInput,
+    CreateProductOptionInput,
+    CreateProductVariantInput,
+} from '@vendure/common/lib/generated-types';
+import { ID } from '@vendure/common/lib/shared-types';
+import { Connection } from 'typeorm';
+
+import { Channel } from '../../../entity/channel/channel.entity';
+import { ProductOptionGroupTranslation } from '../../../entity/product-option-group/product-option-group-translation.entity';
+import { ProductOptionGroup } from '../../../entity/product-option-group/product-option-group.entity';
+import { ProductOptionTranslation } from '../../../entity/product-option/product-option-translation.entity';
+import { ProductOption } from '../../../entity/product-option/product-option.entity';
+import { ProductVariantPrice } from '../../../entity/product-variant/product-variant-price.entity';
+import { ProductVariantTranslation } from '../../../entity/product-variant/product-variant-translation.entity';
+import { ProductVariant } from '../../../entity/product-variant/product-variant.entity';
+import { ProductTranslation } from '../../../entity/product/product-translation.entity';
+import { Product } from '../../../entity/product/product.entity';
+import { TranslatableSaver } from '../../../service/helpers/translatable-saver/translatable-saver';
+import { ChannelService } from '../../../service/services/channel.service';
+import { StockMovementService } from '../../../service/services/stock-movement.service';
+
+/**
+ * A service to import entities into the database. This replaces the regular `create` methods of the service layer with faster
+ * versions which skip much of the defensive checks and other DB calls which are not needed when running an import.
+ *
+ * In testing, the use of the FastImporterService approximately doubled the speed of bulk imports.
+ */
+@Injectable()
+export class FastImporterService {
+    private defaultChannel: Channel;
+    constructor(
+        @InjectConnection() private connection: Connection,
+        private channelService: ChannelService,
+        private stockMovementService: StockMovementService,
+        private translatableSaver: TranslatableSaver,
+    ) {}
+
+    async initialize() {
+        this.defaultChannel = this.channelService.getDefaultChannel();
+    }
+
+    async createProduct(input: CreateProductInput): Promise<ID> {
+        const product = await this.translatableSaver.create({
+            input,
+            entityType: Product,
+            translationType: ProductTranslation,
+            beforeSave: async p => {
+                p.channels = [this.defaultChannel];
+                if (input.facetValueIds) {
+                    p.facetValues = input.facetValueIds.map(id => ({ id } as any));
+                }
+                if (input.featuredAssetId) {
+                    p.featuredAsset = { id: input.featuredAssetId } as any;
+                }
+                if (input.assetIds) {
+                    p.assets = input.assetIds.map(id => ({ id } as any));
+                }
+            },
+        });
+        return product.id;
+    }
+
+    async createProductOptionGroup(input: CreateProductOptionGroupInput): Promise<ID> {
+        const group = await this.translatableSaver.create({
+            input,
+            entityType: ProductOptionGroup,
+            translationType: ProductOptionGroupTranslation,
+        });
+        return group.id;
+    }
+
+    async createProductOption(input: CreateProductOptionInput): Promise<ID> {
+        const option = await this.translatableSaver.create({
+            input,
+            entityType: ProductOption,
+            translationType: ProductOptionTranslation,
+            beforeSave: po => (po.group = { id: input.productOptionGroupId } as any),
+        });
+        return option.id;
+    }
+
+    async addOptionGroupToProduct(productId: ID, optionGroupId: ID) {
+        await this.connection
+            .createQueryBuilder()
+            .relation(Product, 'optionGroups')
+            .of(productId)
+            .add(optionGroupId);
+    }
+
+    async createProductVariant(input: CreateProductVariantInput): Promise<ID> {
+        if (!input.optionIds) {
+            input.optionIds = [];
+        }
+        if (input.price == null) {
+            input.price = 0;
+        }
+
+        const createdVariant = await this.translatableSaver.create({
+            input,
+            entityType: ProductVariant,
+            translationType: ProductVariantTranslation,
+            beforeSave: async variant => {
+                const { optionIds } = input;
+                if (optionIds && optionIds.length) {
+                    variant.options = optionIds.map(id => ({ id } as any));
+                }
+                if (input.facetValueIds) {
+                    variant.facetValues = input.facetValueIds.map(id => ({ id } as any));
+                }
+                variant.product = { id: input.productId } as any;
+                variant.taxCategory = { id: input.taxCategoryId } as any;
+                if (input.featuredAssetId) {
+                    variant.featuredAsset = { id: input.featuredAssetId } as any;
+                }
+                if (input.assetIds) {
+                    variant.assets = input.assetIds.map(id => ({ id } as any));
+                }
+            },
+        });
+        if (input.stockOnHand != null && input.stockOnHand !== 0) {
+            await this.stockMovementService.adjustProductVariantStock(
+                createdVariant.id,
+                0,
+                input.stockOnHand,
+            );
+        }
+        const variantPrice = new ProductVariantPrice({
+            price: createdVariant.price,
+            channelId: this.defaultChannel.id,
+        });
+        variantPrice.variant = createdVariant;
+        await this.connection.getRepository(ProductVariantPrice).save(variantPrice);
+        return createdVariant.id;
+    }
+}

packages/core/src/data-import/providers/importer/importer.ts

@@ -17,10 +17,6 @@ import { AssetService } from '../../../service/services/asset.service';
 import { ChannelService } from '../../../service/services/channel.service';
 import { FacetValueService } from '../../../service/services/facet-value.service';
 import { FacetService } from '../../../service/services/facet.service';
-import { ProductOptionGroupService } from '../../../service/services/product-option-group.service';
-import { ProductOptionService } from '../../../service/services/product-option.service';
-import { ProductVariantService } from '../../../service/services/product-variant.service';
-import { ProductService } from '../../../service/services/product.service';
 import { TaxCategoryService } from '../../../service/services/tax-category.service';
 import {
     ImportParser,
@@ -28,6 +24,8 @@ import {
     ParsedProductWithVariants,
 } from '../import-parser/import-parser';
 
+import { FastImporterService } from './fast-importer.service';
+
 export interface ImportProgress extends ImportInfo {
     currentProduct: string;
 }
@@ -46,14 +44,11 @@ export class Importer {
         private configService: ConfigService,
         private importParser: ImportParser,
         private channelService: ChannelService,
-        private productService: ProductService,
         private facetService: FacetService,
         private facetValueService: FacetValueService,
-        private productVariantService: ProductVariantService,
-        private productOptionGroupService: ProductOptionGroupService,
         private assetService: AssetService,
         private taxCategoryService: TaxCategoryService,
-        private productOptionService: ProductOptionService,
+        private fastImporter: FastImporterService,
     ) {}
 
     parseAndImport(
@@ -149,13 +144,14 @@ export class Importer {
         let imported = 0;
         const languageCode = ctx.languageCode;
         const taxCategories = await this.taxCategoryService.findAll();
+        await this.fastImporter.initialize();
         for (const { product, variants } of rows) {
             const createProductAssets = await this.getAssets(product.assetPaths);
             const productAssets = createProductAssets.assets;
             if (createProductAssets.errors.length) {
                 errors = errors.concat(createProductAssets.errors);
             }
-            const createdProduct = await this.productService.create(ctx, {
+            const createdProductId = await this.fastImporter.createProduct({
                 featuredAssetId: productAssets.length ? (productAssets[0].id as string) : undefined,
                 assetIds: productAssets.map(a => a.id) as string[],
                 facetValueIds: await this.getFacetValueIds(product.facets, languageCode),
@@ -173,7 +169,7 @@ export class Importer {
             const optionsMap: { [optionName: string]: string } = {};
             for (const optionGroup of product.optionGroups) {
                 const code = normalizeString(`${product.name}-${optionGroup.name}`, '-');
-                const group = await this.productOptionGroupService.create(ctx, {
+                const groupId = await this.fastImporter.createProductOptionGroup({
                     code,
                     options: optionGroup.values.map(name => ({} as any)),
                     translations: [
@@ -184,7 +180,8 @@ export class Importer {
                     ],
                 });
                 for (const option of optionGroup.values) {
-                    const createdOption = await this.productOptionService.create(ctx, group, {
+                    const createdOptionId = await this.fastImporter.createProductOption({
+                        productOptionGroupId: groupId as string,
                         code: normalizeString(option, '-'),
                         translations: [
                             {
@@ -193,9 +190,9 @@ export class Importer {
                             },
                         ],
                     });
-                    optionsMap[option] = createdOption.id as string;
+                    optionsMap[option] = createdOptionId as string;
                 }
-                await this.productService.addOptionGroupToProduct(ctx, createdProduct.id, group.id);
+                await this.fastImporter.addOptionGroupToProduct(createdProductId, groupId);
             }
 
             for (const variant of variants) {
@@ -208,8 +205,8 @@ export class Importer {
                 if (0 < variant.facets.length) {
                     facetValueIds = await this.getFacetValueIds(variant.facets, languageCode);
                 }
-                const createdVariant = await this.productVariantService.create(ctx, {
-                    productId: createdProduct.id as string,
+                const createdVariant = await this.fastImporter.createProductVariant({
+                    productId: createdProductId as string,
                     facetValueIds,
                     featuredAssetId: variantAssets.length ? (variantAssets[0].id as string) : undefined,
                     assetIds: variantAssets.map(a => a.id) as string[],
@@ -247,7 +244,8 @@ export class Importer {
         const assets: Asset[] = [];
         const errors: string[] = [];
         const { importAssetsDir } = this.configService.importExportOptions;
-        for (const assetPath of assetPaths) {
+        const uniqueAssetPaths = new Set(assetPaths);
+        for (const assetPath of uniqueAssetPaths.values()) {
             const cachedAsset = this.assetMap.get(assetPath);
             if (cachedAsset) {
                 assets.push(cachedAsset);

packages/core/src/service/service.module.ts

@@ -78,6 +78,7 @@ const exportedProviders = [
     TaxRateService,
     UserService,
     ZoneService,
+    TranslatableSaver,
 ];
 
 let defaultTypeOrmModule: DynamicModule;
@@ -93,7 +94,6 @@ let workerTypeOrmModule: DynamicModule;
     providers: [
         ...exportedProviders,
         PasswordCiper,
-        TranslatableSaver,
         TaxCalculator,
         OrderCalculator,
         OrderStateMachine,