feat(core): Expose & document DataImportModule providers

Closes #1336

docs/content/developer-guide/importing-product-data.md

@@ -178,6 +178,7 @@ export const initialData: InitialData = {
 
 ## Populating The Server
 
+### The `populate()` function
 The `@vendure/core` package exposes a [`populate()` function]({{< relref "populate" >}}) which can be used along with the data formats described above to populate your Vendure server:
 
 ```TypeScript
@@ -216,7 +217,16 @@ populate(
 );
 ```
 
-{{< alert >}}
+### Custom populate scripts
+
 If you require more control over how your data is being imported - for example if you also need to import data into custom entities - you can create your own CLI script to do this: see [Stand-Alone CLI Scripts]({{< relref "stand-alone-scripts" >}}).
-{{< /alert >}} 
 
+In your script you can make use of the internal parse and import services:
+
+* [Importer]({{< relref "importer" >}})
+* [ImportParser]({{< relref "import-parser" >}})
+* [FastImporterService]({{< relref "fast-importer-service" >}})
+* [AssetImporter]({{< relref "asset-importer" >}})
+* [Populator]({{< relref "populator" >}})
+
+Using these specialized import services is preferable to using the normal service-layer services (ProductService, ProductVariantService etc.) for bulk imports. This is because these import services are optimized for bulk imports (they omit unnecessary checks, use optimized SQL queries) and also do not publish events when creating new entities.

packages/core/src/cli/populate.ts

@@ -8,7 +8,38 @@ import { logColored } from './cli-utils';
 /**
  * @description
  * Populates the Vendure server with some initial data and (optionally) product data from
- * a supplied CSV file.
+ * a supplied CSV file. The format of the CSV file is described in the section
+ * [Importing Product Data](/docs/developer-guide/importing-product-data).
+ *
+ * Internally the `populate()` function does the following:
+ *
+ * 1. Uses the {@link Populator} to populate the {@link InitialData}.
+ * 2. If `productsCsvPath` is provided, uses {@link Importer} to populate Product data.
+ * 3. Uses {@Populator} to populate collections specified in the {@link InitialData}.
+ *
+ * @example
+ * ```TypeScript
+ * import { bootstrap } from '\@vendure/core';
+ * import { populate } from '\@vendure/core/cli';
+ * import { config } from './vendure-config.ts'
+ * import { initialData } from './my-initial-data.ts';
+ *
+ * const productsCsvFile = path.join(__dirname, 'path/to/products.csv')
+ *
+ * populate(
+ *   () => bootstrap(config),
+ *   initialData,
+ *   productsCsvFile,
+ * )
+ * .then(app => app.close())
+ * .then(
+ *   () => process.exit(0),
+ *   err => {
+ *     console.log(err);
+ *     process.exit(1);
+ *   },
+ * );
+ * ```
  *
  * @docsCategory import-export
  */

packages/core/src/data-import/index.ts

@@ -1,3 +1,6 @@
 export * from './providers/populator/populator';
 export * from './providers/importer/importer';
+export * from './providers/importer/fast-importer.service';
+export * from './providers/asset-importer/asset-importer';
+export * from './providers/import-parser/import-parser';
 export * from './types';

packages/core/src/data-import/providers/asset-importer/asset-importer.ts

@@ -6,13 +6,23 @@ import { ConfigService } from '../../../config/config.service';
 import { Asset } from '../../../entity/asset/asset.entity';
 import { AssetService } from '../../../service/services/asset.service';
 
+/**
+ * @description
+ * This service creates new {@link Asset} entities based on string paths provided in the CSV
+ * import format. The source files are resolved by joining the value of `importExportOptions.importAssetsDir`
+ * with the asset path. This service is used internally by the {@link Importer} service.
+ *
+ * @docsCategory import-export
+ */
 @Injectable()
 export class AssetImporter {
     private assetMap = new Map<string, Asset>();
 
+    /** @internal */
     constructor(private configService: ConfigService, private assetService: AssetService) {}
 
     /**
+     * @description
      * Creates Asset entities for the given paths, using the assetMap cache to prevent the
      * creation of duplicates.
      */

packages/core/src/data-import/providers/import-parser/import-parser.ts

@@ -34,6 +34,14 @@ const requiredColumns: string[] = [
     'variantFacets',
 ];
 
+/**
+ * @description
+ * The intermediate representation of an OptionGroup after it has been parsed
+ * by the {@link ImportParser}.
+ *
+ * @docsCategory import-export
+ * @docsPage ImportParser
+ */
 export interface ParsedOptionGroup {
     translations: Array<{
         languageCode: LanguageCode;
@@ -42,6 +50,14 @@ export interface ParsedOptionGroup {
     }>;
 }
 
+/**
+ * @description
+ * The intermediate representation of a Facet after it has been parsed
+ * by the {@link ImportParser}.
+ *
+ * @docsCategory import-export
+ * @docsPage ImportParser
+ */
 export interface ParsedFacet {
     translations: Array<{
         languageCode: LanguageCode;
@@ -50,6 +66,14 @@ export interface ParsedFacet {
     }>;
 }
 
+/**
+ * @description
+ * The intermediate representation of a ProductVariant after it has been parsed
+ * by the {@link ImportParser}.
+ *
+ * @docsCategory import-export
+ * @docsPage ImportParser
+ */
 export interface ParsedProductVariant {
     sku: string;
     price: number;
@@ -67,6 +91,14 @@ export interface ParsedProductVariant {
     }>;
 }
 
+/**
+ * @description
+ * The intermediate representation of a Product after it has been parsed
+ * by the {@link ImportParser}.
+ *
+ * @docsCategory import-export
+ * @docsPage ImportParser
+ */
 export interface ParsedProduct {
     assetPaths: string[];
     optionGroups: ParsedOptionGroup[];
@@ -82,11 +114,26 @@ export interface ParsedProduct {
     }>;
 }
 
+/**
+ * @description
+ * The data structure into which an import CSV file is parsed by the
+ * {@link ImportParser} `parseProducts()` method.
+ *
+ * @docsCategory import-export
+ * @docsPage ImportParser
+ */
 export interface ParsedProductWithVariants {
     product: ParsedProduct;
     variants: ParsedProductVariant[];
 }
 
+/**
+ * @description
+ * The result returned by the {@link ImportParser} `parseProducts()` method.
+ *
+ * @docsCategory import-export
+ * @docsPage ImportParser
+ */
 export interface ParseResult<T> {
     results: T[];
     errors: string[];
@@ -94,12 +141,24 @@ export interface ParseResult<T> {
 }
 
 /**
+ * @description
  * Validates and parses CSV files into a data structure which can then be used to created new entities.
+ * This is used internally by the {@link Importer}.
+ *
+ * @docsCategory import-export
+ * @docsPage ImportParser
+ * @docsWeight 0
  */
 @Injectable()
 export class ImportParser {
+    /** @internal */
     constructor(private configService: ConfigService) {}
 
+    /**
+     * @description
+     * Parses the contents of the [product import CSV file](/docs/developer-guide/importing-product-data/#product-import-format) and
+     * returns a data structure which can then be used to populate Vendure using the {@link FastImporterService}.
+     */
     async parseProducts(
         input: string | Stream,
         mainLanguage: LanguageCode = this.configService.defaultLanguageCode,

packages/core/src/data-import/providers/importer/fast-importer.service.ts

@@ -26,14 +26,20 @@ import { ChannelService } from '../../../service/services/channel.service';
 import { StockMovementService } from '../../../service/services/stock-movement.service';
 
 /**
+ * @description
  * A service to import entities into the database. This replaces the regular `create` methods of the service layer with faster
- * versions which skip much of the defensive checks and other DB calls which are not needed when running an import.
+ * versions which skip much of the defensive checks and other DB calls which are not needed when running an import. It also
+ * does not publish any events, so e.g. will not trigger search index jobs.
  *
  * In testing, the use of the FastImporterService approximately doubled the speed of bulk imports.
+ *
+ * @docsCategory import-export
  */
 @Injectable()
 export class FastImporterService {
     private defaultChannel: Channel;
+
+    /** @internal */
     constructor(
         private connection: TransactionalConnection,
         private channelService: ChannelService,

packages/core/src/data-import/providers/importer/importer.ts

@@ -28,6 +28,16 @@ export interface ImportProgress extends ImportInfo {
 
 export type OnProgressFn = (progess: ImportProgress) => void;
 
+/**
+ * @description
+ * Parses and imports Products using the CSV import format.
+ *
+ * Internally it is using the {@link ImportParser} to parse the CSV file, and then the
+ * {@link FastImporterService} and the {@link AssetImporter} to actually create the resulting
+ * entities in the Vendure database.
+ *
+ * @docsCategory import-export
+ */
 @Injectable()
 export class Importer {
     private taxCategoryMatches: { [name: string]: ID } = {};
@@ -36,6 +46,7 @@ export class Importer {
     private facetMap = new Map<string, Facet>();
     private facetValueMap = new Map<string, FacetValue>();
 
+    /** @internal */
     constructor(
         private configService: ConfigService,
         private importParser: ImportParser,
@@ -47,6 +58,13 @@ export class Importer {
         private fastImporter: FastImporterService,
     ) {}
 
+    /**
+     * @description
+     * Parses the contents of the [product import CSV file](/docs/developer-guide/importing-product-data/#product-import-format) and imports
+     * the resulting Product & ProductVariants, as well as any associated Assets, Facets & FacetValues.
+     *
+     * The `ctxOrLanguageCode` argument is used to specify the languageCode to be used when creating the Products.
+     */
     parseAndImport(
         input: string | Stream,
         ctxOrLanguageCode: RequestContext | LanguageCode,

packages/core/src/data-import/providers/populator/populator.ts

@@ -30,10 +30,15 @@ import {
 import { AssetImporter } from '../asset-importer/asset-importer';
 
 /**
- * Responsible for populating the database with initial data.
+ * @description
+ * Responsible for populating the database with {@link InitialData}, i.e. non-product data such as countries, tax rates,
+ * shipping methods, payment methods & roles.
+ *
+ * @docsCategory import-export
  */
 @Injectable()
 export class Populator {
+    /** @internal */
     constructor(
         private countryService: CountryService,
         private zoneService: ZoneService,
@@ -50,6 +55,7 @@ export class Populator {
     ) {}
 
     /**
+     * @description
      * Should be run *before* populating the products, so that there are TaxRates by which
      * product prices can be set.
      */
@@ -96,6 +102,7 @@ export class Populator {
     }
 
     /**
+     * @description
      * Should be run *after* the products have been populated, otherwise the expected FacetValues will not
      * yet exist.
      */

packages/core/src/plugin/plugin-common.module.ts

@@ -3,6 +3,7 @@ import { Module } from '@nestjs/common';
 import { CacheModule } from '../cache/cache.module';
 import { ConfigModule } from '../config/config.module';
 import { ConnectionModule } from '../connection/connection.module';
+import { DataImportModule } from '../data-import/data-import.module';
 import { EventBusModule } from '../event-bus/event-bus.module';
 import { HealthCheckModule } from '../health-check/health-check.module';
 import { I18nModule } from '../i18n/i18n.module';
@@ -37,6 +38,7 @@ import { ServiceModule } from '../service/service.module';
         CacheModule,
         I18nModule,
         ProcessContextModule,
+        DataImportModule,
     ],
     exports: [
         EventBusModule,
@@ -48,6 +50,7 @@ import { ServiceModule } from '../service/service.module';
         CacheModule,
         I18nModule,
         ProcessContextModule,
+        DataImportModule,
     ],
 })
 export class PluginCommonModule {}