feat(core): Improved Collection import

Closes #173

packages/core/cli/populate.ts

@@ -21,14 +21,17 @@ try {
  */
 export async function populate(
     bootstrapFn: () => Promise<INestApplication | undefined>,
-    initialDataPath: string,
+    initialDataPathOrObject: string | object,
     productsCsvPath?: string,
 ): Promise<INestApplication> {
     const app = await bootstrapFn();
     if (!app) {
         throw new Error('Could not bootstrap the Vendure app');
     }
-    const initialData = require(initialDataPath);
+    const initialData =
+        typeof initialDataPathOrObject === 'string'
+            ? require(initialDataPathOrObject)
+            : initialDataPathOrObject;
     await populateInitialData(app, initialData);
     if (productsCsvPath) {
         await importProductsFromFile(app, productsCsvPath, initialData.defaultLanguage);
@@ -113,10 +116,13 @@ export async function populateInitialData(app: INestApplication, initialData: ob
     }
 }
 
-export async function populateCollections(app: INestApplication, initialData: object) {
+export async function populateCollections(app: INestApplication, initialData: { collections: any[] }) {
     const populator = app.get(Populator);
     try {
-        await populator.populateCollections(initialData);
+        if (initialData.collections.length) {
+            logColored(`Populating ${initialData.collections.length} Collections...`);
+            await populator.populateCollections(initialData);
+        }
     } catch (err) {
         console.error(err.message);
     }

packages/core/mock-data/data-sources/initial-data.ts

@@ -14,54 +14,81 @@ export const initialData: InitialData = {
     collections: [
         {
             name: 'Electronics',
-            facetNames: ['Electronics'],
-            featuredAssetFilename: 'jakob-owens-274337-unsplash.jpg',
+            filters: [
+                {
+                    code: 'facet-value-filter',
+                    args: { facetValueNames: ['Electronics'], containsAny: false },
+                },
+            ],
+            assetPaths: ['jakob-owens-274337-unsplash.jpg'],
         },
         {
             name: 'Computers',
-            facetNames: ['Computers'],
+            filters: [
+                { code: 'facet-value-filter', args: { facetValueNames: ['Computers'], containsAny: false } },
+            ],
             parentName: 'Electronics',
-            featuredAssetFilename: 'alexandru-acea-686569-unsplash.jpg',
+            assetPaths: ['alexandru-acea-686569-unsplash.jpg'],
         },
         {
             name: 'Camera & Photo',
-            facetNames: ['Photo'],
+            filters: [
+                { code: 'facet-value-filter', args: { facetValueNames: ['Photo'], containsAny: false } },
+            ],
             parentName: 'Electronics',
-            featuredAssetFilename: 'eniko-kis-663725-unsplash.jpg',
+            assetPaths: ['eniko-kis-663725-unsplash.jpg'],
         },
         {
             name: 'Home & Garden',
-            facetNames: ['Home & Garden'],
-            featuredAssetFilename: 'paul-weaver-1120584-unsplash.jpg',
+            filters: [
+                {
+                    code: 'facet-value-filter',
+                    args: { facetValueNames: ['Home & Garden'], containsAny: false },
+                },
+            ],
+            assetPaths: ['paul-weaver-1120584-unsplash.jpg'],
         },
         {
             name: 'Furniture',
-            facetNames: ['Furniture'],
+            filters: [
+                { code: 'facet-value-filter', args: { facetValueNames: ['Furniture'], containsAny: false } },
+            ],
             parentName: 'Home & Garden',
-            featuredAssetFilename: 'nathan-fertig-249917-unsplash.jpg',
+            assetPaths: ['nathan-fertig-249917-unsplash.jpg'],
         },
         {
             name: 'Plants',
-            facetNames: ['Plants'],
+            filters: [
+                { code: 'facet-value-filter', args: { facetValueNames: ['Plants'], containsAny: false } },
+            ],
             parentName: 'Home & Garden',
-            featuredAssetFilename: 'alex-rodriguez-santibanez-200278-unsplash.jpg',
+            assetPaths: ['alex-rodriguez-santibanez-200278-unsplash.jpg'],
         },
         {
             name: 'Sports & Outdoor',
-            facetNames: ['Sports & Outdoor'],
-            featuredAssetFilename: 'michael-guite-571169-unsplash.jpg',
+            filters: [
+                {
+                    code: 'facet-value-filter',
+                    args: { facetValueNames: ['Sports & Outdoor'], containsAny: false },
+                },
+            ],
+            assetPaths: ['michael-guite-571169-unsplash.jpg'],
         },
         {
             name: 'Equipment',
-            facetNames: ['Equipment'],
+            filters: [
+                { code: 'facet-value-filter', args: { facetValueNames: ['Equipment'], containsAny: false } },
+            ],
             parentName: 'Sports & Outdoor',
-            featuredAssetFilename: 'neonbrand-428982-unsplash.jpg',
+            assetPaths: ['neonbrand-428982-unsplash.jpg'],
         },
         {
             name: 'Footwear',
-            facetNames: ['Footwear'],
+            filters: [
+                { code: 'facet-value-filter', args: { facetValueNames: ['Footwear'], containsAny: false } },
+            ],
             parentName: 'Sports & Outdoor',
-            featuredAssetFilename: 'thomas-serer-420833-unsplash.jpg',
+            assetPaths: ['thomas-serer-420833-unsplash.jpg'],
         },
     ],
     countries: [

packages/core/src/data-import/data-import.module.ts

@@ -4,6 +4,7 @@ import { ConfigModule } from '../config/config.module';
 import { PluginModule } from '../plugin/plugin.module';
 import { ServiceModule } from '../service/service.module';
 
+import { AssetImporter } from './providers/asset-importer/asset-importer';
 import { ImportParser } from './providers/import-parser/import-parser';
 import { FastImporterService } from './providers/importer/fast-importer.service';
 import { Importer } from './providers/importer/importer';
@@ -14,7 +15,7 @@ import { Populator } from './providers/populator/populator';
     // in order that overrides of Services (e.g. SearchService) are correctly
     // registered with the injector.
     imports: [PluginModule, ServiceModule.forRoot(), ConfigModule],
-    exports: [ImportParser, Importer, Populator, FastImporterService],
-    providers: [ImportParser, Importer, Populator, FastImporterService],
+    exports: [ImportParser, Importer, Populator, FastImporterService, AssetImporter],
+    providers: [ImportParser, Importer, Populator, FastImporterService, AssetImporter],
 })
 export class DataImportModule {}

packages/core/src/data-import/index.ts

@@ -1,2 +1,3 @@
 export * from './providers/populator/populator';
 export * from './providers/importer/importer';
+export * from './types';

packages/core/src/data-import/providers/asset-importer/asset-importer.ts

@@ -0,0 +1,50 @@
+import { Injectable } from '@nestjs/common';
+import fs from 'fs-extra';
+import path from 'path';
+
+import { ConfigService } from '../../../config/config.service';
+import { Asset } from '../../../entity/asset/asset.entity';
+import { AssetService } from '../../../service/services/asset.service';
+
+@Injectable()
+export class AssetImporter {
+    private assetMap = new Map<string, Asset>();
+
+    constructor(private configService: ConfigService, private assetService: AssetService) {}
+
+    /**
+     * Creates Asset entities for the given paths, using the assetMap cache to prevent the
+     * creation of duplicates.
+     */
+    async getAssets(assetPaths: string[]): Promise<{ assets: Asset[]; errors: string[] }> {
+        const assets: Asset[] = [];
+        const errors: string[] = [];
+        const { importAssetsDir } = this.configService.importExportOptions;
+        const uniqueAssetPaths = new Set(assetPaths);
+        for (const assetPath of uniqueAssetPaths.values()) {
+            const cachedAsset = this.assetMap.get(assetPath);
+            if (cachedAsset) {
+                assets.push(cachedAsset);
+            } else {
+                const filename = path.join(importAssetsDir, assetPath);
+
+                if (fs.existsSync(filename)) {
+                    const fileStat = fs.statSync(filename);
+                    if (fileStat.isFile()) {
+                        try {
+                            const stream = fs.createReadStream(filename);
+                            const asset = await this.assetService.createFromFileStream(stream);
+                            this.assetMap.set(assetPath, asset);
+                            assets.push(asset);
+                        } catch (err) {
+                            errors.push(err.toString());
+                        }
+                    }
+                } else {
+                    errors.push(`File "${filename}" does not exist`);
+                }
+            }
+        }
+        return { assets, errors };
+    }
+}

packages/core/src/data-import/providers/importer/importer.ts

@@ -1,23 +1,20 @@
 import { Injectable } from '@nestjs/common';
 import { ImportInfo, LanguageCode } from '@vendure/common/lib/generated-types';
 import { normalizeString } from '@vendure/common/lib/normalize-string';
-import fs from 'fs-extra';
-import path from 'path';
 import ProgressBar from 'progress';
 import { Observable } from 'rxjs';
 import { Stream } from 'stream';
 
 import { RequestContext } from '../../../api/common/request-context';
 import { ConfigService } from '../../../config/config.service';
-import { Asset } from '../../../entity/asset/asset.entity';
 import { FacetValue } from '../../../entity/facet-value/facet-value.entity';
 import { Facet } from '../../../entity/facet/facet.entity';
 import { TaxCategory } from '../../../entity/tax-category/tax-category.entity';
-import { AssetService } from '../../../service/services/asset.service';
 import { ChannelService } from '../../../service/services/channel.service';
 import { FacetValueService } from '../../../service/services/facet-value.service';
 import { FacetService } from '../../../service/services/facet.service';
 import { TaxCategoryService } from '../../../service/services/tax-category.service';
+import { AssetImporter } from '../asset-importer/asset-importer';
 import {
     ImportParser,
     ParsedProductVariant,
@@ -36,7 +33,6 @@ export class Importer {
     private taxCategoryMatches: { [name: string]: string } = {};
     // These Maps are used to cache newly-created entities and prevent duplicates
     // from being created.
-    private assetMap = new Map<string, Asset>();
     private facetMap = new Map<string, Facet>();
     private facetValueMap = new Map<string, FacetValue>();
 
@@ -46,8 +42,8 @@ export class Importer {
         private channelService: ChannelService,
         private facetService: FacetService,
         private facetValueService: FacetValueService,
-        private assetService: AssetService,
         private taxCategoryService: TaxCategoryService,
+        private assetImporter: AssetImporter,
         private fastImporter: FastImporterService,
     ) {}
 
@@ -146,7 +142,7 @@ export class Importer {
         const taxCategories = await this.taxCategoryService.findAll();
         await this.fastImporter.initialize();
         for (const { product, variants } of rows) {
-            const createProductAssets = await this.getAssets(product.assetPaths);
+            const createProductAssets = await this.assetImporter.getAssets(product.assetPaths);
             const productAssets = createProductAssets.assets;
             if (createProductAssets.errors.length) {
                 errors = errors.concat(createProductAssets.errors);
@@ -196,7 +192,7 @@ export class Importer {
             }
 
             for (const variant of variants) {
-                const createVariantAssets = await this.getAssets(variant.assetPaths);
+                const createVariantAssets = await this.assetImporter.getAssets(variant.assetPaths);
                 const variantAssets = createVariantAssets.assets;
                 if (createVariantAssets.errors.length) {
                     errors = errors.concat(createVariantAssets.errors);
@@ -236,42 +232,6 @@ export class Importer {
         return errors;
     }
 
-    /**
-     * Creates Asset entities for the given paths, using the assetMap cache to prevent the
-     * creation of duplicates.
-     */
-    private async getAssets(assetPaths: string[]): Promise<{ assets: Asset[]; errors: string[] }> {
-        const assets: Asset[] = [];
-        const errors: string[] = [];
-        const { importAssetsDir } = this.configService.importExportOptions;
-        const uniqueAssetPaths = new Set(assetPaths);
-        for (const assetPath of uniqueAssetPaths.values()) {
-            const cachedAsset = this.assetMap.get(assetPath);
-            if (cachedAsset) {
-                assets.push(cachedAsset);
-            } else {
-                const filename = path.join(importAssetsDir, assetPath);
-
-                if (fs.existsSync(filename)) {
-                    const fileStat = fs.statSync(filename);
-                    if (fileStat.isFile()) {
-                        try {
-                            const stream = fs.createReadStream(filename);
-                            const asset = await this.assetService.createFromFileStream(stream);
-                            this.assetMap.set(assetPath, asset);
-                            assets.push(asset);
-                        } catch (err) {
-                            errors.push(err.toString());
-                        }
-                    }
-                } else {
-                    errors.push(`File "${filename}" does not exist`);
-                }
-            }
-        }
-        return { assets, errors };
-    }
-
     private async getFacetValueIds(
         facets: ParsedProductVariant['facets'],
         languageCode: LanguageCode,

packages/core/src/data-import/providers/populator/populator.ts

@@ -1,45 +1,20 @@
 import { Injectable } from '@nestjs/common';
-import { LanguageCode } from '@vendure/common/lib/generated-types';
+import { ConfigurableOperationInput } from '@vendure/common/lib/generated-types';
 import { normalizeString } from '@vendure/common/lib/normalize-string';
-import { ID } from '@vendure/common/lib/shared-types';
 import { notNullOrUndefined } from '@vendure/common/lib/shared-utils';
 
 import { RequestContext } from '../../../api/common/request-context';
 import { defaultShippingCalculator, defaultShippingEligibilityChecker } from '../../../config';
-import { facetValueCollectionFilter } from '../../../config/collection/default-collection-filters';
-import { Channel, Collection, TaxCategory } from '../../../entity';
-import { Zone } from '../../../entity/zone/zone.entity';
-import { AssetService, CollectionService, FacetValueService, ShippingMethodService } from '../../../service';
+import { Channel, Collection, FacetValue, TaxCategory } from '../../../entity';
+import { CollectionService, FacetValueService, ShippingMethodService } from '../../../service';
 import { ChannelService } from '../../../service/services/channel.service';
 import { CountryService } from '../../../service/services/country.service';
 import { SearchService } from '../../../service/services/search.service';
 import { TaxCategoryService } from '../../../service/services/tax-category.service';
 import { TaxRateService } from '../../../service/services/tax-rate.service';
 import { ZoneService } from '../../../service/services/zone.service';
-
-type ZoneMap = Map<string, { entity: Zone; members: string[] }>;
-
-export interface CountryData {
-    code: string;
-    name: string;
-    zone: string;
-}
-
-export interface CollectionData {
-    name: string;
-    facetNames: string[];
-    parentName?: string;
-    featuredAssetFilename?: string;
-}
-
-export interface InitialData {
-    defaultLanguage: LanguageCode;
-    defaultZone: string;
-    countries: CountryData[];
-    taxRates: Array<{ name: string; percentage: number }>;
-    shippingMethods: Array<{ name: string; price: number }>;
-    collections: CollectionData[];
-}
+import { CollectionFilterDefinition, CountryDefinition, InitialData, ZoneMap } from '../../types';
+import { AssetImporter } from '../asset-importer/asset-importer';
 
 /**
  * Responsible for populating the database with initial data.
@@ -47,7 +22,6 @@ export interface InitialData {
 @Injectable()
 export class Populator {
     constructor(
-        private assetService: AssetService,
         private countryService: CountryService,
         private zoneService: ZoneService,
         private channelService: ChannelService,
@@ -57,6 +31,7 @@ export class Populator {
         private collectionService: CollectionService,
         private facetValueService: FacetValueService,
         private searchService: SearchService,
+        private assetImporter: AssetImporter,
     ) {}
 
     /**
@@ -82,52 +57,25 @@ export class Populator {
         const allFacetValues = await this.facetValueService.findAll(ctx.languageCode);
         const collectionMap = new Map<string, Collection>();
         for (const collectionDef of data.collections) {
-            const facetValueIds = collectionDef.facetNames
-                .map(name => allFacetValues.find(fv => fv.name === name))
-                .filter(notNullOrUndefined)
-                .map(fv => fv.id);
             const parent = collectionDef.parentName && collectionMap.get(collectionDef.parentName);
             const parentId = parent ? parent.id.toString() : undefined;
-
-            let featuredAssetId: string | null = null;
-            if (collectionDef.featuredAssetFilename) {
-                const asset = await this.assetService.findByFileName(
-                    collectionDef.featuredAssetFilename,
-                    false,
-                );
-                if (asset) {
-                    featuredAssetId = asset.id as string;
-                }
-            }
+            const { assets } = await this.assetImporter.getAssets(collectionDef.assetPaths || []);
 
             const collection = await this.collectionService.create(ctx, {
                 translations: [
                     {
                         languageCode: ctx.languageCode,
                         name: collectionDef.name,
-                        description: '',
+                        description: collectionDef.description || '',
                     },
                 ],
+                isPrivate: collectionDef.private || false,
                 parentId,
-                assetIds: [featuredAssetId].filter(notNullOrUndefined),
-                featuredAssetId,
-                filters: [
-                    {
-                        code: facetValueCollectionFilter.code,
-                        arguments: [
-                            {
-                                name: 'facetValueIds',
-                                type: 'facetValueIds',
-                                value: JSON.stringify(facetValueIds),
-                            },
-                            {
-                                name: 'containsAny',
-                                value: `false`,
-                                type: 'boolean',
-                            },
-                        ],
-                    },
-                ],
+                assetIds: assets.map(a => a.id.toString()),
+                featuredAssetId: assets.length ? assets[0].id.toString() : undefined,
+                filters: (collectionDef.filters || []).map(filter =>
+                    this.processFilterDefinition(filter, allFacetValues),
+                ),
             });
             collectionMap.set(collectionDef.name, collection);
         }
@@ -137,6 +85,36 @@ export class Populator {
         await this.searchService.reindex(ctx);
     }
 
+    private processFilterDefinition(
+        filter: CollectionFilterDefinition,
+        allFacetValues: FacetValue[],
+    ): ConfigurableOperationInput {
+        switch (filter.code) {
+            case 'facet-value-filter':
+                const facetValueIds = filter.args.facetValueNames
+                    .map(name => allFacetValues.find(fv => fv.name === name))
+                    .filter(notNullOrUndefined)
+                    .map(fv => fv.id);
+                return {
+                    code: filter.code,
+                    arguments: [
+                        {
+                            name: 'facetValueIds',
+                            type: 'facetValueIds',
+                            value: JSON.stringify(facetValueIds),
+                        },
+                        {
+                            name: 'containsAny',
+                            value: filter.args.containsAny.toString(),
+                            type: 'boolean',
+                        },
+                    ],
+                };
+            default:
+                throw new Error(`Filter with code "${filter.code}" is not recognized.`);
+        }
+    }
+
     private async createRequestContext(data: InitialData) {
         const channel = await this.channelService.getDefaultChannel();
         const ctx = new RequestContext({
@@ -164,7 +142,7 @@ export class Populator {
         });
     }
 
-    private async populateCountries(ctx: RequestContext, countries: CountryData[]): Promise<ZoneMap> {
+    private async populateCountries(ctx: RequestContext, countries: CountryDefinition[]): Promise<ZoneMap> {
         const zones: ZoneMap = new Map();
         for (const { name, code, zone } of countries) {
             const countryEntity = await this.countryService.create(ctx, {

packages/core/src/data-import/types.ts

@@ -0,0 +1,39 @@
+import { LanguageCode } from '@vendure/common/lib/generated-types';
+
+import { Zone } from '../entity/zone/zone.entity';
+
+export type ZoneMap = Map<string, { entity: Zone; members: string[] }>;
+
+export interface CountryDefinition {
+    code: string;
+    name: string;
+    zone: string;
+}
+
+export interface FacetValueCollectionFilterDefinition {
+    code: 'facet-value-filter';
+    args: {
+        facetValueNames: string[];
+        containsAny: boolean;
+    };
+}
+
+export type CollectionFilterDefinition = FacetValueCollectionFilterDefinition;
+
+export interface CollectionDefinition {
+    name: string;
+    description?: string;
+    private?: boolean;
+    filters?: CollectionFilterDefinition[];
+    parentName?: string;
+    assetPaths?: string[];
+}
+
+export interface InitialData {
+    defaultLanguage: LanguageCode;
+    defaultZone: string;
+    countries: CountryDefinition[];
+    taxRates: Array<{ name: string; percentage: number }>;
+    shippingMethods: Array<{ name: string; price: number }>;
+    collections: CollectionDefinition[];
+}

packages/dev-server/populate-dev-server.ts

@@ -5,6 +5,7 @@ import { populate } from '@vendure/core/cli/populate';
 import path from 'path';
 
 import { clearAllTables } from '../core/mock-data/clear-all-tables';
+import { initialData } from '../core/mock-data/data-sources/initial-data';
 import { populateCustomers } from '../core/mock-data/populate-customers';
 
 import { devConfig } from './dev-config';
@@ -17,7 +18,7 @@ import { devConfig } from './dev-config';
 if (require.main === module) {
     // Running from command line
     const populateConfig: VendureConfig = {
-        ...devConfig as any,
+        ...(devConfig as any),
         authOptions: {
             tokenMethod: 'bearer',
             requireVerification: false,
@@ -31,10 +32,13 @@ if (require.main === module) {
         customFields: {},
     };
     clearAllTables(populateConfig, true)
-        .then(() => populate(() => bootstrap(populateConfig),
-            path.join(__dirname, '../create/assets/initial-data.json'),
-            path.join(__dirname, '../create/assets/products.csv'),
-        ))
+        .then(() =>
+            populate(
+                () => bootstrap(populateConfig),
+                initialData,
+                path.join(__dirname, '../create/assets/products.csv'),
+            ),
+        )
         .then(async app => {
             console.log('populating customers...');
             await populateCustomers(10, populateConfig as any, true);