perf(asset-server-plugin): Implement hashed directory naming for assets

Relates to #258. This strategy for storing files limits the total number of files in a given directory, which otherwise can impact performance when the size grows too large.

packages/asset-server-plugin/src/constants.ts

@@ -0,0 +1 @@
+export const loggerCtx = 'AssetServerPlugin';

packages/asset-server-plugin/src/hashed-asset-naming-strategy.ts

@@ -0,0 +1,36 @@
+import { DefaultAssetNamingStrategy } from '@vendure/core';
+import { createHash } from 'crypto';
+import path from 'path';
+
+/**
+ * @description
+ * An extension of the {@link DefaultAssetNamingStrategy} which prefixes file names with
+ * the type (`'source'` or `'preview'`) as well as a 2-character sub-directory based on
+ * the md5 hash of the original file name.
+ *
+ * This is an implementation of the technique knows as "hashed directory" file storage,
+ * and the purpose is to reduce the number of files in a single directory, since a very large
+ * number of files can lead to performance issues when reading and writing to that directory.
+ *
+ * With this strategory, even with 200,000 total assets stored, each directory would
+ * only contain less than 800 files.
+ *
+ * @docsCategory AssetServerPlugin
+ */
+export class HashedAssetNamingStrategy extends DefaultAssetNamingStrategy {
+    generateSourceFileName(originalFileName: string, conflictFileName?: string): string {
+        const filename = super.generateSourceFileName(originalFileName, conflictFileName);
+        return path.join('source', this.getHashedDir(filename), filename);
+    }
+    generatePreviewFileName(originalFileName: string, conflictFileName?: string): string {
+        const filename = super.generatePreviewFileName(originalFileName, conflictFileName);
+        return path.join('preview', this.getHashedDir(filename), filename);
+    }
+
+    private getHashedDir(filename: string): string {
+        return createHash('md5')
+            .update(filename)
+            .digest('hex')
+            .slice(0, 2);
+    }
+}

packages/asset-server-plugin/src/local-asset-storage-strategy.ts

@@ -1,4 +1,5 @@
 import { AssetStorageStrategy } from '@vendure/core';
+import { createHash } from 'crypto';
 import { Request } from 'express';
 import { ReadStream } from 'fs';
 import fs from 'fs-extra';
@@ -6,7 +7,10 @@ import path from 'path';
 import { Stream } from 'stream';
 
 /**
+ * @description
  * A persistence strategy which saves files to the local file system.
+ *
+ * @docsCategory AssetServerPlugin
  */
 export class LocalAssetStorageStrategy implements AssetStorageStrategy {
     toAbsoluteUrl: ((reqest: Request, identifier: string) => string) | undefined;
@@ -15,14 +19,15 @@ export class LocalAssetStorageStrategy implements AssetStorageStrategy {
         private readonly uploadPath: string,
         private readonly toAbsoluteUrlFn?: (reqest: Request, identifier: string) => string,
     ) {
-        this.ensureUploadPathExists(this.uploadPath);
+        fs.ensureDirSync(this.uploadPath);
         if (toAbsoluteUrlFn) {
             this.toAbsoluteUrl = toAbsoluteUrlFn;
         }
     }
 
-    writeFileFromStream(fileName: string, data: ReadStream): Promise<string> {
+    async writeFileFromStream(fileName: string, data: ReadStream): Promise<string> {
         const filePath = path.join(this.uploadPath, fileName);
+        await fs.ensureDir(path.dirname(filePath));
         const writeStream = fs.createWriteStream(filePath, 'binary');
         return new Promise<string>((resolve, reject) => {
             data.pipe(writeStream);
@@ -33,6 +38,7 @@ export class LocalAssetStorageStrategy implements AssetStorageStrategy {
 
     async writeFileFromBuffer(fileName: string, data: Buffer): Promise<string> {
         const filePath = path.join(this.uploadPath, fileName);
+        await fs.ensureDir(path.dirname(filePath));
         await fs.writeFile(filePath, data, 'binary');
         return this.filePathToIdentifier(filePath);
     }
@@ -54,21 +60,14 @@ export class LocalAssetStorageStrategy implements AssetStorageStrategy {
         return Promise.resolve(readStream);
     }
 
-    private ensureUploadPathExists(uploadDir: string): void {
-        if (!fs.existsSync(this.uploadPath)) {
-            fs.mkdirSync(this.uploadPath);
-        }
-        const cachePath = path.join(this.uploadPath, 'cache');
-        if (!fs.existsSync(cachePath)) {
-            fs.mkdirSync(cachePath);
-        }
-    }
-
     private filePathToIdentifier(filePath: string): string {
-        return `${path.basename(filePath)}`;
+        const filePathDirname = path.dirname(filePath);
+        const deltaDirname = filePathDirname.replace(this.uploadPath, '');
+        const identifier = path.join(deltaDirname, path.basename(filePath));
+        return identifier.replace(/^[\\/]+/, '');
     }
 
     private identifierToFilePath(identifier: string): string {
-        return path.join(this.uploadPath, path.basename(identifier));
+        return path.join(this.uploadPath, identifier);
     }
 }

packages/asset-server-plugin/src/plugin.ts

@@ -8,11 +8,15 @@ import {
     RuntimeVendureConfig,
     VendurePlugin,
 } from '@vendure/core';
+import { createHash } from 'crypto';
 import express, { NextFunction, Request, Response } from 'express';
+import fs from 'fs-extra';
 import { Server } from 'http';
 import path from 'path';
 
+import { loggerCtx } from './constants';
 import { defaultAssetStorageStrategyFactory } from './default-asset-storage-strategy-factory';
+import { HashedAssetNamingStrategy } from './hashed-asset-naming-strategy';
 import { SharpAssetPreviewStrategy } from './sharp-asset-preview-strategy';
 import { transformImage } from './transform-image';
 import { AssetServerOptions, ImageTransformPreset } from './types';
@@ -145,6 +149,7 @@ export class AssetServerPlugin implements OnVendureBootstrap, OnVendureClose {
             maxHeight: this.options.previewMaxHeight || 1600,
         });
         config.assetOptions.assetStorageStrategy = this.assetStorage;
+        config.assetOptions.assetNamingStrategy = new HashedAssetNamingStrategy();
         config.middleware.push({
             handler: createProxyHandler({ ...this.options, label: 'Asset Server' }),
             route: this.options.route,
@@ -164,6 +169,9 @@ export class AssetServerPlugin implements OnVendureBootstrap, OnVendureClose {
                 }
             }
         }
+
+        const cachePath = path.join(AssetServerPlugin.options.assetUploadDir, this.cacheDir);
+        fs.ensureDirSync(cachePath);
         this.createAssetServer();
     }
 
@@ -205,6 +213,7 @@ export class AssetServerPlugin implements OnVendureBootstrap, OnVendureClose {
         return async (err: any, req: Request, res: Response, next: NextFunction) => {
             if (err && err.status === 404) {
                 if (req.query) {
+                    Logger.debug(`Pre-cached Asset not found: ${req.path}`, loggerCtx);
                     let file: Buffer;
                     try {
                         file = await AssetServerPlugin.assetStorage.readFileToBuffer(req.path);
@@ -217,6 +226,7 @@ export class AssetServerPlugin implements OnVendureBootstrap, OnVendureClose {
                         const imageBuffer = await image.toBuffer();
                         const cachedFileName = this.getFileNameFromRequest(req);
                         await AssetServerPlugin.assetStorage.writeFileFromBuffer(cachedFileName, imageBuffer);
+                        Logger.debug(`Saved cached asset: ${cachedFileName}`, loggerCtx);
                         res.set('Content-Type', `image/${(await image.metadata()).format}`);
                         res.send(imageBuffer);
                     } catch (e) {
@@ -232,28 +242,34 @@ export class AssetServerPlugin implements OnVendureBootstrap, OnVendureClose {
     private getFileNameFromRequest(req: Request): string {
         const { w, h, mode, preset, fpx, fpy } = req.query;
         const focalPoint = fpx && fpy ? `_fpx${fpx}_fpy${fpy}` : '';
+        let imageParamHash: string | null = null;
         if (w || h) {
             const width = w || '';
             const height = h || '';
-            // TODO: make sure no bug / attack vector with the naming of files
-            return (
-                this.cacheDir +
-                '/' +
-                this.addSuffix(req.path, `_transform_w${width}_h${height}_m${mode}${focalPoint}`)
-            );
+            imageParamHash = this.md5(`_transform_w${width}_h${height}_m${mode}${focalPoint}`);
         } else if (preset) {
             if (this.presets && !!this.presets.find(p => p.name === preset)) {
-                return (
-                    this.cacheDir + '/' + this.addSuffix(req.path, `_transform_pre_${preset}${focalPoint}`)
-                );
+                imageParamHash = this.md5(`_transform_pre_${preset}${focalPoint}`);
             }
         }
-        return req.path;
+
+        if (imageParamHash) {
+            return path.join(this.cacheDir, this.addSuffix(req.path, imageParamHash));
+        } else {
+            return req.path;
+        }
+    }
+
+    private md5(input: string): string {
+        return createHash('md5')
+            .update(input)
+            .digest('hex');
     }
 
     private addSuffix(fileName: string, suffix: string): string {
         const ext = path.extname(fileName);
         const baseName = path.basename(fileName, ext);
-        return `${baseName}${suffix}${ext}`;
+        const dirName = path.dirname(fileName);
+        return path.join(dirName, `${baseName}${suffix}${ext}`);
     }
 }

packages/core/src/config/asset-naming-strategy/default-asset-naming-strategy.ts

@@ -1,12 +1,14 @@
-import path from 'path';
-
 import { normalizeString } from '@vendure/common/lib/normalize-string';
+import path from 'path';
 
 import { AssetNamingStrategy } from './asset-naming-strategy';
 
 /**
+ * @description
  * The default strategy normalizes the file names to remove unwanted characters and
  * in the case of conflicts, increments a counter suffix.
+ *
+ * @docsCategory assets
  */
 export class DefaultAssetNamingStrategy implements AssetNamingStrategy {
     private readonly numberingRe = /__(\d+)\.[^.]+$/;

packages/core/src/service/services/asset.service.ts

@@ -192,7 +192,7 @@ export class AssetService {
             type,
             width,
             height,
-            name: sourceFileName,
+            name: path.basename(sourceFileName),
             fileSize: sourceFile.byteLength,
             mimeType: mimetype,
             source: sourceFileIdentifier,

packages/dev-server/dev-config.ts

@@ -42,7 +42,7 @@ export const devConfig: VendureConfig = {
             { name: 'markup', type: 'float', internal: true },
         ],*/
     },
-    logger: new DefaultLogger({ level: LogLevel.Info }),
+    logger: new DefaultLogger({ level: LogLevel.Debug }),
     importExportOptions: {
         importAssetsDir: path.join(__dirname, 'import-assets'),
     },