feat(asset-server-plugin): Update s3 asset storage strategy to use AWS sdk v3 (#2102)

v2 is deprecated and v3 is re-architected to be modular.

BREAKING CHANGE: If you are using the s3 storage strategy of the AssetServerPlugin, it has been updated to use v3 of the AWS SDKs. This update introduces [an improved modular architecture to the AWS sdk](https://aws.amazon.com/blogs/developer/modular-packages-in-aws-sdk-for-javascript/), resulting in smaller bundle sizes. You need to install the `@aws-sdk/client-s3` & `@aws-sdk/lib-storage` packages, and can remove the `aws-sdk` package.

packages/asset-server-plugin/package.json

@@ -21,6 +21,8 @@
         "access": "public"
     },
     "devDependencies": {
+        "@aws-sdk/client-s3": "^3.312.0",
+        "@aws-sdk/lib-storage": "^3.312.0",
         "@types/express": "^4.17.8",
         "@types/fs-extra": "^9.0.8",
         "@types/node-fetch": "^2.5.8",

packages/asset-server-plugin/src/s3-asset-storage-strategy.ts

@@ -1,21 +1,14 @@
+import { PutObjectRequest, S3ClientConfig } from '@aws-sdk/client-s3';
+import { AwsCredentialIdentity, AwsCredentialIdentityProvider } from '@aws-sdk/types';
 import { AssetStorageStrategy, Logger } from '@vendure/core';
 import { Request } from 'express';
-import * as path from 'path';
-import { Readable, Stream } from 'stream';
+import * as path from 'node:path';
+import { Readable } from 'node:stream';
 
 import { getAssetUrlPrefixFn } from './common';
 import { loggerCtx } from './constants';
 import { AssetServerOptions } from './types';
 
-export type S3Credentials = {
-    accessKeyId: string;
-    secretAccessKey: string;
-};
-
-export type S3CredentialsProfile = {
-    profile: string;
-};
-
 /**
  * @description
  * Configuration for connecting to AWS S3.
@@ -26,12 +19,12 @@ export type S3CredentialsProfile = {
 export interface S3Config {
     /**
      * @description
-     * The credentials used to access your s3 account. You can supply either the access key ID & secret,
-     * or you can make use of a
+     * The credentials used to access your s3 account. You can supply either the access key ID & secret, or you can make use of a
      * [shared credentials file](https://docs.aws.amazon.com/sdk-for-javascript/v2/developer-guide/loading-node-credentials-shared.html)
-     * and supply the profile name (e.g. `'default'`).
+     * To use a shared credentials file, import the `fromIni()` function from the "@aws-sdk/credential-provider-ini" or "@aws-sdk/credential-providers" package and supply
+     * the profile name (e.g. `{ profile: 'default' }`) as its argument.
      */
-    credentials?: S3Credentials | S3CredentialsProfile;
+    credentials: AwsCredentialIdentity | AwsCredentialIdentityProvider;
     /**
      * @description
      * The S3 bucket in which to store the assets. If it does not exist, it will be created on startup.
@@ -58,16 +51,17 @@ export interface S3Config {
  * Returns a configured instance of the {@link S3AssetStorageStrategy} which can then be passed to the {@link AssetServerOptions}
  * `storageStrategyFactory` property.
  *
- * Before using this strategy, make sure you have the `aws-sdk` package installed:
+ * Before using this strategy, make sure you have the `@aws-sdk/client-s3` and `@aws-sdk/lib-storage` package installed:
  *
  * ```sh
- * npm install aws-sdk
+ * npm install @aws-sdk/client-s3 @aws-sdk/lib-storage
  * ```
  *
  * @example
  * ```TypeScript
  * import { AssetServerPlugin, configureS3AssetStorage } from '\@vendure/asset-server-plugin';
  * import { DefaultAssetNamingStrategy } from '\@vendure/core';
+ * import { fromEnv } from '\@aws-sdk/credential-providers';
  *
  * // ...
  *
@@ -78,10 +72,7 @@ export interface S3Config {
  *     namingStrategy: new DefaultAssetNamingStrategy(),
  *     storageStrategyFactory: configureS3AssetStorage({
  *       bucket: 'my-s3-bucket',
- *       credentials: {
- *         accessKeyId: process.env.AWS_ACCESS_KEY_ID,
- *         secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
- *       },
+ *       credentials: fromEnv(), // or any other credential provider
  *     }),
  * }),
  * ```
@@ -121,7 +112,6 @@ export interface S3Config {
  */
 export function configureS3AssetStorage(s3Config: S3Config) {
     return (options: AssetServerOptions) => {
-        const { assetUrlPrefix, route } = options;
         const prefixFn = getAssetUrlPrefixFn(options);
         const toAbsoluteUrlFn = (request: Request, identifier: string): string => {
             if (!identifier) {
@@ -138,12 +128,12 @@ export function configureS3AssetStorage(s3Config: S3Config) {
  * @description
  * An {@link AssetStorageStrategy} which uses [Amazon S3](https://aws.amazon.com/s3/) object storage service.
  * To us this strategy you must first have access to an AWS account.
- * See their [getting started guide](https://aws.amazon.com/s3/getting-started/?nc=sn&loc=5) for how to get set up.
+ * See their [getting started guide](https://aws.amazon.com/s3/getting-started/) for how to get set up.
  *
- * Before using this strategy, make sure you have the `aws-sdk` package installed:
+ * Before using this strategy, make sure you have the `@aws-sdk/client-s3` and `@aws-sdk/lib-storage` package installed:
  *
  * ```sh
- * npm install aws-sdk
+ * npm install @aws-sdk/client-s3 @aws-sdk/lib-storage
  * ```
  *
  * **Note:** Rather than instantiating this manually, use the {@link configureS3AssetStorage} function.
@@ -157,113 +147,113 @@ export function configureS3AssetStorage(s3Config: S3Config) {
  * @docsWeight 0
  */
 export class S3AssetStorageStrategy implements AssetStorageStrategy {
-    private AWS: typeof import('aws-sdk');
-    private s3: import('aws-sdk').S3;
-    constructor(
-        private s3Config: S3Config,
-        public readonly toAbsoluteUrl: (request: Request, identifier: string) => string,
-    ) {}
+    private AWS: typeof import('@aws-sdk/client-s3');
+    private libStorage: typeof import('@aws-sdk/lib-storage');
+    private s3Client: import('@aws-sdk/client-s3').S3Client;
+
+    constructor(private s3Config: S3Config, public readonly toAbsoluteUrl: (request: Request, identifier: string) => string) {}
 
     async init() {
         try {
-            this.AWS = await import('aws-sdk');
-        } catch (e: any) {
-            Logger.error(
-                'Could not find the "aws-sdk" package. Make sure it is installed',
-                loggerCtx,
-                e.stack,
-            );
+            this.AWS = await import('@aws-sdk/client-s3');
+        } catch (err: any) {
+            Logger.error('Could not find the "@aws-sdk/client-s3" package. Make sure it is installed', loggerCtx, err.stack);
+        }
+
+        try {
+            this.libStorage = await import('@aws-sdk/lib-storage');
+        } catch (err: any) {
+            Logger.error('Could not find the "@aws-sdk/lib-storage" package. Make sure it is installed', loggerCtx, err.stack);
         }
 
         const config = {
-            credentials: this.getS3Credentials(),
             ...this.s3Config.nativeS3Configuration,
-        };
-        this.s3 = new this.AWS.S3(config);
-        await this.ensureBucket(this.s3Config.bucket);
+            credentials: await this.getCredentials() // Avoid credentials overriden by nativeS3Configuration
+        } satisfies S3ClientConfig
+
+        this.s3Client = new this.AWS.S3Client(config);
+
+        await this.ensureBucket();
     }
 
     destroy?: (() => void | Promise<void>) | undefined;
 
-    async writeFileFromBuffer(fileName: string, data: Buffer): Promise<string> {
-        const result = await this.s3
-            .upload(
-                {
-                    Bucket: this.s3Config.bucket,
-                    Key: fileName,
-                    Body: data,
-                },
-                this.s3Config.nativeS3UploadConfiguration,
-            )
-            .promise();
-        return result.Key;
+    async writeFileFromBuffer(fileName: string, data: Buffer) {
+        return this.writeFile(fileName, data);
     }
 
-    async writeFileFromStream(fileName: string, data: Stream): Promise<string> {
-        const result = await this.s3
-            .upload(
-                {
-                    Bucket: this.s3Config.bucket,
-                    Key: fileName,
-                    Body: data,
-                },
-                this.s3Config.nativeS3UploadConfiguration,
-            )
-            .promise();
-        return result.Key;
+    async writeFileFromStream(fileName: string, data: Readable) {
+        return this.writeFile(fileName, data);
     }
 
-    async readFileToBuffer(identifier: string): Promise<Buffer> {
-        const result = await this.s3.getObject(this.getObjectParams(identifier)).promise();
-        const body = result.Body;
+    async readFileToBuffer(identifier: string) {
+        const body = await this.readFile(identifier);
+
         if (!body) {
             Logger.error(`Got undefined Body for ${identifier}`, loggerCtx);
             return Buffer.from('');
         }
-        if (body instanceof Buffer) {
-            return body;
-        }
-        if (body instanceof Uint8Array || typeof body === 'string') {
-            return Buffer.from(body);
-        }
-        if (body instanceof Readable) {
-            return new Promise((resolve, reject) => {
-                const buf: any[] = [];
-                body.on('data', data => buf.push(data));
-                body.on('error', err => reject(err));
-                body.on('end', () => {
-                    const intArray = Uint8Array.from(buf);
-                    resolve(Buffer.concat([intArray]));
-                });
-            });
+
+        const chunks: Buffer[] = [];
+        for await (const chunk of body) {
+            chunks.push(chunk);
         }
-        return Buffer.from(body as any);
+
+        return Buffer.concat(chunks);
     }
 
-    async readFileToStream(identifier: string): Promise<Stream> {
-        const result = await this.s3.getObject(this.getObjectParams(identifier)).promise();
-        const body = result.Body;
-        if (!(body instanceof Stream)) {
-            const readable = new Readable();
-            readable._read = () => {
-                /* noop */
-            };
-            readable.push(body);
-            readable.push(null);
-            return readable;
+    async readFileToStream(identifier: string) {
+        const body = await this.readFile(identifier);
+
+        if (!body) {
+            return new Readable({ read() { this.push(null); } });
         }
+
         return body;
     }
 
-    async deleteFile(identifier: string): Promise<void> {
-        await this.s3.deleteObject(this.getObjectParams(identifier)).promise();
+    private async readFile(identifier: string) {
+        const { GetObjectCommand } = this.AWS;
+
+        const result = await this.s3Client.send(new GetObjectCommand(this.getObjectParams(identifier)));
+        return result.Body as Readable | undefined;
     }
 
-    async fileExists(fileName: string): Promise<boolean> {
+    private async writeFile(fileName: string, data: PutObjectRequest['Body'] | string | Uint8Array | Buffer) {
+        const { Upload } = this.libStorage
+
+        const upload = new Upload({
+            client: this.s3Client,
+            params: {
+                ...this.s3Config.nativeS3UploadConfiguration,
+                Bucket: this.s3Config.bucket,
+                Key: fileName,
+                Body: data,
+            },
+        });
+
+        return upload.done().then((result) => {
+            if (!('Key' in result) || !result.Key) {
+                Logger.error(`Got undefined Key for ${fileName}`, loggerCtx);
+                throw new Error(`Got undefined Key for ${fileName}`);
+            }
+
+            return result.Key;
+        });
+    }
+
+    async deleteFile(identifier: string) {
+        const { DeleteObjectCommand } = this.AWS
+        await this.s3Client.send(new DeleteObjectCommand(this.getObjectParams(identifier)));
+    }
+
+    async fileExists(fileName: string) {
+        const { HeadObjectCommand } = this.AWS
+
         try {
-            await this.s3.headObject(this.getObjectParams(fileName)).promise();
+            await this.s3Client.send(new HeadObjectCommand(this.getObjectParams(fileName)));
             return true;
-        } catch (e: any) {
+        } catch (err: any) {
             return false;
         }
     }
@@ -275,44 +265,43 @@ export class S3AssetStorageStrategy implements AssetStorageStrategy {
         };
     }
 
-    private getS3Credentials() {
-        const { credentials } = this.s3Config;
-        if (credentials == null) {
-            return null;
-        } else if (this.isCredentialsProfile(credentials)) {
-            return new this.AWS.SharedIniFileCredentials(credentials);
-        }
-        return new this.AWS.Credentials(credentials);
-    }
+    private async ensureBucket(bucket = this.s3Config.bucket) {
+        const { HeadBucketCommand, CreateBucketCommand } = this.AWS
 
-    private async ensureBucket(bucket: string) {
-        let bucketExists = false;
         try {
-            await this.s3.headBucket({ Bucket: this.s3Config.bucket }).promise();
-            bucketExists = true;
+            await this.s3Client.send(new HeadBucketCommand({ Bucket: bucket }));
             Logger.verbose(`Found S3 bucket "${bucket}"`, loggerCtx);
-        } catch (e: any) {
-            Logger.verbose(
-                `Could not find bucket "${bucket}: ${JSON.stringify(e.message)}". Attempting to create...`,
-            );
+            return
+        } catch (err: any) {
+            Logger.verbose(`Could not find bucket "${bucket}: ${JSON.stringify(err.message)}". Attempting to create...`);
         }
-        if (!bucketExists) {
-            try {
-                await this.s3.createBucket({ Bucket: bucket, ACL: 'private' }).promise();
-                Logger.verbose(`Created S3 bucket "${bucket}"`, loggerCtx);
-            } catch (e: any) {
-                Logger.error(
-                    `Could not find nor create the S3 bucket "${bucket}: ${JSON.stringify(e.message)}"`,
-                    loggerCtx,
-                    e.stack,
-                );
-            }
+
+        try {
+            await this.s3Client.send(new CreateBucketCommand({Bucket: bucket, ACL: 'private'}));
+            Logger.verbose(`Created S3 bucket "${bucket}"`, loggerCtx);
+        } catch (err: any) {
+            Logger.error(`Could not find nor create the S3 bucket "${bucket}: ${JSON.stringify(err.message)}"`, loggerCtx, err.stack);
+        }
+    }
+
+    private async getCredentials() {
+        if (this.s3Config.credentials == null) {
+            return undefined;
         }
+
+        if (this.isCredentialsProfile(this.s3Config.credentials)) {
+            Logger.warn(
+                'The "profile" property of the "s3Config.credentials" is deprecated. ' +
+                'Please use the "fromIni()" function from the "@aws-sdk/credential-provider-ini" or "@aws-sdk/credential-providers" package instead.',
+                loggerCtx
+            );
+            return (await import('@aws-sdk/credential-provider-ini')).fromIni({ profile: this.s3Config.credentials.profile  });
+        }
+
+        return this.s3Config.credentials
     }
 
-    private isCredentialsProfile(
-        credentials: S3Credentials | S3CredentialsProfile,
-    ): credentials is S3CredentialsProfile {
-        return credentials.hasOwnProperty('profile');
+    private isCredentialsProfile(credentials: AwsCredentialIdentity | AwsCredentialIdentityProvider,): credentials is AwsCredentialIdentity & { profile: string } {
+        return credentials !== null && typeof credentials === 'object' && 'profile' in credentials && Object.keys(credentials).length === 1;
     }
 }