chore: Set up benchmarks for #1506

packages/dev-server/load-testing/benchmarks.ts

@@ -0,0 +1,188 @@
+/* tslint:disable:no-console */
+import { INestApplication } from '@nestjs/common';
+import { GlobalFlag } from '@vendure/common/lib/generated-types';
+import {
+    bootstrap,
+    Importer,
+    isGraphQlErrorResult,
+    LanguageCode,
+    OrderService,
+    RequestContextService,
+} from '@vendure/core';
+import { populate } from '@vendure/core/cli/populate';
+import { ParsedProductWithVariants } from '@vendure/core/src/index';
+import { clearAllTables } from '@vendure/testing';
+import { spawn } from 'child_process';
+import program from 'commander';
+import path from 'path';
+import ProgressBar from 'progress';
+
+import { getLoadTestConfig } from './load-test-config';
+
+/**
+ * This set of benchmarks aims to specifically test the performance issues discussed
+ * in issue https://github.com/vendure-ecommerce/vendure/issues/1506.
+ *
+ * In order to test these issues, we need a test dataset that will create:
+ *
+ * 1. 1k Products, each with 10 ProductVariants
+ * 2. 10k Orders, each with 10 OrderLines, each OrderLine with qty 5
+ *
+ * Then we will test:
+ *
+ * 1. Fetching 10 Products from the `products` query. This will test the effects of indexes, ListQueryBuilder & dataloader
+ * 2. As above but with Orders
+ * 3. Fetching a list of orders and selecting only the Order id.
+ *    This will test optimization of selecting & joining only the needed fields.
+ */
+
+const DATABASE_NAME = 'vendure-benchmarks';
+const PRODUCT_COUNT = 1000;
+const VARIANTS_PER_PRODUCT = 10;
+const ORDER_COUNT = 10000;
+const LINES_PER_ORDER = 10;
+const QUANTITY_PER_ORDER_LINE = 5;
+
+interface Options {
+    script?: string;
+    db: 'mysql' | 'postgres';
+    populate?: boolean;
+    variant?: string;
+}
+
+program
+    .option('--script <script>', `Specify the k6 script to run`)
+    .option('--db <db>', 'Select which database to test against', /^(mysql|postgres)$/, 'mysql')
+    .option('--populate', 'Whether to populate the database')
+    .option('--variant <variant>', 'Which variant of the given script')
+    .parse(process.argv);
+
+const opts = program.opts() as any;
+
+runBenchmark(opts).then(() => process.exit(0));
+
+async function runBenchmark(options: Options) {
+    const config = getLoadTestConfig('bearer', DATABASE_NAME);
+    if (options.populate) {
+        console.log(`Populating benchmark database "${DATABASE_NAME}"`);
+        await clearAllTables(config, true);
+        const populateApp = await populate(
+            () => bootstrap(config),
+            path.join(__dirname, '../../create/assets/initial-data.json'),
+        );
+        await createProducts(populateApp);
+        await createOrders(populateApp);
+        await populateApp.close();
+    } else {
+        const app = await bootstrap(config);
+        await new Promise((resolve, reject) => {
+            const runArgs: string[] = ['run', `./scripts/${options.script}`];
+            if (options.variant) {
+                console.log(`Using variant "${options.variant}"`);
+                runArgs.push('-e', `variant=${options.variant}`);
+            }
+            const loadTest = spawn('k6', runArgs, {
+                cwd: __dirname,
+                stdio: 'inherit',
+            });
+            loadTest.on('exit', code => {
+                if (code === 0) {
+                    resolve(code);
+                } else {
+                    reject();
+                }
+            });
+            loadTest.on('error', err => {
+                reject(err);
+            });
+        });
+        await app.close();
+    }
+}
+
+async function createProducts(app: INestApplication) {
+    const importer = app.get(Importer);
+    const ctx = await app.get(RequestContextService).create({
+        apiType: 'admin',
+    });
+
+    const bar = new ProgressBar('creating products [:bar] (:current/:total) :percent :etas', {
+        complete: '=',
+        incomplete: ' ',
+        total: PRODUCT_COUNT,
+        width: 40,
+    });
+
+    const products: ParsedProductWithVariants[] = [];
+    for (let i = 0; i < PRODUCT_COUNT; i++) {
+        const product: ParsedProductWithVariants = {
+            product: {
+                optionGroups: [
+                    {
+                        translations: [
+                            {
+                                languageCode: LanguageCode.en,
+                                name: `prod-${i}-option`,
+                                values: Array.from({ length: VARIANTS_PER_PRODUCT }).map(
+                                    (_, opt) => `prod-${i}-option-${opt}`,
+                                ),
+                            },
+                        ],
+                    },
+                ],
+                translations: [
+                    {
+                        languageCode: LanguageCode.en,
+                        name: `Product ${i}`,
+                        slug: `product-${i}`,
+                        description: '',
+                        customFields: {},
+                    },
+                ],
+                assetPaths: [],
+                facets: [],
+            },
+            variants: Array.from({ length: VARIANTS_PER_PRODUCT }).map((value, index) => ({
+                sku: `PROD-${i}-${index}`,
+                facets: [],
+                assetPaths: [],
+                price: 1000,
+                stockOnHand: 100,
+                taxCategory: 'standard',
+                trackInventory: GlobalFlag.INHERIT,
+                translations: [
+                    {
+                        languageCode: LanguageCode.en,
+                        optionValues: [`prod-${i}-option-${index}`],
+                        customFields: {},
+                    },
+                ],
+            })),
+        };
+        products.push(product);
+    }
+    await importer.importProducts(ctx, products, progess => bar.tick());
+}
+
+async function createOrders(app: INestApplication) {
+    const orderService = app.get(OrderService);
+    const bar = new ProgressBar('creating orders [:bar] (:current/:total) :percent :etas', {
+        complete: '=',
+        incomplete: ' ',
+        total: ORDER_COUNT,
+        width: 40,
+    });
+
+    for (let i = 0; i < ORDER_COUNT; i++) {
+        const ctx = await app.get(RequestContextService).create({
+            apiType: 'shop',
+        });
+        const order = await orderService.create(ctx);
+        const variantId = (i % (PRODUCT_COUNT * VARIANTS_PER_PRODUCT)) + 1;
+        const result = await orderService.addItemToOrder(ctx, order.id, variantId, QUANTITY_PER_ORDER_LINE);
+        if (isGraphQlErrorResult(result)) {
+            console.log(result);
+        }
+        bar.tick();
+    }
+}

packages/dev-server/load-testing/graphql/admin/test.graphql

@@ -1,10 +0,0 @@
-query {
-    activeChannel {
-        code
-    }
-    administrators {
-        items {
-            createdAt
-        }
-    }
-}

packages/dev-server/load-testing/init-load-test.ts

@@ -24,12 +24,13 @@ import {
  */
 if (require.main === module) {
     // Running from command line
-    isDatabasePopulated()
+    const count = getProductCount();
+    const databaseName = `vendure-load-testing-${count}`;
+    isDatabasePopulated(databaseName)
         .then(isPopulated => {
             console.log(`isPopulated:`, isPopulated);
             if (!isPopulated) {
-                const count = getProductCount();
-                const config = getLoadTestConfig('bearer');
+                const config = getLoadTestConfig('bearer', databaseName);
                 const csvFile = getProductCsvFilePath();
                 return clearAllTables(config, true)
                     .then(() => {
@@ -69,13 +70,12 @@ if (require.main === module) {
 /**
  * Tests to see whether the load test database is already populated.
  */
-async function isDatabasePopulated(): Promise<boolean> {
+async function isDatabasePopulated(databaseName: string): Promise<boolean> {
     const isPostgres = process.env.DB === 'postgres';
-    const count = getProductCount();
     if (isPostgres) {
         console.log(`Checking whether data is populated (postgres)`);
         const pg = require('pg');
-        const postgresConnectionOptions = getPostgresConnectionOptions(count);
+        const postgresConnectionOptions = getPostgresConnectionOptions(databaseName);
         const client = new pg.Client({
             host: postgresConnectionOptions.host,
             user: postgresConnectionOptions.username,
@@ -96,7 +96,7 @@ async function isDatabasePopulated(): Promise<boolean> {
     } else {
         const mysql = require('mysql');
 
-        const mysqlConnectionOptions = getMysqlConnectionOptions(count);
+        const mysqlConnectionOptions = getMysqlConnectionOptions(databaseName);
         const connection = mysql.createConnection({
             host: mysqlConnectionOptions.host,
             user: mysqlConnectionOptions.username,
@@ -120,7 +120,7 @@ async function isDatabasePopulated(): Promise<boolean> {
                         reject(err);
                         return;
                     }
-                    resolve(results[0].prodCount === count);
+                    resolve(true);
                 });
             });
         });

packages/dev-server/load-testing/load-test-config.ts

@@ -13,32 +13,34 @@ import {
 } from '@vendure/core';
 import path from 'path';
 
-export function getMysqlConnectionOptions(count: number) {
+export function getMysqlConnectionOptions(databaseName: string) {
     return {
         type: 'mysql' as const,
         host: '127.0.0.1',
         port: 3306,
         username: 'root',
         password: '',
-        database: `vendure-load-testing-${count}`,
+        database: databaseName,
         extra: {
             // connectionLimit: 150,
         },
     };
 }
-export function getPostgresConnectionOptions(count: number) {
+export function getPostgresConnectionOptions(databaseName: string) {
     return {
         type: 'postgres' as const,
         host: '127.0.0.1',
         port: 5432,
         username: 'admin',
         password: 'secret',
-        database: `vendure-load-testing-${count}`,
+        database: databaseName,
     };
 }
 
-export function getLoadTestConfig(tokenMethod: 'cookie' | 'bearer'): Required<VendureConfig> {
-    const count = getProductCount();
+export function getLoadTestConfig(
+    tokenMethod: 'cookie' | 'bearer',
+    databaseName: string,
+): Required<VendureConfig> {
     return mergeConfig(defaultConfig, {
         paymentOptions: {
             paymentMethodHandlers: [dummyPaymentHandler],
@@ -49,8 +51,8 @@ export function getLoadTestConfig(tokenMethod: 'cookie' | 'bearer'): Required<Ve
         logger: new DefaultLogger({ level: LogLevel.Info }),
         dbConnectionOptions:
             process.env.DB === 'postgres'
-                ? getPostgresConnectionOptions(count)
-                : getMysqlConnectionOptions(count),
+                ? getPostgresConnectionOptions(databaseName)
+                : getMysqlConnectionOptions(databaseName),
         authOptions: {
             tokenMethod,
             requireVerification: false,

packages/dev-server/load-testing/run-load-test.ts

@@ -28,7 +28,8 @@ if (require.main === module) {
 
     init.on('exit', code => {
         if (code === 0) {
-            return bootstrap(getLoadTestConfig('cookie'))
+            const databaseName = `vendure-load-testing-${count}`;
+            return bootstrap(getLoadTestConfig('cookie', databaseName))
                 .then(async app => {
                     // await app.get(JobQueueService).start();
                     const summaries: LoadTestSummary[] = [];

packages/dev-server/load-testing/utils/api-request.js

@@ -2,21 +2,30 @@
 import http from 'k6/http';
 import { check, fail } from 'k6';
 
-export class ShopApiRequest {
-    constructor(fileName) {
+const AUTH_TOKEN_HEADER = 'Vendure-Auth-Token';
+
+export class ApiRequest {
+    constructor(apiUrl, fileName) {
         this.document = open('../graphql/' + fileName);
+        this.apiUrl = apiUrl;
+        this.authToken = undefined;
     }
 
     /**
      * Post the GraphQL request
      */
-    post(variables = {}) {
-        const res = http.post('http://localhost:3000/shop-api/', {
-            query: this.document,
-            variables: JSON.stringify(variables),
-        }, {
-            timeout: 120 * 1000,
-        });
+    post(variables = {}, authToken) {
+        const res = http.post(
+            this.apiUrl,
+            {
+                query: this.document,
+                variables: JSON.stringify(variables),
+            },
+            {
+                timeout: 120 * 1000,
+                headers: { Authorization: authToken ? `Bearer ${authToken}` : undefined },
+            },
+        );
         check(res, {
             'Did not error': r => r.json().errors == null && r.status === 200,
         });
@@ -24,6 +33,23 @@ export class ShopApiRequest {
         if (result.errors) {
             fail('Errored: ' + result.errors[0].message);
         }
+        if (res.headers[AUTH_TOKEN_HEADER]) {
+            authToken = res.headers[AUTH_TOKEN_HEADER];
+            console.log(`Setting auth token: ${authToken}`);
+            this.authToken = authToken;
+        }
         return res.json();
     }
 }
+
+export class ShopApiRequest extends ApiRequest {
+    constructor(fileName) {
+        super('http://localhost:3000/shop-api/', fileName);
+    }
+}
+
+export class AdminApiRequest extends ApiRequest {
+    constructor(fileName) {
+        super('http://localhost:3000/admin-api/', fileName);
+    }
+}

packages/dev-server/package.json

@@ -26,7 +26,9 @@
     "@types/csv-stringify": "^3.1.0",
     "@vendure/testing": "^1.5.1",
     "@vendure/ui-devkit": "^1.5.1",
+    "commander": "^7.1.0",
     "concurrently": "^5.0.0",
-    "csv-stringify": "^5.3.3"
+    "csv-stringify": "^5.3.3",
+    "progress": "^2.0.3"
   }
 }