chore(dev-server): Create very large order load test

Relates to #226. Before embarking on performance tuning, we need some kind of benchmark against which we can measure improvements.

packages/dev-server/README.md

@@ -38,8 +38,19 @@ The load tests assume the existence of the following tables in the MySQL databas
 
 The npm scripts `load-test:1k`, `load-test:10k` and `load-test:100k` will populate their respective databases with test data and then run the k6 scripts against them.
 
+## Running individual scripts
+
+An individual test script may be by specifying the script name as an argument:
+
+```
+yarn ts-node load-testing/run-load-test.ts 1000 deep-query.js
+```
+
 ### Results
 
-The results of the test are saved to the [`./load-testing/results`](./load-testing/results) directory.
+The results of the test are saved to the [`./load-testing/results`](./load-testing/results) directory. Each test run creates two files:
+
+* `load-test-<date>-<product-count>-<script-name>.json` Contains a summary of the test
+* `load-test-<date>-<product-count>-<script-name>.csv` Contains time-series data which can be used to create charts
 
 

packages/dev-server/load-testing/generate-summary.ts

@@ -3,15 +3,15 @@ import path from 'path';
 import readline from 'readline';
 
 export type DataPoint = {
-    type: 'Point',
+    type: 'Point';
     data: {
         time: string;
         value: number;
-    },
+    };
     metric: string;
 };
 
-export type TimeSeriesPoint = { timestamp: number; value: number; };
+export type TimeSeriesPoint = { timestamp: number; value: number };
 
 export type LoadTestSummary = {
     timestamp: string;
@@ -28,20 +28,19 @@ export type LoadTestSummary = {
         p90: number;
         p95: number;
         p99: number;
-    },
+    };
     requestDurationTimeSeries: TimeSeriesPoint[];
     concurrentUsersTimeSeries: TimeSeriesPoint[];
+    requestCountTimeSeries: TimeSeriesPoint[];
 };
 
 if (require.main === module) {
     const resultsFile = process.argv[2];
-    generateSummary(resultsFile)
-        .then(result => {
-            // tslint:disable-next-line:no-console
-            console.log(JSON.stringify(result, null, 2));
-            process.exit(0);
-        });
-
+    generateSummary(resultsFile).then(result => {
+        // tslint:disable-next-line:no-console
+        console.log(JSON.stringify(result, null, 2));
+        process.exit(0);
+    });
 }
 
 /**
@@ -61,6 +60,7 @@ export async function generateSummary(rawResultsFile: string): Promise<LoadTestS
     const durations: number[] = [];
     const requestDurationTimeSeries: TimeSeriesPoint[] = [];
     const concurrentUsersTimeSeries: TimeSeriesPoint[] = [];
+    const requestCountTimeSeries: TimeSeriesPoint[] = [];
 
     return new Promise((resolve, reject) => {
         lineReader.on('line', line => {
@@ -68,11 +68,15 @@ export async function generateSummary(rawResultsFile: string): Promise<LoadTestS
             if (isDataPoint(row)) {
                 if (row.metric === 'http_reqs') {
                     reqs++;
+                    requestCountTimeSeries.push({ timestamp: +new Date(row.data.time), value: reqs });
                 }
                 if (row.metric === 'http_req_duration') {
                     const duration = row.data.value;
                     durations.push(duration);
-                    requestDurationTimeSeries.push({ timestamp: +(new Date(row.data.time)), value: row.data.value });
+                    requestDurationTimeSeries.push({
+                        timestamp: +new Date(row.data.time),
+                        value: row.data.value,
+                    });
                     if (duration > max) {
                         max = duration;
                     }
@@ -82,12 +86,15 @@ export async function generateSummary(rawResultsFile: string): Promise<LoadTestS
                     sum += duration;
                 }
                 if (row.metric === 'vus') {
-                    concurrentUsersTimeSeries.push({ timestamp: +(new Date(row.data.time)), value: row.data.value });
+                    concurrentUsersTimeSeries.push({
+                        timestamp: +new Date(row.data.time),
+                        value: row.data.value,
+                    });
                 }
                 if (!startTime) {
-                    startTime = +(new Date(row.data.time));
+                    startTime = +new Date(row.data.time);
                 }
-                endTime = +(new Date(row.data.time));
+                endTime = +new Date(row.data.time);
             }
         });
 
@@ -98,7 +105,7 @@ export async function generateSummary(rawResultsFile: string): Promise<LoadTestS
             resolve({
                 timestamp: new Date().toISOString(),
                 script: rawResultsFile.split('.')[0],
-                productCount: +(rawResultsFile.split('.')[2]),
+                productCount: +rawResultsFile.split('.')[2],
                 testDuration: duration,
                 requests: reqs,
                 throughput: reqs / duration,
@@ -113,6 +120,7 @@ export async function generateSummary(rawResultsFile: string): Promise<LoadTestS
                 },
                 requestDurationTimeSeries,
                 concurrentUsersTimeSeries,
+                requestCountTimeSeries,
             });
         });
     });
@@ -123,7 +131,7 @@ function isDataPoint(row: any): row is DataPoint {
 }
 
 function percentile(p: number, sortedValues: number[]): number {
-    const ordinalRank = ((p / 100) * sortedValues.length) - 1;
+    const ordinalRank = (p / 100) * sortedValues.length - 1;
     if (Number.isInteger(ordinalRank)) {
         return sortedValues[ordinalRank];
     }

packages/dev-server/load-testing/load-test-config.ts

@@ -65,3 +65,10 @@ export function getProductCount() {
     }
     return count;
 }
+
+export function getScriptToRun(): string[] | undefined {
+    const script = process.argv[3];
+    if (script) {
+        return [script];
+    }
+}

packages/dev-server/load-testing/run-load-test.ts

@@ -9,14 +9,18 @@ import path from 'path';
 import { omit } from '../../common/src/omit';
 
 import { generateSummary, LoadTestSummary } from './generate-summary';
-import { getLoadTestConfig, getProductCount } from './load-test-config';
+import { getLoadTestConfig, getProductCount, getScriptToRun } from './load-test-config';
 
 const count = getProductCount();
 
 if (require.main === module) {
+    const ALL_SCRIPTS = ['deep-query.js', 'search-and-checkout.js', 'very-large-order.js'];
+
+    const scriptsToRun = getScriptToRun() || ALL_SCRIPTS;
+
     console.log(`\n============= Vendure Load Test: ${count} products ============\n`);
 
-// Runs the init script to generate test data and populate the test database
+    // Runs the init script to generate test data and populate the test database
     const init = spawn('node', ['-r', 'ts-node/register', './init-load-test.ts', count.toString()], {
         cwd: __dirname,
         stdio: 'inherit',
@@ -25,12 +29,13 @@ if (require.main === module) {
     init.on('exit', code => {
         if (code === 0) {
             return bootstrap(getLoadTestConfig('cookie'))
-                .then(app => {
-                    return runLoadTestScript('deep-query.js')
-                        .then((summary1) => runLoadTestScript('search-and-checkout.js').then(summary2 => [summary1, summary2]))
-                        .then(summaries => {
-                            closeAndExit(app, summaries);
-                        });
+                .then(async app => {
+                    const summaries: LoadTestSummary[] = [];
+                    for (const script of scriptsToRun) {
+                        const summary = await runLoadTestScript(script);
+                        summaries.push(summary);
+                    }
+                    return closeAndExit(app, summaries);
                 })
                 .catch(err => {
                     // tslint:disable-next-line
@@ -46,10 +51,14 @@ function runLoadTestScript(script: string): Promise<LoadTestSummary> {
     const rawResultsFile = `${script}.${count}.json`;
 
     return new Promise((resolve, reject) => {
-        const loadTest = spawn('k6', ['run', `./scripts/${script}`, '--out', `json=results/${rawResultsFile}`], {
-            cwd: __dirname,
-            stdio: 'inherit',
-        });
+        const loadTest = spawn(
+            'k6',
+            ['run', `./scripts/${script}`, '--out', `json=results/${rawResultsFile}`],
+            {
+                cwd: __dirname,
+                stdio: 'inherit',
+            },
+        );
         loadTest.on('exit', code => {
             if (code === 0) {
                 resolve(code);
@@ -60,8 +69,7 @@ function runLoadTestScript(script: string): Promise<LoadTestSummary> {
         loadTest.on('error', err => {
             reject(err);
         });
-    })
-        .then(() => generateSummary(rawResultsFile));
+    }).then(() => generateSummary(rawResultsFile));
 }
 
 async function closeAndExit(app: INestApplication, summaries: LoadTestSummary[]) {
@@ -72,7 +80,9 @@ async function closeAndExit(app: INestApplication, summaries: LoadTestSummary[])
     const dateString = getDateString();
 
     // write summary JSON
-    const summaryData = summaries.map(s => omit(s, ['requestDurationTimeSeries', 'concurrentUsersTimeSeries']));
+    const summaryData = summaries.map(s =>
+        omit(s, ['requestDurationTimeSeries', 'concurrentUsersTimeSeries', 'requestCountTimeSeries']),
+    );
     const summaryFile = path.join(__dirname, `results/load-test-${dateString}-${count}.json`);
     fs.writeFileSync(summaryFile, JSON.stringify(summaryData, null, 2), 'utf-8');
     console.log(`Summary written to ${path.relative(__dirname, summaryFile)}`);
@@ -80,7 +90,10 @@ async function closeAndExit(app: INestApplication, summaries: LoadTestSummary[])
     // write time series CSV
     for (const summary of summaries) {
         const csvData = await getTimeSeriesCsvData(summary);
-        const timeSeriesFile = path.join(__dirname, `results/load-test-${dateString}-${count}-${summary.script}.csv`);
+        const timeSeriesFile = path.join(
+            __dirname,
+            `results/load-test-${dateString}-${count}-${summary.script}.csv`,
+        );
         fs.writeFileSync(timeSeriesFile, csvData, 'utf-8');
         console.log(`Time series data written to ${path.relative(__dirname, timeSeriesFile)}`);
     }
@@ -89,7 +102,6 @@ async function closeAndExit(app: INestApplication, summaries: LoadTestSummary[])
 }
 
 async function getTimeSeriesCsvData(summary: LoadTestSummary): Promise<string> {
-
     const stringifier = stringify({
         delimiter: ',',
     });
@@ -99,7 +111,7 @@ async function getTimeSeriesCsvData(summary: LoadTestSummary): Promise<string> {
     stringifier.on('readable', () => {
         let row;
         // tslint:disable-next-line:no-conditional-assignment
-        while (row = stringifier.read()) {
+        while ((row = stringifier.read())) {
             data.push(row);
         }
     });
@@ -108,6 +120,7 @@ async function getTimeSeriesCsvData(summary: LoadTestSummary): Promise<string> {
         `${summary.script}:elapsed`,
         `${summary.script}:request_duration`,
         `${summary.script}:user_count`,
+        `${summary.script}:reqs`,
     ]);
 
     let startTime: number | undefined;
@@ -116,13 +129,19 @@ async function getTimeSeriesCsvData(summary: LoadTestSummary): Promise<string> {
         if (!startTime) {
             startTime = row.timestamp;
         }
-        stringifier.write([row.timestamp - startTime, row.value, '']);
+        stringifier.write([row.timestamp - startTime, row.value, '', '']);
     }
     for (const row of summary.concurrentUsersTimeSeries) {
         if (!startTime) {
             startTime = row.timestamp;
         }
-        stringifier.write([row.timestamp - startTime, '', row.value]);
+        stringifier.write([row.timestamp - startTime, '', row.value, '']);
+    }
+    for (const row of summary.requestCountTimeSeries) {
+        if (!startTime) {
+            startTime = row.timestamp;
+        }
+        stringifier.write([row.timestamp - startTime, '', '', row.value]);
     }
 
     stringifier.end();
@@ -138,5 +157,8 @@ async function getTimeSeriesCsvData(summary: LoadTestSummary): Promise<string> {
 }
 
 function getDateString(): string {
-    return (new Date().toISOString()).split('.')[0].replace(/[:\.]/g, '_');
+    return new Date()
+        .toISOString()
+        .split('.')[0]
+        .replace(/[:\.]/g, '_');
 }

packages/dev-server/load-testing/scripts/very-large-order.js

@@ -0,0 +1,39 @@
+// @ts-check
+import {check} from 'k6';
+import {ShopApiRequest} from '../utils/api-request.js';
+
+const searchQuery = new ShopApiRequest('shop/search.graphql');
+const addItemToOrderMutation = new ShopApiRequest('shop/add-to-order.graphql');
+
+export let options = {
+    stages: [
+        { duration: '4m', target: 1 },
+    ],
+};
+
+export function setup() {
+    const searchResult = searchQuery.post();
+    return searchResult.data.search.items;
+}
+
+/**
+ * Continuously adds random items to a single order for the duration of the test.
+ */
+export default function(products) {
+    for (let i = 0; i < 10000; i ++) {
+        addToCart(randomItem(products).productVariantId);
+    }
+}
+
+function addToCart(variantId) {
+    const qty = Math.ceil(Math.random() * 4);
+    const result = addItemToOrderMutation.post({ id: variantId, qty });
+    check(result.data, {
+        'Product added to cart': r => !!r.addItemToOrder.lines
+            .find(l => l.productVariant.id === variantId && l.quantity >= qty),
+    });
+}
+
+function randomItem(items) {
+    return items[Math.floor(Math.random() * items.length)];
+}