Преглед изворни кода

feat(test): Create complete load test script (generate, serve, test)

Relates to #74
Michael Bromley пре 6 година
родитељ
комит
d7713b6be4

+ 2 - 0
packages/dev-server/.gitignore

@@ -2,3 +2,5 @@ assets
 vendure
 test-emails
 vendure.sqlite
+vendure-import-error.log
+load-testing/data-sources/products*.csv

BIN
packages/dev-server/load-testing/data-sources/product-image.jpg


+ 181 - 0
packages/dev-server/load-testing/init-load-test.ts

@@ -0,0 +1,181 @@
+// tslint:disable-next-line:no-reference
+/// <reference path="../../core/typings.d.ts" />
+import { bootstrap, VendureConfig } from '@vendure/core';
+import { populate } from '@vendure/core/cli/populate';
+import { BaseProductRecord } from '@vendure/core/dist/data-import/providers/import-parser/import-parser';
+import stringify from 'csv-stringify';
+import fs from 'fs';
+import path from 'path';
+
+import { clearAllTables } from '../../core/mock-data/clear-all-tables';
+import { initialData } from '../../core/mock-data/data-sources/initial-data';
+import { populateCustomers } from '../../core/mock-data/populate-customers';
+import { devConfig } from '../dev-config';
+
+import { getLoadTestConfig, getMysqlConnectionOptions, getProductCount, getProductCsvFilePath } from './load-test-config';
+
+// tslint:disable:no-console
+
+/**
+ * A script used to populate a database with test data for load testing.
+ */
+if (require.main === module) {
+    // Running from command line
+    isDatabasePopulated()
+        .then(isPopulated => {
+            if (!isPopulated) {
+                const count = getProductCount();
+                const config = getLoadTestConfig();
+                const csvFile = getProductCsvFilePath();
+                return clearAllTables(config.dbConnectionOptions, true)
+                    .then(() => {
+                        if (!fs.existsSync(csvFile)) {
+                            return generateProductsCsv(count);
+                        }
+                    })
+                    .then(() => populate(() => bootstrap(config),
+                        path.join(__dirname, '../../create/assets/initial-data.json'),
+                        csvFile,
+                        path.join(__dirname, './data-sources'),
+                    ))
+                    .then(async app => {
+                        console.log('populating customers...');
+                        await populateCustomers(10, config as any, true);
+                        return app.close();
+                    });
+            } else {
+                console.log('Database is already populated!');
+            }
+        })
+        .then(
+            () => process.exit(0),
+            err => {
+                console.log(err);
+                process.exit(1);
+            },
+        );
+}
+
+/**
+ * Tests to see whether the load test database is already populated.
+ */
+function isDatabasePopulated(): Promise<boolean> {
+    const mysql = require('mysql');
+    const count = getProductCount();
+    const mysqlConnectionOptions = getMysqlConnectionOptions(count);
+    const connection = mysql.createConnection({
+        host: mysqlConnectionOptions.host,
+        user: mysqlConnectionOptions.username,
+        password: mysqlConnectionOptions.password,
+        database: mysqlConnectionOptions.database,
+    });
+
+    return new Promise<boolean>((resolve, reject) => {
+        connection.connect((error: any) => {
+            if (error) {
+                reject(error);
+                return;
+            }
+
+            connection.query('SELECT COUNT(id) as prodCount FROM product', (err: any, results: any) => {
+                if (err) {
+                    if (err.code === 'ER_NO_SUCH_TABLE') {
+                        resolve(false);
+                        return;
+                    }
+                    reject(err);
+                    return;
+                }
+                resolve(results[0].prodCount === count);
+            });
+        });
+    });
+
+}
+
+/**
+ * Generates a CSV file of test product data which can then be imported into Vendure.
+ */
+function generateProductsCsv(productCount: number = 100): Promise<void> {
+    const result: BaseProductRecord[] = [];
+
+    const stringifier = stringify({
+        delimiter: ',',
+    });
+
+    const data: string[] = [];
+
+    console.log(`Generating ${productCount} rows of test product data...`);
+
+    stringifier.on('readable', () => {
+        let row;
+        // tslint:disable-next-line:no-conditional-assignment
+        while (row = stringifier.read()) {
+            data.push(row);
+        }
+    });
+
+    return new Promise((resolve, reject) => {
+        const csvFile = getProductCsvFilePath();
+        stringifier.on('error', (err: any) => {
+            reject(err.message);
+        });
+        stringifier.on('finish', async () => {
+            fs.writeFileSync(csvFile, data.join(''));
+            console.log(`Done! Saved to ${csvFile}`);
+            resolve();
+        });
+        generateMockData(productCount, row => stringifier.write(row));
+        stringifier.end();
+    });
+}
+
+function generateMockData(productCount: number, writeFn: (row: string[]) => void) {
+    const headers: Array<keyof BaseProductRecord> = [
+        'name',
+        'slug',
+        'description',
+        'assets',
+        'facets',
+        'optionGroups',
+        'optionValues',
+        'sku',
+        'price',
+        'taxCategory',
+        'variantAssets',
+        'variantFacets',
+    ];
+
+    writeFn(headers);
+
+    const LOREM = 'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna ' +
+        'aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. ' +
+        'Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. ' +
+        'Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.'
+            .replace(/\r\n/g, '');
+
+    const categories = getCategoryNames();
+
+    for (let i = 1; i <= productCount; i++) {
+        const outputRow: BaseProductRecord = {
+            name: `Product ${i}`,
+            slug: `product-${i}`,
+            description: LOREM,
+            assets: 'product-image.jpg',
+            facets: `category:${categories[i % categories.length]}`,
+            optionGroups: '',
+            optionValues: '',
+            sku: `PRODID${i}`,
+            price: '12345',
+            taxCategory: 'standard',
+            variantAssets: '',
+            variantFacets: '',
+        };
+        writeFn(Object.values(outputRow) as string[]);
+    }
+}
+
+function getCategoryNames() {
+    const allNames = initialData.collections.reduce((all, c) => [...all, ...c.facetNames], [] as string[]);
+    return Array.from(new Set(allNames));
+}

+ 46 - 0
packages/dev-server/load-testing/load-test-config.ts

@@ -0,0 +1,46 @@
+/* tslint:disable:no-console */
+import { VendureConfig } from '@vendure/core';
+import path from 'path';
+
+import { devConfig } from '../dev-config';
+
+export function getMysqlConnectionOptions(count: number) {
+    return {
+        type: 'mysql',
+        host: '192.168.99.100',
+        port: 3306,
+        username: 'root',
+        password: '',
+        database: `vendure-load-testing-${count}`,
+    };
+}
+
+export function getLoadTestConfig(): VendureConfig {
+    const count = getProductCount();
+    return {
+        ...devConfig as any,
+        dbConnectionOptions: getMysqlConnectionOptions(count),
+        authOptions: {
+            tokenMethod: 'bearer',
+            requireVerification: false,
+        },
+        importExportOptions: {
+            importAssetsDir: path.join(__dirname, './data-sources'),
+        },
+        customFields: {},
+    };
+}
+
+export function getProductCsvFilePath() {
+    const count = getProductCount();
+    return path.join(__dirname, `./data-sources/products-${count}.csv`);
+}
+
+export function getProductCount() {
+    const count = +process.argv[2];
+    if (!count) {
+        console.error(`Please specify the number of products to generate`);
+        process.exit(1);
+    }
+    return count;
+}

+ 37 - 0
packages/dev-server/load-testing/run-load-test.ts

@@ -0,0 +1,37 @@
+/* tslint:disable:no-console */
+import { bootstrap } from '@vendure/core';
+import { ChildProcess, spawn } from 'child_process';
+
+import { getLoadTestConfig, getProductCount } from './load-test-config';
+
+const count = getProductCount();
+
+console.log(`\n============= Vendure Load Test: ${count} products ============\n`);
+
+// Runs the init script to generate test data and populate the test database
+const init = spawn('node', ['-r', 'ts-node/register', './init-load-test.ts', count.toString()], {
+    cwd: __dirname,
+    stdio: 'inherit',
+});
+
+init.on('exit', code => {
+    if (code === 0) {
+        return bootstrap(getLoadTestConfig())
+            .then(app => {
+                const loadTest = spawn('k6', ['run', './scripts/search-and-checkout.js'], {
+                    cwd: __dirname,
+                    stdio: 'inherit',
+                });
+                loadTest.on('exit', () => {
+                    app.close();
+                    process.exit(0);
+                });
+            })
+            .catch(err => {
+                // tslint:disable-next-line
+                console.log(err);
+            });
+    } else {
+        process.exit(code || 1);
+    }
+});

+ 9 - 0
packages/dev-server/load-testing/scripts/search-and-checkout.js

@@ -6,6 +6,15 @@ const searchQuery = new ShopApiRequest('shop/search.graphql');
 const productQuery = new ShopApiRequest('shop/product.graphql');
 const addItemToOrderMutation = new ShopApiRequest('shop/add-to-order.graphql');
 
+export let options = {
+  stages: [
+      { duration: '30s', target: 10 },
+      { duration: '1m', target: 75 },
+      { duration: '1m', target: 150 },
+      { duration: '1m', target: 0 },
+  ],
+};
+
 /**
  * Searches for products, adds to order, checks out.
  */

+ 6 - 3
packages/dev-server/package.json

@@ -7,17 +7,20 @@
   "scripts": {
     "populate": "node -r ts-node/register populate-dev-server.ts",
     "start": "nodemon --config nodemon-debug.json index.ts",
-    "load-test": "k6 run load-testing/scripts/search-and-checkout.js"
+    "load-test:1k": "node -r ts-node/register load-testing/run-load-test.ts 1000",
+    "load-test:10k": "node -r ts-node/register load-testing/run-load-test.ts 10000"
   },
   "dependencies": {
+    "@vendure/admin-ui-plugin": "~0.1.0",
+    "@vendure/asset-server-plugin": "~0.1.0",
     "@vendure/common": "~0.1.0",
     "@vendure/core": ">=0.1.0-alpha.18",
-    "@vendure/asset-server-plugin": "~0.1.0",
-    "@vendure/admin-ui-plugin": "~0.1.0",
     "@vendure/email-plugin": "~0.1.0",
     "typescript": "^3.3.4000"
   },
   "devDependencies": {
+    "@types/csv-stringify": "^1.4.3",
+    "csv-stringify": "^5.3.0",
     "nodemon": "^1.18.9"
   }
 }

+ 15 - 1
yarn.lock

@@ -1200,6 +1200,13 @@
   dependencies:
     "@types/node" "*"
 
+"@types/csv-stringify@^1.4.3":
+  version "1.4.3"
+  resolved "https://registry.yarnpkg.com/@types/csv-stringify/-/csv-stringify-1.4.3.tgz#538326331445809293fe8755694173dd01590f32"
+  integrity sha512-vfbvOBhuTjDYQjdG8dCxLjOsyQMjCE0oN0bIUkJtiolqkCe1WTCjh36w4hEhtkdLHUgsB4aN4r6SFD8iLJgiGQ==
+  dependencies:
+    "@types/node" "*"
+
 "@types/dateformat@^3.0.0":
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/@types/dateformat/-/dateformat-3.0.0.tgz#d3cbfa8ac3789990592f3d9585331cdb2c5f979f"
@@ -3304,6 +3311,13 @@ csv-parse@^4.3.0:
   resolved "https://registry.yarnpkg.com/csv-parse/-/csv-parse-4.3.4.tgz#fc896c170ebbdf6fb286de85c41bbaea4973d25f"
   integrity sha512-M1R4WL+vt81+GnkKzi0s1qQM6WXvHQKDecNkpozzAEG8LHvIW9bq5eBnOKFQn50fTuAos7JodBh/07MK+J6G2Q==
 
+csv-stringify@^5.3.0:
+  version "5.3.0"
+  resolved "https://registry.yarnpkg.com/csv-stringify/-/csv-stringify-5.3.0.tgz#ff2dfafa6fcccd455ff5039be9c202475aa3bbe0"
+  integrity sha512-VMYPbE8zWz475smwqb9VbX9cj0y4J0PBl59UdcqzLkzXHZZ8dh4Rmbb0ZywsWEtUml4A96Hn7Q5MW9ppVghYzg==
+  dependencies:
+    lodash.get "~4.4.2"
+
 currently-unhandled@^0.4.1:
   version "0.4.1"
   resolved "https://registry.yarnpkg.com/currently-unhandled/-/currently-unhandled-0.4.1.tgz#988df33feab191ef799a61369dd76c17adf957ea"
@@ -6861,7 +6875,7 @@ lodash.foreach@^4.3.0:
   resolved "https://registry.yarnpkg.com/lodash.foreach/-/lodash.foreach-4.5.0.tgz#1a6a35eace401280c7f06dddec35165ab27e3e53"
   integrity sha1-Gmo16s5AEoDH8G3d7DUWWrJ+PlM=
 
-lodash.get@^4.4.2:
+lodash.get@^4.4.2, lodash.get@~4.4.2:
   version "4.4.2"
   resolved "https://registry.yarnpkg.com/lodash.get/-/lodash.get-4.4.2.tgz#2d177f652fa31e939b4438d5341499dfa3825e99"
   integrity sha1-LRd/ZS+jHpObRDjVNBSZ36OCXpk=