Browse Source

test(core): Add default search plugin benchmark tests (#2573)

Hans 2 years ago
parent
commit
4000bc6dcf

+ 36 - 0
e2e-common/vitest.config.bench.ts

@@ -0,0 +1,36 @@
+import path from 'path';
+import swc from 'unplugin-swc';
+import { defineConfig } from 'vitest/config';
+
+export default defineConfig({
+    test: {
+        include: ['**/*.bench.ts'],
+        /**
+         * For local debugging of the e2e tests, we set a very long timeout value otherwise tests will
+         * automatically fail for going over the 5 second default timeout.
+         */
+        testTimeout: process.env.E2E_DEBUG ? 1800 * 1000 : process.env.CI ? 30 * 1000 : 15 * 1000,
+        // threads: false,
+        // singleThread: true,
+        // reporters: ['verbose'],
+        typecheck: {
+            tsconfig: path.join(__dirname, 'tsconfig.e2e.json'),
+        },
+        // In jobs-queue.e2e-spec.ts, we use `it.only()` for sqljs, so we need this
+        // set to true to avoid failures in CI.
+        allowOnly: true,
+    },
+    plugins: [
+        // SWC required to support decorators used in test plugins
+        // See https://github.com/vitest-dev/vitest/issues/708#issuecomment-1118628479
+        // Vite plugin
+        swc.vite({
+            jsc: {
+                transform: {
+                    // See https://github.com/vendure-ecommerce/vendure/issues/2099
+                    useDefineForClassFields: false,
+                },
+            },
+        }),
+    ],
+});

+ 1 - 0
package.json

@@ -50,6 +50,7 @@
     "lerna": "^7.1.5",
     "lint-staged": "^10.5.4",
     "prettier": "^2.2.1",
+    "tinybench": "^2.5.1",
     "ts-node": "^10.9.1",
     "typescript": "4.9.5",
     "unplugin-swc": "^1.3.2",

+ 114 - 0
packages/core/e2e/default-search-plugin.bench.ts

@@ -0,0 +1,114 @@
+/* eslint-disable @typescript-eslint/no-non-null-assertion */
+import { DefaultJobQueuePlugin, DefaultSearchPlugin, mergeConfig } from '@vendure/core';
+import { createTestEnvironment, registerInitializer, SqljsInitializer } from '@vendure/testing';
+import path from 'path';
+import { Bench } from 'tinybench';
+import { afterAll, beforeAll, describe, expect, it } from 'vitest';
+
+import { initialData } from '../../../e2e-common/e2e-initial-data';
+import { testConfig, TEST_SETUP_TIMEOUT_MS } from '../../../e2e-common/test-config';
+
+import {
+    SearchProductsShopQuery,
+    SearchProductsShopQueryVariables,
+} from './graphql/generated-e2e-shop-types';
+import { SEARCH_PRODUCTS_SHOP } from './graphql/shop-definitions';
+import { awaitRunningJobs } from './utils/await-running-jobs';
+
+registerInitializer('sqljs', new SqljsInitializer(path.join(__dirname, '__data__'), 1000));
+
+interface SearchProductsShopQueryVariablesExt extends SearchProductsShopQueryVariables {
+    input: SearchProductsShopQueryVariables['input'] & {
+        // This input field is dynamically added only when the `indexStockStatus` init option
+        // is set to `true`, and therefore not included in the generated type. Therefore
+        // we need to manually patch it here.
+        inStock?: boolean;
+    };
+}
+
+const { server, adminClient, shopClient } = createTestEnvironment(
+    mergeConfig(testConfig(), {
+        plugins: [DefaultSearchPlugin.init({ indexStockStatus: true }), DefaultJobQueuePlugin],
+    }),
+);
+
+let marginFactor = 1; // Defaults to 1, will be adjusted during test
+let cpuFactor = 1; // Defaults to 1, will be adjusted during test
+const fibonacci = (i: number): number => {
+    if (i <= 1) return i;
+    return fibonacci(i - 1) + fibonacci(i - 2);
+};
+
+beforeAll(async () => {
+    await server.init({
+        initialData,
+        productsCsvPath: path.join(__dirname, 'fixtures/e2e-products-default-search.csv'),
+        customerCount: 1,
+    });
+    await adminClient.asSuperAdmin();
+    await awaitRunningJobs(adminClient);
+}, TEST_SETUP_TIMEOUT_MS);
+
+afterAll(async () => {
+    await server.destroy();
+});
+
+const isDevelopment = process.env.NODE_ENV === 'development';
+describe.skipIf(isDevelopment)('Default search plugin - benchmark', () => {
+    it('defines benchmark cpu and margin factor', async () => {
+        const bench = new Bench({
+            warmupTime: 0,
+            warmupIterations: 1,
+            time: 0,
+            iterations: 10,
+        });
+
+        bench.add('measure time to calcualate fibonacci', () => {
+            fibonacci(41); // If this task would take 1000 ms the cpuFactor would be 1.
+        });
+
+        const tasks = await bench.run();
+
+        // console.table(bench.table());
+
+        tasks.forEach(task => {
+            expect(task.result?.rme).toBeDefined();
+            expect(task.result?.mean).toBeDefined();
+            if (task.result?.rme && task.result?.mean) {
+                marginFactor = 1 + task.result.rme / 100;
+                cpuFactor = 1000 / task.result.mean;
+            }
+        });
+    });
+
+    it('performs when grouped by product', async () => {
+        const bench = new Bench({
+            warmupTime: 0,
+            warmupIterations: 3,
+            time: 0,
+            iterations: 1000,
+        });
+
+        bench.add('group by product', async () => {
+            await shopClient.query<SearchProductsShopQuery, SearchProductsShopQueryVariablesExt>(
+                SEARCH_PRODUCTS_SHOP,
+                {
+                    input: {
+                        groupByProduct: true,
+                    },
+                },
+            );
+        });
+
+        const tasks = await bench.run();
+
+        // console.table(bench.table());
+
+        tasks.forEach(task => {
+            expect(task.result?.mean).toBeDefined();
+            if (task.result?.mean) {
+                expect(task.result.mean * cpuFactor).toBeLessThan(6.835 * marginFactor);
+            }
+        });
+    });
+});

File diff suppressed because it is too large
+ 260 - 310
packages/core/e2e/default-search-plugin.e2e-spec.ts


+ 1 - 0
packages/core/package.json

@@ -27,6 +27,7 @@
         "test": "vitest --config ./vitest.config.ts --run",
         "e2e": "cross-env PACKAGE=core vitest --config ../../e2e-common/vitest.config.ts --run",
         "e2e:watch": "cross-env PACKAGE=core vitest --config ../../e2e-common/vitest.config.ts",
+        "bench": "cross-env PACKAGE=core vitest --config ../../e2e-common/vitest.config.bench.ts --run",
         "ci": "yarn build"
     },
     "publishConfig": {

+ 5 - 0
yarn.lock

@@ -17746,6 +17746,11 @@ tinybench@^2.5.0:
   resolved "https://registry.yarnpkg.com/tinybench/-/tinybench-2.5.0.tgz#4711c99bbf6f3e986f67eb722fed9cddb3a68ba5"
   integrity sha512-kRwSG8Zx4tjF9ZiyH4bhaebu+EDz1BOx9hOigYHlUW4xxI/wKIUQUqo018UlU4ar6ATPBsaMrdbKZ+tmPdohFA==
 
+tinybench@^2.5.1:
+  version "2.5.1"
+  resolved "https://registry.yarnpkg.com/tinybench/-/tinybench-2.5.1.tgz#3408f6552125e53a5a48adee31261686fd71587e"
+  integrity sha512-65NKvSuAVDP/n4CqH+a9w2kTlLReS9vhsAP06MWx+/89nMinJyB2icyl58RIcqCmIggpojIGeuJGhjU1aGMBSg==
+
 tinypool@^0.7.0:
   version "0.7.0"
   resolved "https://registry.yarnpkg.com/tinypool/-/tinypool-0.7.0.tgz#88053cc99b4a594382af23190c609d93fddf8021"

Some files were not shown because too many files changed in this diff