ci: continuous benchmarking

.gitignore

@@ -126,7 +126,3 @@ dist
 .yarn/build-state.yml
 .yarn/install-state.gz
 .pnp.*
-
-# editor
-.vscode/
-.idea/

.gitlab-ci.yml

@@ -10,9 +10,9 @@ variables:
   GH_PROJECT_PATH: "MatrixAI/${CI_PROJECT_NAME}"
   GH_PROJECT_URL: "https://${GITHUB_TOKEN}@github.com/${GH_PROJECT_PATH}.git"
   # Cache .npm
-  NPM_CONFIG_CACHE: "${CI_PROJECT_DIR}/tmp/npm"
+  npm_config_cache: "${CI_PROJECT_DIR}/tmp/npm"
   # Prefer offline node module installation
-  NPM_CONFIG_PREFER_OFFLINE: "true"
+  npm_config_prefer_offline: "true"
   # Homebrew cache only used by macos runner
   HOMEBREW_CACHE: "${CI_PROJECT_DIR}/tmp/Homebrew"
 

benches/gitgc.ts

@@ -1,13 +1,14 @@
+import path from 'path';
 import b from 'benny';
-import packageJson from '../package.json';
+import { suiteCommon } from './utils';
 
 async function main () {
   let map = new Map();
   let obj = {};
-  let arr = [];
+  let arr: any = [];
   let set = new Set();
   const summary = await b.suite(
-    'gitgc',
+    path.basename(__filename, path.extname(__filename)),
     b.add('map', async () => {
       map = new Map();
       return async () => {
@@ -78,19 +79,7 @@ async function main () {
         }
       };
     }),
-    b.cycle(),
-    b.complete(),
-    b.save({
-      file: 'gitgc',
-      folder: 'benches/results',
-      version: packageJson.version,
-      details: true,
-    }),
-    b.save({
-      file: 'gitgc',
-      folder: 'benches/results',
-      format: 'chart.html',
-    }),
+    ...suiteCommon,
   );
   return summary;
 }

benches/index.ts

@@ -1,18 +1,41 @@
-#!/usr/bin/env node
+#!/usr/bin/env ts-node
 
 import fs from 'fs';
+import path from 'path';
 import si from 'systeminformation';
 import gitgc from './gitgc';
 
 async function main(): Promise<void> {
+  await fs.promises.mkdir(path.join(__dirname, 'results'), { recursive: true });
   await gitgc();
+  const resultFilenames = await fs.promises.readdir(
+    path.join(__dirname, 'results'),
+  );
+  const metricsFile = await fs.promises.open(
+    path.join(__dirname, 'results', 'metrics.txt'),
+    'w',
+  );
+  let concatenating = false;
+  for (const resultFilename of resultFilenames) {
+    if (/.+_metrics\.txt$/.test(resultFilename)) {
+      const metricsData = await fs.promises.readFile(
+        path.join(__dirname, 'results', resultFilename),
+      );
+      if (concatenating) {
+        await metricsFile.write('\n');
+      }
+      await metricsFile.write(metricsData);
+      concatenating = true;
+    }
+  }
+  await metricsFile.close();
   const systemData = await si.get({
     cpu: '*',
     osInfo: 'platform, distro, release, kernel, arch',
     system: 'model, manufacturer',
   });
   await fs.promises.writeFile(
-    'benches/results/system.json',
+    path.join(__dirname, 'results', 'system.json'),
     JSON.stringify(systemData, null, 2),
   );
 }

benches/utils/index.ts

@@ -0,0 +1 @@
+export * from './utils';

benches/utils/utils.ts

@@ -0,0 +1,61 @@
+import fs from 'fs';
+import path from 'path';
+import b from 'benny';
+import { codeBlock } from 'common-tags';
+import packageJson from '../../package.json';
+
+const suiteCommon = [
+  b.cycle(),
+  b.complete(),
+  b.save({
+    file: (summary) => summary.name,
+    folder: path.join(__dirname, '../results'),
+    version: packageJson.version,
+    details: true,
+  }),
+  b.save({
+    file: (summary) => summary.name,
+    folder: path.join(__dirname, '../results'),
+    version: packageJson.version,
+    format: 'chart.html',
+  }),
+  b.complete((summary) => {
+    const filePath = path.join(
+      __dirname,
+      '../results',
+      summary.name + '_metrics.txt',
+    );
+    fs.writeFileSync(
+      filePath,
+      codeBlock`
+      # TYPE ${summary.name}_ops gauge
+      ${summary.results
+        .map(
+          (result) =>
+            `${summary.name}_ops{name="${result.name}"} ${result.ops}`,
+        )
+        .join('\n')}
+
+      # TYPE ${summary.name}_margin gauge
+      ${summary.results
+        .map(
+          (result) =>
+            `${summary.name}_margin{name="${result.name}"} ${result.margin}`,
+        )
+        .join('\n')}
+
+      # TYPE ${summary.name}_samples counter
+      ${summary.results
+        .map(
+          (result) =>
+            `${summary.name}_samples{name="${result.name}"} ${result.samples}`,
+        )
+        .join('\n')}
+      ` + '\n',
+    );
+    // eslint-disable-next-line no-console
+    console.log('\nSaved to:', path.resolve(filePath));
+  }),
+];
+
+export { suiteCommon };