Merge pull request #20079 from cypress-io/issue-19403-perf-reporter-changes

chore: Performance reporter changes

Author: Blue F

circle.yml

@@ -206,6 +206,9 @@ commands:
           name: Restore cache state, to check for known modules cache existence
           keys:
             - v{{ .Environment.CACHE_VERSION }}-{{ arch }}-system-tests-projects-node-modules-cache-state-{{ checksum "system_tests_cache_key" }}
+      - run:
+          name: Send root honeycomb event for this CI build
+          command: cd system-tests/scripts && node ./send-root-honecomb-event.js
       - run:
           name: Bail if specific cache exists
           command: |
@@ -2042,6 +2045,7 @@ linux-workflow: &linux-workflow
         requires:
           - build
     - system-tests-node-modules-install:
+        context: test-runner:performance-tracking
         requires:
           - build
     - system-tests-chrome:

scripts/get-next-version.js

@@ -24,6 +24,12 @@ const getNextVersionForPath = async (path) => {
   return semver.inc(currentVersion, releaseType || 'patch')
 }
 
+if (require.main !== module) {
+  module.exports.getNextVersionForPath = getNextVersionForPath
+
+  return
+}
+
 Bluebird.mapSeries(paths, async (path) => {
   const pathNextVersion = await getNextVersionForPath(path)
 

system-tests/lib/performance-reporter.js

@@ -1,36 +1,109 @@
 const path = require('path')
 const chalk = require('chalk')
 const Libhoney = require('libhoney')
+const { v4: uuidv4 } = require('uuid')
 
-const pkg = require('@packages/root')
 const ciProvider = require('@packages/server/lib/util/ci_provider')
 const { commitInfo } = require('@cypress/commit-info')
+const { getNextVersionForPath } = require('../../scripts/get-next-version')
 
-class StatsdReporter {
+const honey = new Libhoney({
+  dataset: 'systemtest-performance',
+  writeKey: process.env.HONEYCOMB_API_KEY,
+})
+
+// This event is created here independently every time the reporter
+// is imported (in each parallel instance of the system-tests
+// in circleci) so that we can use it as the parent,
+// but ../scripts/send-root-honeycomb-event.js
+// is only invoked once at the start of the build,
+// and is responsible for sending it to honeycomb.
+const spanId = process.env.CIRCLE_WORKFLOW_ID || uuidv4()
+const circleCiRootEvent = honey.newEvent()
+
+circleCiRootEvent.timestamp = Date.now()
+circleCiRootEvent.add({
+  buildUrl: process.env.CIRCLE_BUILD_URL,
+  platform: process.platform,
+  arch: process.arch,
+  name: 'ci_run',
+
+  spanId,
+  traceId: spanId,
+})
+
+// Mocha events ('test', 'test end', etc) have no way to wait
+// for async callbacks, so we can't guarantee we have this
+// data ready by the time any of the reporter's events are emitted.
+
+// Therefore, we have each honeycomb event await this promise
+// before sending itself.
+let asyncInfo = Promise.all([getNextVersionForPath(path.resolve(__dirname, '../../packages')), commitInfo()])
+.then(([nextVersion, commitInformation]) => {
+  const ciInformation = ciProvider.commitParams() || {}
+
+  return {
+    nextVersion,
+    branch: commitInformation.branch || ciInformation.branch,
+    commitSha: commitInformation.sha || ciInformation.sha,
+  }
+})
+
+function addAsyncInfoAndSend (honeycombEvent) {
+  return asyncInfo.then((info) => {
+    honeycombEvent.add(info)
+    honeycombEvent.send()
+  })
+}
+
+class HoneycombReporter {
   constructor (runner) {
     if (!process.env.HONEYCOMB_API_KEY) {
       return
     }
 
     console.log(chalk.green('Reporting to honeycomb'))
 
-    let branch
-    let commitSha
+    runner.on('suite', (suite) => {
+      if (!suite.title) {
+        return
+      }
 
-    this.honey = new Libhoney({
-      dataset: 'systemtest-performance',
-      writeKey: process.env.HONEYCOMB_API_KEY,
-    })
+      const parent = suite.parent && suite.parent.honeycombEvent ? suite.parent.honeycombEvent : circleCiRootEvent
 
-    commitInfo().then((commitInformation) => {
-      const ciInformation = ciProvider.commitParams() || {}
+      suite.honeycombEvent = honey.newEvent()
+      suite.honeycombEvent.timestamp = Date.now()
+      suite.honeycombEvent.add({
+        ...parent.data,
+        suite: suite.title,
+        specFile: suite.file && path.basename(suite.file),
+        name: 'spec_execution',
 
-      branch = commitInformation.branch || ciInformation.branch
-      commitSha = commitInformation.sha || ciInformation.sha
+        spanId: uuidv4(),
+        parentId: parent.data.spanId,
+      })
     })
 
     runner.on('test', (test) => {
-      test.wallclockStart = Date.now()
+      const path = test.titlePath()
+      // This regex pulls apart a string like `failing1 [electron]`
+      // into `failing1` and `electron`, letting us use the same
+      // test name for all browsers, with the browser as a separate field.
+      // The browser capture group is optional because some tests aren't browser specific,
+      // in which case it will be undefined and not passed as a field to honeycomb.
+      const [, testTitle, browser] = path[path.length - 1].match(/(.+?)(?: \[([a-z]+)\])?$/)
+
+      test.honeycombEvent = honey.newEvent()
+      test.honeycombEvent.timestamp = Date.now()
+      test.honeycombEvent.add({
+        ...test.parent.honeycombEvent.data,
+        test: testTitle,
+        browser,
+        name: 'test_execution',
+
+        spanId: uuidv4(),
+        parentId: test.parent.honeycombEvent.data.spanId,
+      })
     })
 
     runner.on('test end', (test) => {
@@ -39,44 +112,42 @@ class StatsdReporter {
         return
       }
 
-      const title = test.titlePath().join(' / ')
-      // This regex pulls apart a string like `e2e async timeouts / failing1 [electron]`
-      // into `e2e async timeouts / failing1` and `electron`, letting us use the same
-      // test name for all browsers, with the browser as a separate field.
-      // The browser capture group is optional because some tests aren't browser specific,
-      // in which case it will be undefined and not passed as a field to honeycomb.
-      const [, testTitle, browser] = title.match(/(.+?)(?: \[([a-z]+)\])?$/)
-
-      const honeycombEvent = this.honey.newEvent()
-
-      honeycombEvent.timestamp = test.wallclockStart
-      honeycombEvent.add({
-        test: testTitle,
-        specFile: path.basename(test.file),
-        browser,
+      test.honeycombEvent.add({
         state: test.state,
         err: test.err && test.err.message,
         errStack: test.err && test.err.stack,
-        durationMs: Date.now() - test.wallclockStart,
-        mochaDurationMs: test.duration,
-        branch,
-        commitSha,
-        buildUrl: process.env.CIRCLE_BUILD_URL,
-        platform: process.platform,
-        arch: process.arch,
-        version: pkg.version,
+        durationMs: Date.now() - test.honeycombEvent.timestamp,
       })
 
-      honeycombEvent.send()
+      addAsyncInfoAndSend(test.honeycombEvent)
+    })
+
+    runner.on('suite end', (suite) => {
+      if (!suite.honeycombEvent) {
+        return
+      }
+
+      suite.honeycombEvent.add({
+        durationMs: Date.now() - suite.honeycombEvent.timestamp,
+      })
+
+      addAsyncInfoAndSend(suite.honeycombEvent)
     })
   }
 
-  // If there is no done callback, then mocha-multi-reporter will kill the process without waiting for our honeycomb post to complete.
+  // If there is no done method, then mocha-multi-reporter will kill the process
+  // without waiting for our honeycomb posts to complete.
   done (failures, callback) {
-    if (this.honey) {
-      this.honey.flush().then(callback)
-    }
+    // Await the asyncInfo promise one last time, to ensure all events have
+    // added the data and sent themselves before we flush honeycomb's queue and exit.
+    asyncInfo
+    .then(() => honey.flush())
+    .then(callback)
   }
 }
 
-module.exports = StatsdReporter
+module.exports = HoneycombReporter
+
+HoneycombReporter.honey = honey
+HoneycombReporter.circleCiRootEvent = circleCiRootEvent
+HoneycombReporter.addAsyncInfoAndSend = addAsyncInfoAndSend

system-tests/scripts/send-root-honecomb-event.js

@@ -0,0 +1,13 @@
+const { addAsyncInfoAndSend, circleCiRootEvent, honey } = require('../lib/performance-reporter')
+
+// This file is executed once during the circleci build,
+// so that we can send the root event honeycomb event for this
+// run of the system tests exactly once.
+// All the system test build hosts reference this root event,
+// joining them into a single trace.
+if (require.main === module) {
+  addAsyncInfoAndSend(circleCiRootEvent).then(() => {
+    console.log(circleCiRootEvent.data)
+    honey.flush()
+  })
+}