Add oversight to tools

tools/oversight/acquisition.js

@@ -0,0 +1,150 @@
+/*
+ * Copyright 2024 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+const vendorClassifications = [
+  { regex: /google|googleads|google-ads|google_search|google_deman|adwords|dv360|gdn|doubleclick|dbm|gmb/i, result: 'google' },
+  { regex: /instagram|ig/i, result: 'instagram' },
+  { regex: /facebook|fb|meta/i, result: 'facebook' },
+  { regex: /bing/i, result: 'bing' },
+  { regex: /tiktok/i, result: 'tiktok' },
+  { regex: /youtube|yt/i, result: 'youtube' },
+  { regex: /linkedin/i, result: 'linkedin' },
+  { regex: /twitter/i, result: 'x' },
+  { regex: /snapchat/i, result: 'snapchat' },
+  { regex: /microsoft/i, result: 'microsoft' },
+  { regex: /pinterest/i, result: 'pinterest' },
+  { regex: /reddit/i, result: 'reddit' },
+  { regex: /spotify/i, result: 'spotify' },
+  { regex: /criteo/i, result: 'criteo' },
+  { regex: /taboola/i, result: 'taboola' },
+  { regex: /outbrain/i, result: 'outbrain' },
+  { regex: /yahoo/i, result: 'yahoo' },
+  { regex: /marketo/i, result: 'marketo' },
+  { regex: /eloqua/i, result: 'eloqua' },
+  { regex: /substack/i, result: 'substack' },
+  { regex: /line/i, result: 'line' },
+  { regex: /yext/i, result: 'yext' },
+  { regex: /teads/i, result: 'teads' },
+  { regex: /yandex/i, result: 'yandex' },
+  { regex: /baidu/i, result: 'baidu' },
+  { regex: /amazon|ctv/i, result: 'amazon' },
+  { regex: /direct/i, result: 'direct' },
+];
+
+/* is the vendor paid or owned */
+const vendorTypeLookup = {
+  yext: 'paid',
+  reddit: 'paid',
+  tiktok: 'paid',
+  amazon: 'paid',
+  direct: 'earned',
+};
+
+const categoryClassifications = [
+  { regex: /search|sem|sea$/i, result: 'search' },
+  { regex: /display|programmatic|banner|gdn|dbm/i, result: 'display' },
+  { regex: /video|dv360|tv/i, result: 'video' },
+  { regex: /email|newsletter/i, result: 'email' },
+  { regex: /social|bio/i, result: 'social' },
+  { regex: /affiliate/i, result: 'affiliate' },
+  { regex: /local|gmb/i, result: 'local' },
+  { regex: /sms/i, result: 'sms' },
+  { regex: /qr/i, result: 'qr' },
+  { regex: /push/i, result: 'push' },
+  { regex: /print/i, result: 'print' },
+  { regex: /web/i, result: 'web' },
+];
+
+const paidOwnedClassifications = [
+  { regex: /cpc|ppc|paid|cpm|cpv|banner|display|programmatic|affiliate|^sea$|ads|dv360/i, result: 'paid' },
+  // "organic" is treated as "owned" as it is not paid and not earned
+  // (noone puts UTM tags on real organic traffic)
+  { regex: /email|newsletter|hs_email|organic|sms|qr|qrcode|print|website|web|linkin.bio/i, result: 'owned' },
+  { regex: /push/i, result: 'owned' },
+  { regex: /gmb/i, result: '' },
+  // { regex: /social/i, result: 'earned' },
+];
+
+const vendorCategoryLookup = {
+  google: 'search',
+  bing: 'search',
+  yahoo: 'search',
+  facebook: 'social',
+  instagram: 'social',
+  linkedin: 'social',
+  x: 'social',
+  snapchat: 'social',
+  pinterest: 'social',
+  reddit: 'social',
+  youtube: 'video',
+  spotify: 'display',
+  yext: 'local',
+  line: 'social',
+  substack: 'email',
+  outbrain: 'display',
+  taboola: 'display',
+  criteo: 'display',
+  eloqua: 'email',
+  microsoft: 'display',
+  marketo: 'email',
+  tiktok: 'video',
+  amazon: 'display',
+  yandex: 'search',
+  baidu: 'search',
+  direct: 'direct',
+};
+
+const categoryTypeLookup = {
+  search: 'paid',
+  display: 'paid',
+  affiliate: 'paid',
+  email: 'owned',
+  web: 'owned',
+  sms: 'owned',
+  qr: 'owned',
+  print: 'owned',
+};
+
+export function vendor(origin) {
+  const result = vendorClassifications.find(({ regex }) => regex.test(origin));
+  return result ? result.result : '';
+}
+
+function category(origin, vendorResult) {
+  const categoryResult = categoryClassifications.find(({ regex }) => regex.test(origin));
+
+  if (categoryResult) return categoryResult.result;
+  return vendorCategoryLookup[vendorResult] || '';
+}
+
+function paidowned(origin, vendorResult, categoryResult) {
+  const paidOwnedResult = paidOwnedClassifications.find(({ regex }) => regex.test(origin));
+
+  if (paidOwnedResult) return paidOwnedResult.result;
+  return vendorTypeLookup[vendorResult] || categoryTypeLookup[categoryResult] || '';
+}
+
+export function classifyAcquisition(origin, isPaid = false) {
+  const vendorResult = vendor(origin);
+  const categoryResult = category(origin, vendorResult);
+  const paidOwnedResult = isPaid
+    ? (typeof isPaid === 'string' && isPaid) || 'paid'
+    : paidowned(origin, vendorResult, categoryResult);
+
+  let result = paidOwnedResult;
+  if (categoryResult || vendorResult) {
+    result += `:${categoryResult}`;
+  }
+  if (vendorResult) {
+    result += `:${vendorResult}`;
+  }
+  return result;
+}

tools/oversight/charts/barchart.js

@@ -0,0 +1,194 @@
+import {
+  Chart, LinearScale, registerables,
+  // eslint-disable-next-line import/no-unresolved, import/extensions
+} from 'chartjs';
+import AbstractChart from './chart.js';
+import {
+  toHumanReadable, scoreBundle, cwvInterpolationFn, cssVariable,
+} from '../utils.js';
+
+Chart.register(LinearScale, ...registerables);
+
+const INTERPOLATION_THRESHOLD = 10;
+export default class BarChart extends AbstractChart {
+  defineSeries() {
+    const { dataChunks } = this;
+
+    dataChunks.addSeries('goodCWV', (bundle) => (scoreBundle(bundle) === 'good' ? bundle.weight : undefined));
+    dataChunks.addSeries('poorCWV', (bundle) => (scoreBundle(bundle) === 'poor' ? bundle.weight : undefined));
+    dataChunks.addSeries('niCWV', (bundle) => (scoreBundle(bundle) === 'ni' ? bundle.weight : undefined));
+    dataChunks.addSeries('noCWV', (bundle) => (scoreBundle(bundle) === null ? bundle.weight : undefined));
+
+    // interpolated series
+    dataChunks.addInterpolation(
+      'iGoodCWV', // name of the series
+      ['goodCWV', 'niCWV', 'poorCWV', 'noCWV'], // calculate from these series
+      cwvInterpolationFn('goodCWV'), // interpolation function
+    );
+
+    dataChunks.addInterpolation(
+      'iNiCWV',
+      ['goodCWV', 'niCWV', 'poorCWV', 'noCWV'],
+      cwvInterpolationFn('niCWV'),
+    );
+
+    dataChunks.addInterpolation(
+      'iPoorCWV',
+      ['goodCWV', 'niCWV', 'poorCWV', 'noCWV'],
+      cwvInterpolationFn('poorCWV'),
+    );
+
+    dataChunks.addInterpolation(
+      'iNoCWV',
+      ['goodCWV', 'niCWV', 'poorCWV', 'noCWV'],
+      ({
+        goodCWV, niCWV, poorCWV, noCWV,
+      }) => {
+        const valueCount = goodCWV.count + niCWV.count + poorCWV.count;
+        if (valueCount < INTERPOLATION_THRESHOLD) {
+          // not enough data to interpolate the other values, so
+          // we report as if there are no CWV at all
+          const totalWeight = goodCWV.weight + niCWV.weight + poorCWV.weight + noCWV.weight;
+          return totalWeight;
+        }
+        return 0;
+      },
+    );
+  }
+
+  async draw() {
+    let params = new URL(window.location.href).searchParams;
+
+    if (this.dataChunks.filtered.length < 1000) {
+      this.elems.lowDataWarning.ariaHidden = 'false';
+    } else {
+      this.elems.lowDataWarning.ariaHidden = 'true';
+    }
+
+    this.defineSeries();
+
+    if (!params.get('drilldown')) {
+      const u = new URL(window.location.href);
+      u.searchParams.set('drilldown', 'url');
+      window.history.replaceState({}, '', u);
+      params = u.searchParams;
+    }
+
+    const drilldown = params.get('drilldown');
+
+    const drilldowns = {
+      url: (bundle) => bundle.domain || bundle.url,
+      'click.source': (bundle) => bundle.events
+        .filter((event) => event.checkpoint === 'click')
+        .filter((event) => event.source)
+        .map((event) => event.source),
+      'click.target': (bundle) => bundle.events
+        .filter((event) => event.checkpoint === 'click')
+        .filter((event) => event.target)
+        .map((event) => event.target),
+    };
+
+    const drilldownFn = (bundle) => (typeof drilldowns[drilldown] === 'function'
+      ? drilldowns[drilldown](bundle)
+      : bundle.events
+        .filter((event) => event.checkpoint === drilldown.split('.')[0])
+        .map((event) => event[drilldown.split('.')[1]])
+        .filter((value) => value));
+
+    this.dataChunks.group((bundle) => drilldownFn(bundle));
+    const topgroups = Object.entries(this.dataChunks.aggregates)
+      .sort(([, a], [, b]) => b.pageViews.sum - a.pageViews.sum)
+      .slice(0, 30);
+    this.chart.data.labels = topgroups.map(([path]) => path);
+
+    // clear the data
+    this.chart.data.datasets.forEach((dataset) => {
+      dataset.data = [];
+    });
+    // set new data
+    this.chart.data.labels.forEach((path) => {
+      this.chart.data.datasets[0].data.push(this.dataChunks.aggregates[path].iNoCWV.weight);
+      this.chart.data.datasets[1].data.push(this.dataChunks.aggregates[path].iGoodCWV.weight);
+      this.chart.data.datasets[2].data.push(this.dataChunks.aggregates[path].iNiCWV.weight);
+      this.chart.data.datasets[3].data.push(this.dataChunks.aggregates[path].iPoorCWV.weight);
+    });
+
+    this.chart.update();
+  }
+
+  render() {
+    // eslint-disable-next-line no-undef
+    this.chart = new Chart(this.elems.canvas, {
+      type: 'bar',
+      data: {
+        labels: [],
+        datasets: [{
+          label: 'No CVW',
+          backgroundColor: cssVariable('--spectrum-gray-600'),
+          data: [],
+        },
+        {
+          label: 'Good',
+          backgroundColor: cssVariable('--spectrum-green-600'),
+          data: [],
+        },
+        {
+          label: 'Needs Improvement',
+          backgroundColor: cssVariable('--spectrum-orange-600'),
+          data: [],
+        },
+        {
+          label: 'Poor',
+          backgroundColor: cssVariable('--spectrum-red-600'),
+          data: [],
+        }],
+      },
+      options: {
+        indexAxis: 'y',
+        maintainAspectRatio: false,
+        plugins: {
+          legend: {
+            display: false,
+          },
+          customCanvasBackgroundColor: {
+            color: 'white',
+          },
+        },
+        interaction: {
+          mode: 'y',
+        },
+        animation: {
+          duration: 300,
+        },
+        datasets: {
+          bar: {
+            barPercentage: 1,
+            categoryPercentage: 0.9,
+            borderSkipped: false,
+            borderRadius: {
+              topLeft: 3,
+              topRight: 3,
+              bottomLeft: 3,
+              bottomRight: 3,
+            },
+          },
+        },
+        responsive: true,
+        scales: {
+          x: {
+            stacked: true,
+            ticks: {
+              callback: (value) => toHumanReadable(value),
+            },
+          },
+          y: {
+            stacked: true,
+            grid: {
+              display: false,
+            },
+          },
+        },
+      },
+    });
+  }
+}

tools/oversight/charts/chart.js

@@ -0,0 +1,24 @@
+export default class AbstractChart {
+  constructor(dataChunks, elems) {
+    this.chartConfig = {};
+    this.dataChunks = dataChunks;
+    this.elems = elems;
+    this.chart = {};
+  }
+
+  set config(config) {
+    this.chartConfig = config;
+  }
+
+  get config() {
+    return this.chartConfig || {};
+  }
+
+  render() {
+    throw new Error('render method must be implemented', this);
+  }
+
+  async draw() {
+    throw new Error('draw method must be implemented', this);
+  }
+}