Merge pull request #639 from egraphs-good/oflatt-compile-times

Add summary table and compile times to nightly

infra/generate_line_counts.py

@@ -79,8 +79,8 @@ def detailed_linecount_table():
 def round_fmt(v):
     return "{:.3f}".format(round(v, 3))
 
-# given a list of integers (cycles taken for each run)
-# return the mean of the cycles
+# given a list of numbers, compute the mean
+# numbers may be floating-point or integers (for cycles)
 def mean_cycles(cycles):
     return sum(cycles) / len(cycles)
 

infra/nightly-resources/chart.js

@@ -16,7 +16,7 @@ const BASELINE_MODE = "llvm-O0";
 
 // Given a list of integers, compute the mean
 // number of cycles
-function mean_cycles(cycles) {
+function mean(cycles) {
   return cycles.reduce((a, b) => a + b, 0) / cycles.length;
 }
 
@@ -62,18 +62,22 @@ function getEntry(benchmark, runMode) {
   }
 }
 
+function speedup(entry, baseline) {
+  const baseV = mean(baseline["cycles"]);
+  const expV = mean(entry["cycles"]);
+  // If you change this, also change the displayed formula in index.html
+  return baseV / expV;
+}
+
 function getValue(entry) {
   if (GLOBAL_DATA.chart.mode === "absolute") {
-    return mean_cycles(entry["cycles"]);
+    return mean(entry["cycles"]);
   } else if (GLOBAL_DATA.chart.mode === "speedup") {
     const baseline = getEntry(entry.benchmark, BASELINE_MODE);
     if (!baseline) {
       addWarning(`No speedup baseline for ${benchmark}`);
     }
-    const baseV = mean_cycles(baseline["cycles"]);
-    const expV = mean_cycles(entry["cycles"]);
-    // If you change this, also change the displayed formula in index.html
-    return baseV / expV;
+    return speedup(entry, baseline);
   } else {
     throw new Error(`unknown chart mode ${GLOBAL_DATA.chart.mode}`);
   }
@@ -90,8 +94,8 @@ function getError(entry) {
       addWarning(`No speedup baseline for ${benchmark}`);
     }
 
-    const baseV = mean_cycles(baseline["cycles"]);
-    const expV = mean_cycles(entry["cycles"]);
+    const baseV = mean(baseline["cycles"]);
+    const expV = mean(entry["cycles"]);
     const baseStd = stddev_cycles(baseline["cycles"]);
     const expStd = stddev_cycles(entry["cycles"]);
 

infra/nightly-resources/data.js

@@ -10,7 +10,15 @@ async function fetchText(url) {
   return data;
 }
 
-function getBaselineCycles(benchmark, runMethod) {
+function getRow(benchmark, runMethod) {
+  return GLOBAL_DATA.currentRun.find(
+    (row) => row.benchmark === benchmark && row.runMethod === runMethod,
+  );
+}
+
+// Get the row for the comparison branch
+// for the given benchmark and run method
+function getComparison(benchmark, runMethod) {
   const baselineData =
     GLOBAL_DATA.baselineRun?.filter((o) => o.benchmark === benchmark) || [];
   if (baselineData.length === 0) {
@@ -24,7 +32,7 @@ function getBaselineCycles(benchmark, runMethod) {
         `Baseline had multiple entries for ${benchmark} ${runMethod}`,
       );
     } else {
-      return baseline[0]["cycles"];
+      return baseline[0];
     }
   }
 }
@@ -52,23 +60,82 @@ function getBrilPathForBenchmark(benchmark) {
   return o.metadata.path;
 }
 
+// calculates the geometric mean over a list of ratios
+function geometricMean(values) {
+  return Math.pow(
+    values.reduce((a, b) => a * b, 1),
+    1 / values.length,
+  );
+}
+
+function getOverallStatistics() {
+  // generate one row per treatment...
+  const result = [];
+  for (const treatment of treatments) {
+    // calculate geometric mean of speedups
+    const speedups = [];
+    // for each benchmark, calculate the speedup
+    for (const benchmark of GLOBAL_DATA.enabledBenchmarks) {
+      const row = getRow(benchmark, treatment);
+      const baseline = getRow(benchmark, BASELINE_MODE);
+      if (row && baseline) {
+        speedups.push(speedup(row, baseline));
+      }
+    }
+
+    const eggcc_compile_times = [];
+    const llvm_compile_times = [];
+    for (const benchmark of GLOBAL_DATA.enabledBenchmarks) {
+      const row = getRow(benchmark, treatment);
+      eggcc_compile_times.push(row.eggccCompileTimeSecs);
+      llvm_compile_times.push(row.llvmCompileTimeSecs);
+    }
+
+    // calculate the geometric mean of the speedups
+    result.push({
+      runMethod: treatment,
+      geoMeanSpeedup: tryRound(geometricMean(speedups)),
+      meanEggccCompileTimeSecs: tryRound(mean(eggcc_compile_times)),
+      meanLlvmCompileTimeSecs: tryRound(mean(llvm_compile_times)),
+    });
+  }
+  return result;
+}
+
 function getDataForBenchmark(benchmark) {
   const executions = GLOBAL_DATA.currentRun
     ?.filter((row) => row.benchmark === benchmark)
     .map((row) => {
-      const baselineCycles = getBaselineCycles(row.benchmark, row.runMethod);
+      const baseline = getRow(benchmark, BASELINE_MODE);
+      const comparisonCycles = getComparison(
+        row.benchmark,
+        row.runMethod,
+      )?.cycles;
       const cycles = row["cycles"];
       const rowData = {
         runMethod: row.runMethod,
-        mean: { class: "", value: tryRound(mean_cycles(cycles)) },
-        meanVsBaseline: getDifference(cycles, baselineCycles, mean_cycles),
+        mean: { class: "", value: tryRound(mean(cycles)) },
+        meanVsBaseline: getDifference(cycles, comparisonCycles, mean),
         min: { class: "", value: tryRound(min_cycles(cycles)) },
-        minVsBaseline: getDifference(cycles, baselineCycles, min_cycles),
+        minVsBaseline: getDifference(cycles, comparisonCycles, min_cycles),
         max: { class: "", value: tryRound(max_cycles(cycles)) },
-        maxVsBaseline: getDifference(cycles, baselineCycles, max_cycles),
+        maxVsBaseline: getDifference(cycles, comparisonCycles, max_cycles),
         median: { class: "", value: tryRound(median_cycles(cycles)) },
-        medianVsBaseline: getDifference(cycles, baselineCycles, median_cycles),
+        medianVsBaseline: getDifference(
+          cycles,
+          comparisonCycles,
+          median_cycles,
+        ),
         stddev: { class: "", value: tryRound(median_cycles(cycles)) },
+        eggccCompileTimeSecs: {
+          class: "",
+          value: tryRound(row.eggccCompileTimeSecs),
+        },
+        llvmCompileTimeSecs: {
+          class: "",
+          value: tryRound(row.llvmCompileTimeSecs),
+        },
+        speedup: { class: "", value: tryRound(speedup(row, baseline)) },
       };
       if (shouldHaveLlvm(row.runMethod)) {
         rowData.runMethod = `<a target="_blank" rel="noopener noreferrer" href="llvm.html?benchmark=${benchmark}&runmode=${row.runMethod}">${row.runMethod}</a>`;

infra/nightly-resources/index.html

@@ -18,7 +18,6 @@
 
 <body onload="load_index()">
   <h2>Nightlies</h2>
-  <h2>Profile</h2>
 
   <div>
     <button type="button" class="collapsible" onclick="toggle(this, '\u25B6 Show Chart', '\u25BC Hide Chart')">&#x25BC; Hide Chart</button>
@@ -67,6 +66,11 @@ <h2>Profile</h2>
     </div>
   </div>
 
+  <h2>Overall Stats</h2>
+  <t>Warning: compile-time statistics are not very trustworthy in parallel mode.</t>
+  <div id="overall-stats-table"></div>
+
+  <h2>Detailed Table</h2>
   <div id="profile"></div>
   <h2>Raw</h2>
   <p>

infra/nightly-resources/index.js

@@ -48,6 +48,11 @@ function refreshView() {
 
   document.getElementById("profile").innerHTML = ConvertJsonToTable(tableData);
 
+  // fill in the overall stats table
+  const overallStats = getOverallStatistics();
+  const overallTable = document.getElementById("overall-stats-table");
+  overallTable.innerHTML = ConvertJsonToTable(overallStats);
+
   renderWarnings();
   refreshChart();
 }

infra/profile.py

@@ -27,29 +27,32 @@
 # Where to write intermediate files that should be cleaned up at the end of this script
 TMP_DIR = "tmp"
 
-def get_eggcc_options(run_mode, benchmark):
-  llvm_out_dir = f"{DATA_DIR}/llvm/{benchmark}/{run_mode}"
-  match run_mode:
+EGGCC_BINARY = "target/release/eggcc"
+
+
+# returns two strings:
+# the first is the eggcc run mode for optimizing the input bril program
+# the second is the command line arguments for producing an output file using llvm
+def get_eggcc_options(benchmark):
+  match benchmark.treatment:
     case "rvsdg-round-trip-to-executable":
-      return f'--run-mode rvsdg-round-trip-to-executable --llvm-output-dir {llvm_out_dir}'
-    case "cranelift-O3":
-      return f'--run-mode cranelift --optimize-egglog false --optimize-brilift true'
+      return (f'rvsdg-round-trip',  f'--run-mode llvm --optimize-egglog false --optimize-bril-llvm O0')
     case "llvm-O0":
-      return f'--run-mode llvm --optimize-egglog false --optimize-bril-llvm O0 --llvm-output-dir {llvm_out_dir}'
+      return (f'parse', f'--run-mode llvm --optimize-egglog false --optimize-bril-llvm O0')
     case "llvm-O1":
-      return f'--run-mode llvm --optimize-egglog false --optimize-bril-llvm O1 --llvm-output-dir {llvm_out_dir}'
+      return (f'parse', f'--run-mode llvm --optimize-egglog false --optimize-bril-llvm O1')
     case "llvm-O2":
-      return f'--run-mode llvm --optimize-egglog false --optimize-bril-llvm O2 --llvm-output-dir {llvm_out_dir}'
+      return (f'parse', f'--run-mode llvm --optimize-egglog false --optimize-bril-llvm O2')
     case "llvm-O3":
-      return f'--run-mode llvm --optimize-egglog false --optimize-bril-llvm O3 --llvm-output-dir {llvm_out_dir}'
-    case "llvm-O0-eggcc":
-      return f'--run-mode llvm --optimize-egglog true --optimize-bril-llvm O0 --llvm-output-dir {llvm_out_dir}'
+      return (f'parse', f'--run-mode llvm --optimize-egglog false --optimize-bril-llvm O3')
     case "llvm-O0-eggcc-sequential":
-      return f'--run-mode llvm --optimize-egglog true --optimize-bril-llvm O0 --llvm-output-dir {llvm_out_dir} --eggcc-schedule sequential'
+      return (f'optimize --eggcc-schedule sequential', f'--run-mode llvm --optimize-egglog false --optimize-bril-llvm O0')
+    case "llvm-O0-eggcc":
+      return (f'optimize', f'--run-mode llvm --optimize-egglog false --optimize-bril-llvm O0')
     case "llvm-O3-eggcc":
-      return f'--run-mode llvm --optimize-egglog true --optimize-bril-llvm O3 --llvm-output-dir {llvm_out_dir}'
+      return (f'optimize', f'--run-mode llvm --optimize-egglog false --optimize-bril-llvm O3')
     case _:
-      raise Exception("Unexpected run mode: " + run_mode)
+      raise Exception("Unexpected run mode: " + benchmark.treatment)
 
 
 class Benchmark:
@@ -69,16 +72,41 @@ def setup_benchmark(name):
   profile_dir = benchmark_profile_dir(name)
   os.mkdir(profile_dir)
 
+
+# runs optimization on the benchmark, a dictionary with
+# a name and treatment (and index for printing)
+# returns a dictionary with the path to the optimized binary,
+# eggcc compile time, and llvm compile time
 def optimize(benchmark):
   print(f'[{benchmark.index}/{benchmark.total}] Optimizing {benchmark.name} with {benchmark.treatment}')
   profile_dir = benchmark_profile_dir(benchmark.name)
-  cmd = f'cargo run --release {benchmark.path} --add-timing {get_eggcc_options(benchmark.treatment, benchmark.name)} -o {profile_dir}/{benchmark.treatment}'
-  print(f'Running: {cmd}', flush=True)
-  start = time.time()
-  process = subprocess.run(cmd, shell=True)
+  optimized_bril_file = f'{profile_dir}/{benchmark.treatment}-optimized.bril'
+
+  # get the commands we need to run
+  (eggcc_run_mode, llvm_args) = get_eggcc_options(benchmark)
+  llvm_out_dir = f"{DATA_DIR}/llvm/{benchmark.name}/{benchmark.treatment}"
+
+  cmd1 = f'{EGGCC_BINARY} {benchmark.path} --run-mode {eggcc_run_mode}'
+  cmd2 = f'{EGGCC_BINARY} {optimized_bril_file} --add-timing {llvm_args} -o {profile_dir}/{benchmark.treatment} --llvm-output-dir {llvm_out_dir}'
+
+  print(f'Running: {cmd1}', flush=True)
+  start_eggcc = time.time()
+  process = subprocess.run(cmd1, shell=True, capture_output=True, text=True)
   process.check_returncode()
-  end = time.time()
-  return (f"{profile_dir}/{benchmark.treatment}", end-start)
+  end_eggcc = time.time()
+
+  # write the std out to the optimized bril file
+  with open(optimized_bril_file, 'w') as f:
+    f.write(process.stdout)
+
+  print(f'Running: {cmd2}', flush=True)
+  start_llvm = time.time()
+  process2 = subprocess.run(cmd2, shell=True)
+  process2.check_returncode()
+  end_llvm = time.time()
+
+  res = {"path": f"{profile_dir}/{benchmark.treatment}", "eggccCompileTimeSecs": end_eggcc-start_eggcc, "llvmCompileTimeSecs": end_llvm-start_llvm}
+  return res
 
 
 
@@ -131,13 +159,17 @@ def should_have_llvm_ir(runMethod):
   ]
 
 # aggregate all profile info into a single json array.
-def aggregate(compile_times, bench_times, benchmark_metadata):
+def aggregate(compile_data, bench_times, benchmark_metadata):
     res = []
 
-    for path in sorted(compile_times.keys()):
+    for path in sorted(compile_data.keys()):
       name = path.split("/")[-2]
       runMethod = path.split("/")[-1]
-      result = {"runMethod": runMethod, "benchmark": name, "cycles": bench_times[path], "compileTime": compile_times[path], "metadata": benchmark_metadata[name]}
+      result = {"runMethod": runMethod, "benchmark": name, "cycles": bench_times[path], "metadata": benchmark_metadata[name]}
+
+      # add compile time info
+      for key in compile_data[path]:
+        result[key] = compile_data[path][key]
 
       res.append(result)
     return res
@@ -161,6 +193,10 @@ def is_looped(bril_file):
     # remove the files in the directory
     os.system(f"rm -rf {TMP_DIR}/*")
 
+  # build eggcc
+  print("Building eggcc")
+  os.system("cargo build --release")
+
 
   bril_dir, DATA_DIR = os.sys.argv[1:]
   profiles = []
@@ -189,7 +225,7 @@ def is_looped(bril_file):
     setup_benchmark(benchmark_name)
 
 
-  compile_times = {}
+  compile_data = {}
   # get the number of cores on this machine 
   parallelism = os.cpu_count()
 
@@ -198,8 +234,11 @@ def is_looped(bril_file):
     futures = {executor.submit(optimize, benchmark) for benchmark in to_run}
     for future in concurrent.futures.as_completed(futures):
       try:
-        (path, compile_time) = future.result()
-        compile_times[path] = compile_time
+        res = future.result()
+        path = res["path"]
+        # remove the path from compile data
+        res.pop("path")
+        compile_data[path] = res
       except Exception as e:
         print(f"Shutting down executor due to error: {e}")
         executor.shutdown(wait=False, cancel_futures=True)
@@ -233,9 +272,10 @@ def is_looped(bril_file):
       (path, _bench_data) = res
       bench_data[path] = _bench_data
 
-  nightly_data = aggregate(compile_times, bench_data, benchmark_metadata)
+  nightly_data = aggregate(compile_data, bench_data, benchmark_metadata)
   with open(f"{DATA_DIR}/profile.json", "w") as profile:
     json.dump(nightly_data, profile, indent=2)
 
-  # Clean up intermediate files
+  # remove the tmp directory
   os.system(f"rm -rf {TMP_DIR}")
+

infra/update-frontend.sh

@@ -0,0 +1,20 @@
+# smalls script to test changes to the javascript or html files
+# copys javascript and html files to the output directory
+
+
+# determine physical directory of this script
+src="${BASH_SOURCE[0]}"
+while [ -L "$src" ]; do
+  dir="$(cd -P "$(dirname "$src")" && pwd)"
+  src="$(readlink "$src")"
+  [[ $src != /* ]] && src="$dir/$src"
+done
+MYDIR="$(cd -P "$(dirname "$src")" && pwd)"
+TOP_DIR="$MYDIR/.."
+RESOURCE_DIR="$MYDIR/nightly-resources"
+NIGHTLY_DIR="$TOP_DIR/nightly"
+
+echo "Copying resources to $NIGHTLY_DIR/output"
+cp "$RESOURCE_DIR"/* "$NIGHTLY_DIR/output"
+
+cd nightly/output && python3 -m http.server 8002