web: load light data from cloud

oss_fuzz_integration/oss-fuzz-patches.diff

@@ -17,45 +17,6 @@ index 4fa7a9100..810ac5608 100644
 +COPY frontends /fuzz-introspector/frontends
 +
  CMD ["compile"]
-diff --git a/infra/base-images/base-builder/compile b/infra/base-images/base-builder/compile
-index 27a335f3b..2a9881a0c 100755
---- a/infra/base-images/base-builder/compile
-+++ b/infra/base-images/base-builder/compile
-@@ -207,6 +207,15 @@ if [ "$SANITIZER" = "introspector" ]; then
- ln -sf /usr/local/bin/llvm-ar /usr/bin/ar
- ln -sf /usr/local/bin/llvm-nm /usr/bin/nm
- ln -sf /usr/local/bin/llvm-ranlib /usr/bin/ranlib
-+
-+ # First, perform an initial light analysis
-+ apt-get install -y libjpeg-dev zlib1g-dev libyaml-dev
-+ python3 -m pip install --upgrade pip setuptools
-+ python3 -m pip install cxxfilt pyyaml beautifulsoup4 lxml soupsieve
-+ python3 -m pip install --prefer-binary matplotlib
-+
-+ python3 /fuzz-introspector/src/main.py light
-+ rsync -avu --delete "$SRC/inspector/" "$OUT/inspector"
- fi
-
- echo "---------------------------------------------------------------"
-@@ -260,10 +269,6 @@ if [ "$SANITIZER" = "introspector" ]; then
- unset CXXFLAGS
- unset CFLAGS
- export G_ANALYTICS_TAG="G-8WTFM1Y62J"
-- apt-get install -y libjpeg-dev zlib1g-dev libyaml-dev
-- python3 -m pip install --upgrade pip setuptools
-- python3 -m pip install cxxfilt pyyaml beautifulsoup4 lxml soupsieve
-- python3 -m pip install --prefer-binary matplotlib
-
- if [ "$FUZZING_LANGUAGE" = "jvm" ]; then
- echo "GOING jvm route"
-@@ -328,6 +333,6 @@ if [ "$SANITIZER" = "introspector" ]; then
- REPORT_ARGS="$REPORT_ARGS --correlation_file=exe_to_fuzz_introspector_logs.yaml"
- python3 /fuzz-introspector/src/main.py report $REPORT_ARGS
-
-- cp -rf $SRC/inspector $OUT/inspector
-+ rsync -avu --delete "$SRC/inspector/" "$OUT/inspector"
- fi
- fi
 diff --git a/infra/base-images/base-clang/Dockerfile b/infra/base-images/base-clang/Dockerfile
 index 757ea3de5..5be6bd30a 100644
 --- a/infra/base-images/base-clang/Dockerfile

tools/web-fuzzing-introspection/app/static/assets/db/oss_fuzz.py

@@ -63,6 +63,72 @@ def get_introspector_project_all_files(project_name, datestr):
  datestr) + "all-files.json"
 
 
+def get_introspector_light_pairs_url(project_name, datestr):
+ return get_introspector_report_url_base(project_name,
+ datestr) + "light/all-files.json"
+
+
+def extract_introspector_light_all_pairs(project_name, date_str):
+ """Gets the list of pairs from introspector light"""
+ debug_data_url = get_introspector_light_pairs_url(
+ project_name, date_str.replace("-", ""))
+ try:
+ raw_introspector_json_request = requests.get(debug_data_url,
+ timeout=10)
+ except:
+ return []
+ try:
+ all_pairs = json.loads(raw_introspector_json_request.text)
+ except:
+ return []
+
+ return all_pairs
+
+
+def get_introspector_light_tests_url(project_name, datestr):
+ return get_introspector_report_url_base(project_name,
+ datestr) + "light/all_tests.json"
+
+
+def extract_introspector_light_all_tests(project_name, date_str):
+ """Gets the list of test files from light"""
+ debug_data_url = get_introspector_light_tests_url(
+ project_name, date_str.replace("-", ""))
+ try:
+ raw_introspector_json_request = requests.get(debug_data_url,
+ timeout=10)
+ except:
+ return []
+ try:
+ all_tests = json.loads(raw_introspector_json_request.text)
+ except:
+ return []
+
+ return all_tests
+
+
+def get_introspector_light_all_files_url(project_name, datestr):
+ return get_introspector_report_url_base(project_name,
+ datestr) + "light/all_files.json"
+
+
+def extract_introspector_light_all_files(project_name, date_str):
+ """Gets the list of all files from light"""
+ debug_data_url = get_introspector_light_all_files_url(
+ project_name, date_str.replace("-", ""))
+ try:
+ raw_introspector_json_request = requests.get(debug_data_url,
+ timeout=10)
+ except:
+ return []
+ try:
+ all_urls = json.loads(raw_introspector_json_request.text)
+ except:
+ return []
+
+ return all_urls
+
+
 def get_introspector_type_map_url_summary(project_name, datestr):
  return get_introspector_report_url_base(
  project_name, datestr) + "all-friendly-debug-types.json"

tools/web-fuzzing-introspection/app/static/assets/db/web_db_creator_from_summary.py

@@ -572,6 +572,19 @@ def extract_project_data(project_name, date_str, should_include_details,
  # Save the report
  save_fuzz_introspector_report(introspector_report, project_name, date_str)
 
+ light_test_files = oss_fuzz.extract_introspector_light_all_tests(
+ project_name, date_str.replace("-", ""))
+ light_all_files = oss_fuzz.extract_introspector_light_all_files(
+ project_name, date_str.replace("-", ""))
+ light_all_pairs = oss_fuzz.extract_introspector_light_all_pairs(
+ project_name, date_str.replace("-", ""))
+
+ light_report = {
+ 'test-files': light_test_files,
+ 'all-files': light_all_files,
+ 'all-pairs': light_all_pairs,
+ }
+
  # Get debug data
  if collect_debug_info and should_include_details:
  debug_report = oss_fuzz.extract_introspector_debug_info(
@@ -701,6 +714,7 @@ def extract_project_data(project_name, date_str, should_include_details,
  'introspector-data': introspector_data_dict,
  'fuzzer-count': amount_of_fuzzers,
  'project_repository': project_repository,
+ 'light-introspector': light_report,
  }
 
  dictionary_key = '%s###%s' % (project_name, date_str)