diff --git a/oss_fuzz_integration/oss-fuzz-patches.diff b/oss_fuzz_integration/oss-fuzz-patches.diff index 9ab50683..0b80827f 100644 --- a/oss_fuzz_integration/oss-fuzz-patches.diff +++ b/oss_fuzz_integration/oss-fuzz-patches.diff @@ -17,45 +17,6 @@ index 4fa7a9100..810ac5608 100644 +COPY frontends /fuzz-introspector/frontends + CMD ["compile"] -diff --git a/infra/base-images/base-builder/compile b/infra/base-images/base-builder/compile -index 27a335f3b..2a9881a0c 100755 ---- a/infra/base-images/base-builder/compile -+++ b/infra/base-images/base-builder/compile -@@ -207,6 +207,15 @@ if [ "$SANITIZER" = "introspector" ]; then - ln -sf /usr/local/bin/llvm-ar /usr/bin/ar - ln -sf /usr/local/bin/llvm-nm /usr/bin/nm - ln -sf /usr/local/bin/llvm-ranlib /usr/bin/ranlib -+ -+ # First, perform an initial light analysis -+ apt-get install -y libjpeg-dev zlib1g-dev libyaml-dev -+ python3 -m pip install --upgrade pip setuptools -+ python3 -m pip install cxxfilt pyyaml beautifulsoup4 lxml soupsieve -+ python3 -m pip install --prefer-binary matplotlib -+ -+ python3 /fuzz-introspector/src/main.py light -+ rsync -avu --delete "$SRC/inspector/" "$OUT/inspector" - fi - - echo "---------------------------------------------------------------" -@@ -260,10 +269,6 @@ if [ "$SANITIZER" = "introspector" ]; then - unset CXXFLAGS - unset CFLAGS - export G_ANALYTICS_TAG="G-8WTFM1Y62J" -- apt-get install -y libjpeg-dev zlib1g-dev libyaml-dev -- python3 -m pip install --upgrade pip setuptools -- python3 -m pip install cxxfilt pyyaml beautifulsoup4 lxml soupsieve -- python3 -m pip install --prefer-binary matplotlib - - if [ "$FUZZING_LANGUAGE" = "jvm" ]; then - echo "GOING jvm route" -@@ -328,6 +333,6 @@ if [ "$SANITIZER" = "introspector" ]; then - REPORT_ARGS="$REPORT_ARGS --correlation_file=exe_to_fuzz_introspector_logs.yaml" - python3 /fuzz-introspector/src/main.py report $REPORT_ARGS - -- cp -rf $SRC/inspector $OUT/inspector -+ rsync -avu --delete "$SRC/inspector/" "$OUT/inspector" - fi - fi diff --git a/infra/base-images/base-clang/Dockerfile b/infra/base-images/base-clang/Dockerfile index 757ea3de5..5be6bd30a 100644 --- a/infra/base-images/base-clang/Dockerfile diff --git a/tools/web-fuzzing-introspection/app/static/assets/db/oss_fuzz.py b/tools/web-fuzzing-introspection/app/static/assets/db/oss_fuzz.py index ab67f31b..1cc577c6 100644 --- a/tools/web-fuzzing-introspection/app/static/assets/db/oss_fuzz.py +++ b/tools/web-fuzzing-introspection/app/static/assets/db/oss_fuzz.py @@ -63,6 +63,72 @@ def get_introspector_project_all_files(project_name, datestr): datestr) + "all-files.json" +def get_introspector_light_pairs_url(project_name, datestr): + return get_introspector_report_url_base(project_name, + datestr) + "light/all-files.json" + + +def extract_introspector_light_all_pairs(project_name, date_str): + """Gets the list of pairs from introspector light""" + debug_data_url = get_introspector_light_pairs_url( + project_name, date_str.replace("-", "")) + try: + raw_introspector_json_request = requests.get(debug_data_url, + timeout=10) + except: + return [] + try: + all_pairs = json.loads(raw_introspector_json_request.text) + except: + return [] + + return all_pairs + + +def get_introspector_light_tests_url(project_name, datestr): + return get_introspector_report_url_base(project_name, + datestr) + "light/all_tests.json" + + +def extract_introspector_light_all_tests(project_name, date_str): + """Gets the list of test files from light""" + debug_data_url = get_introspector_light_tests_url( + project_name, date_str.replace("-", "")) + try: + raw_introspector_json_request = requests.get(debug_data_url, + timeout=10) + except: + return [] + try: + all_tests = json.loads(raw_introspector_json_request.text) + except: + return [] + + return all_tests + + +def get_introspector_light_all_files_url(project_name, datestr): + return get_introspector_report_url_base(project_name, + datestr) + "light/all_files.json" + + +def extract_introspector_light_all_files(project_name, date_str): + """Gets the list of all files from light""" + debug_data_url = get_introspector_light_all_files_url( + project_name, date_str.replace("-", "")) + try: + raw_introspector_json_request = requests.get(debug_data_url, + timeout=10) + except: + return [] + try: + all_urls = json.loads(raw_introspector_json_request.text) + except: + return [] + + return all_urls + + def get_introspector_type_map_url_summary(project_name, datestr): return get_introspector_report_url_base( project_name, datestr) + "all-friendly-debug-types.json" diff --git a/tools/web-fuzzing-introspection/app/static/assets/db/web_db_creator_from_summary.py b/tools/web-fuzzing-introspection/app/static/assets/db/web_db_creator_from_summary.py index e49a9841..754094bf 100644 --- a/tools/web-fuzzing-introspection/app/static/assets/db/web_db_creator_from_summary.py +++ b/tools/web-fuzzing-introspection/app/static/assets/db/web_db_creator_from_summary.py @@ -572,6 +572,19 @@ def extract_project_data(project_name, date_str, should_include_details, # Save the report save_fuzz_introspector_report(introspector_report, project_name, date_str) + light_test_files = oss_fuzz.extract_introspector_light_all_tests( + project_name, date_str.replace("-", "")) + light_all_files = oss_fuzz.extract_introspector_light_all_files( + project_name, date_str.replace("-", "")) + light_all_pairs = oss_fuzz.extract_introspector_light_all_pairs( + project_name, date_str.replace("-", "")) + + light_report = { + 'test-files': light_test_files, + 'all-files': light_all_files, + 'all-pairs': light_all_pairs, + } + # Get debug data if collect_debug_info and should_include_details: debug_report = oss_fuzz.extract_introspector_debug_info( @@ -701,6 +714,7 @@ def extract_project_data(project_name, date_str, should_include_details, 'introspector-data': introspector_data_dict, 'fuzzer-count': amount_of_fuzzers, 'project_repository': project_repository, + 'light-introspector': light_report, } dictionary_key = '%s###%s' % (project_name, date_str)