Merge branch 'ossf:main' into add-mypy-pyright-webapp

scripts/oss-fuzz-gen-e2e/web_run_all.sh

@@ -0,0 +1,80 @@
+#!/bin/bash -eux
+# Copyright 2024 Fuzz Introspector Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+################################################################################
+
+
+ROOT_FI=$PWD/../../
+BASE_DIR=$PWD/workdir
+BENCHMARK_HEURISTICS="${VARIABLE:-far-reach-low-coverage,low-cov-with-fuzz-keyword}"
+OSS_FUZZ_GEN_MODEL=${MODEL}
+VAR_HARNESSES_PER_ORACLE="${HARNESS_PER_ORACLE:-10}"
+VAR_LLM_FIX_LIMIT="${LLM_FIX_LIMIT:-1}"
+PROJECT=${@}
+
+comma_separated=""
+for proj in ${PROJECT}; do
+ echo ${proj}
+ comma_separated="${comma_separated}${proj},"
+done
+comma_separated=${comma_separated::-1}
+
+# Launch virtualenv
+cd ${BASE_DIR}
+. .venv/bin/activate
+
+# Create webserver DB
+echo "[+] Creating the webapp DB"
+cd $ROOT_FI/tools/web-fuzzing-introspection/app/static/assets/db/
+python3 ./web_db_creator_from_summary.py \
+ --since-date=20-04-2023 \
+ --output-dir=$PWD \
+ --input-dir=$PWD \
+ --includes=${comma_separated}
+
+# Start webserver DB 
+echo "Shutting down server in case it's running"
+curl --silent http://localhost:8080/api/shutdown || true
+
+echo "[+] Launching FI webapp"
+cd $ROOT_FI/tools/web-fuzzing-introspection/app/
+FUZZ_INTROSPECTOR_SHUTDOWN=1 python3 ./main.py >> /dev/null &
+
+SECONDS=5
+while true
+do
+ # Checking if exists
+ MSG=$(curl -v --silent 127.0.0.1:8080 2>&1 | grep "Fuzzing" | wc -l)
+ if [[ $MSG > 0 ]]; then
+ echo "Found it"
+ break
+ fi
+ echo "- Waiting for webapp to load. Sleeping ${SECONDS} seconds."
+ sleep ${SECONDS}
+done
+
+# Deactivate
+echo "[+] Running OSS-Fuzz-gen experiment"
+export LLM_FIX_LIMIT=${VAR_LLM_FIX_LIMIT}
+cd $BASE_DIR/oss-fuzz-gen
+./run_all_experiments.py \
+ --model=$OSS_FUZZ_GEN_MODEL \
+ -g ${BENCHMARK_HEURISTICS} \
+ -gp ${comma_separated} \
+ -gm ${VAR_HARNESSES_PER_ORACLE} \
+ -e http://127.0.0.1:8080/api
+
+echo "Shutting down started webserver"
+curl --silent http://localhost:8080/api/shutdown || true

tools/web-fuzzing-introspection/app/main.py

@@ -56,4 +56,4 @@ def create_app():
 if __name__ == "__main__":
  create_app().run(debug=False,
  host="0.0.0.0",
- port=os.environ.get("WEBAPP_PORT", 8080))
+ port=int(os.environ.get("WEBAPP_PORT", '8080')))

tools/web-fuzzing-introspection/app/static/assets/db/oss_fuzz.py

@@ -193,7 +193,7 @@ def extract_new_introspector_constructors(project_name, date_str):
  # Read the introspector artifact
  try:
  raw_introspector_json_request = requests.get(
- introspector_functions_url, timeout=10)
+ introspector_constructor_url, timeout=10)
  introspector_constructors = json.loads(
  raw_introspector_json_request.text)
  except:
@@ -331,3 +331,26 @@ def try_to_get_project_language(project_name):
  project_yaml = yaml.safe_load(r.text)
  return project_yaml['language']
  return "N/A"
+
+
+def try_to_get_project_repository(project_name):
+ if os.path.isdir(constants.OSS_FUZZ_CLONE):
+ local_project_path = os.path.join(constants.OSS_FUZZ_CLONE, "projects",
+ project_name)
+ if os.path.isdir(local_project_path):
+ project_yaml_path = os.path.join(local_project_path,
+ "project.yaml")
+ if os.path.isfile(project_yaml_path):
+ with open(project_yaml_path, "r") as f:
+ project_yaml = yaml.safe_load(f.read())
+ return project_yaml['main_repo']
+ else:
+ proj_yaml_url = 'https://raw.githubusercontent.com/google/oss-fuzz/master/projects/%s/project.yaml' % (
+ project_name)
+ try:
+ r = requests.get(proj_yaml_url, timeout=10)
+ except:
+ return "N/A"
+ project_yaml = yaml.safe_load(r.text)
+ return project_yaml['main_repo']
+ return "N/A"

tools/web-fuzzing-introspection/app/static/assets/db/web_db_creator_from_summary.py

@@ -336,6 +336,11 @@ def extract_local_project_data(project_name, oss_fuzz_path,
 
  # Dump things we dont want to accummulate.
  #save_branch_blockers(branch_pairs, project_name)
+ try:
+ project_repository = oss_fuzz.try_to_get_project_repository(
+ project_name)
+ except:
+ project_repository = 'N/A'
 
  introspector_data_dict = {
  "introspector_report_url": 'introspector_url',
@@ -365,6 +370,7 @@ def extract_local_project_data(project_name, oss_fuzz_path,
  'coverage-data': code_coverage_data_dict,
  'introspector-data': introspector_data_dict,
  'fuzzer-count': amount_of_fuzzers,
+ 'project_repository': project_repository,
  }
 
  dictionary_key = '%s###%s' % (project_name, '')
@@ -408,6 +414,12 @@ def extract_project_data(project_name, date_str, should_include_details,
  # Default set to c++ as this is OSS-Fuzz's default.
  project_language = 'c++'
 
+ try:
+ project_repository = oss_fuzz.try_to_get_project_repository(
+ project_name)
+ except:
+ project_repository = 'N/A'
+
  collect_debug_info = project_language in {'c', 'c++'}
 
  # Extract code coverage and introspector reports.
@@ -587,6 +599,7 @@ def extract_project_data(project_name, date_str, should_include_details,
  'coverage-data': code_coverage_data_dict,
  'introspector-data': introspector_data_dict,
  'fuzzer-count': amount_of_fuzzers,
+ 'project_repository': project_repository,
  }
 
  dictionary_key = '%s###%s' % (project_name, date_str)

tools/web-fuzzing-introspection/app/webapp/__init__.py

@@ -5,19 +5,9 @@
 from . import models
 
 
-def is_db_valid():
- db_timestamps_file = os.path.join(
- os.path.dirname(__file__), "../static/assets/db/db-timestamps.json")
- if not os.path.isfile(db_timestamps_file):
- return False
- return True
-
-
 def load_db():
  """Loads the database"""
  print("Loading db")
- if not is_db_valid():
- update_db()
 
  db_timestamps_file = os.path.join(
  os.path.dirname(__file__), "../static/assets/db/db-timestamps.json")
@@ -92,7 +82,8 @@ def load_db():
  date=project_timestamp['date'],
  coverage_data=project_timestamp['coverage-data'],
  introspector_data=project_timestamp['introspector-data'],
- fuzzer_count=project_timestamp['fuzzer-count']))
+ fuzzer_count=project_timestamp['fuzzer-count'],
+ project_repository=project_timestamp['project_repository']))
 
  introspector_data = project_timestamp.get('introspector-data', None)
  if introspector_data is None:

tools/web-fuzzing-introspection/app/webapp/data_storage.py

@@ -1,30 +1,30 @@
 # Auto-generated
 #from app.site.models import *
 
-from typing import List
+from typing import List, Dict, Any
 
 import os
 import json
 
 from .models import *
 
-PROJECT_TIMESTAMPS = []
+PROJECT_TIMESTAMPS: List[ProjectTimestamp] = []
 
-DB_TIMESTAMPS = []
+DB_TIMESTAMPS: List[DBTimestamp] = []
 
-PROJECTS = []
+PROJECTS: List[Project] = []
 
-FUNCTIONS = []
+FUNCTIONS: List[Function] = []
 
-CONSTRUCTORS = []
+CONSTRUCTORS: List[Function] = []
 
-BLOCKERS = []
+BLOCKERS: List[BranchBlocker] = []
 
 BUILD_STATUS: List[BuildStatus] = []
 
-PROJECT_DEBUG_DATA = []
+PROJECT_DEBUG_DATA: List[DebugStatus] = []
 
-ALL_HEADER_FILES = []
+ALL_HEADER_FILES: List[Dict[str, Any]] = []
 
 
 def get_projects():

tools/web-fuzzing-introspection/app/webapp/models.py

@@ -66,13 +66,14 @@ def has_introspector(self) -> bool:
 class Project:
 
  def __init__(self, name, language, date, coverage_data, introspector_data,
- fuzzer_count):
+ fuzzer_count, project_repository):
  self.name = name
  self.language = language
  self.date = date
  self.coverage_data = coverage_data
  self.introspector_data = introspector_data
  self.fuzzer_count = fuzzer_count
+ self.project_repository = project_repository
 
  def has_introspector(self) -> bool:
  return self.introspector_data != None