diff --git a/documentation/deliberately_vulnerable_flask_app/.gitignore b/documentation/deliberately_vulnerable_flask_app/.gitignore index 959196b4bf1..9e23b7ddc72 100644 --- a/documentation/deliberately_vulnerable_flask_app/.gitignore +++ b/documentation/deliberately_vulnerable_flask_app/.gitignore @@ -1,6 +1,6 @@ -full_result.actual +result.actual position_invariant_result.json position_invariant_result.actual -raw_result.json +full_result.json .pyre/ .pyre_configuration diff --git a/documentation/deliberately_vulnerable_flask_app/full_result.json b/documentation/deliberately_vulnerable_flask_app/result.json similarity index 100% rename from documentation/deliberately_vulnerable_flask_app/full_result.json rename to documentation/deliberately_vulnerable_flask_app/result.json diff --git a/documentation/deliberately_vulnerable_flask_app/run_integration_tests.sh b/documentation/deliberately_vulnerable_flask_app/run_integration_tests.sh index 1d58a8cb537..57237d24ffa 100755 --- a/documentation/deliberately_vulnerable_flask_app/run_integration_tests.sh +++ b/documentation/deliberately_vulnerable_flask_app/run_integration_tests.sh @@ -4,17 +4,9 @@ # This source code is licensed under the MIT license found in the # LICENSE file in the root directory of this source tree. -set +e -python3 ../../tools/pysa_integration_tests/run.py \ +cd "$(dirname "$0")" || exit +exec python3 ../../tools/pysa_integration_tests/run.py \ --skip-model-verification \ - --run-from-source - -exit_code=$? - -if [[ "$exit_code" != "0" ]]; then - echo "--- raw_results.json --" - cat raw_results.json - echo "---" -fi - -exit $exit_code + --run-from-source \ + --ignore-positions \ + "$@" diff --git a/stubs/integration_test/run.py b/stubs/integration_test/run.py index 88b60a1a4b3..8d8df7fa0b2 100755 --- a/stubs/integration_test/run.py +++ b/stubs/integration_test/run.py @@ -4,103 +4,24 @@ # This source code is licensed under the MIT license found in the # LICENSE file in the root directory of this source tree. -import argparse -import json -import logging import os -import subprocess import sys -from pathlib import Path -LOG: logging.Logger = logging.getLogger(__name__) - - -def normalized_json_dump(input: str) -> str: - normalized = json.loads(input) - - normalized = sorted( - normalized, - key=lambda issue: ( - issue["path"], - issue["line"], - issue["column"], - issue["name"], - ), - ) - - return json.dumps(normalized, sort_keys=True, indent=2) + "\n" - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Run pysa stubs integration test") - parser.add_argument( - "-s", - "--save-results-to", - type=str, - help=("Directory to write analysis results to. Default: output is not saved"), - ) - parser.add_argument( - "--compact-ocaml-heap", - action="store_true", - default=False, - help=("Compact OCaml heap during the analysis to save memory."), - ) - arguments = parser.parse_args() - - logging.basicConfig( - level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s" - ) - - # Verify `PYRE_BINARY` is set - if "PYRE_BINARY" not in os.environ: - LOG.error( - "Required environment variable `PYRE_BINARY` is not set. " - "`make all` from this script's directory will " - "automatically set `PYRE_BINARY`" - ) - sys.exit(1) - - # Switch to directory of this script. - os.chdir(os.path.dirname(os.path.abspath(__file__))) - LOG.info("Running in `%s`", os.getcwd()) - - typeshed_directory = Path("../typeshed/typeshed").absolute().as_posix() - - LOG.info("Running `pyre analyze`") - try: - command = [ - "pyre", - "--typeshed", - typeshed_directory, - "--noninteractive", - "analyze", - "--check-invariants", - "--inline-decorators", - ] - if arguments.save_results_to is not None: - command.extend(["--save-results-to", arguments.save_results_to]) - if arguments.compact_ocaml_heap: - command.append("--compact-ocaml-heap") - output = subprocess.check_output(command).decode() - - if arguments.save_results_to is not None: - with open(f"{arguments.save_results_to}/errors.json") as file: - output = file.read() - - except subprocess.CalledProcessError as exception: - LOG.error(f"`pyre analyze` failed with return code {exception.returncode}") - sys.stdout.write(exception.output.decode()) - sys.exit(exception.returncode) - - expected = "" - with open("result.json") as file: - expected = file.read() - - if normalized_json_dump(expected) != normalized_json_dump(output): - with open("result.actual", "w") as file: - file.write(normalized_json_dump(output)) - sys.stdout.write("Output differs from expected:\n") - subprocess.run(["diff", "-u", "result.json", "result.actual"]) - sys.exit(30) # ExitCode.TEST_COMPARISON_DIFFERS - - LOG.info("Run produced expected results") +directory = os.path.dirname(sys.argv[0]) +script_name = os.path.basename(sys.argv[0]) + +if directory != ".": + os.chdir(directory) + +os.execv( + sys.executable, + [ + script_name, + "../../tools/pysa_integration_tests/run.py", + "--require-pyre-env", + "--check-invariants", + "--inline-decorators", + "--typeshed", "../typeshed/typeshed", + ] + + sys.argv[1:], +) diff --git a/tools/pysa_integration_tests/run.py b/tools/pysa_integration_tests/run.py index ae9583be088..c81c3443bac 100644 --- a/tools/pysa_integration_tests/run.py +++ b/tools/pysa_integration_tests/run.py @@ -4,89 +4,157 @@ # LICENSE file in the root directory of this source tree. """ -TODO(T132414938) Add a module-level docstring +A generic script to run integration tests. """ +# pyre-strict import argparse import logging import os +import sys from pathlib import Path -from typing import List, Optional +from typing import List, Optional, TYPE_CHECKING -# pyre-ignore[21] -from runner_lib import run_pysa_integration_test +# This script is meant to be run from the command line. +if TYPE_CHECKING: + import tools.pyre.tools.pysa_integration_tests.runner_lib as test_runner_lib +else: + import runner_lib as test_runner_lib # @manual=//tools/pyre/tools/pysa_integration_tests:runner_lib LOG: logging.Logger = logging.getLogger(__name__) def main( - run_directory: Path, + *, + working_directory: Path, filter_issues: bool, skip_model_verification: bool, run_from_source: bool, - passthrough_args: List[str], + passthrough_args: Optional[List[str]], save_results_to: Optional[Path], + typeshed: Optional[Path], + compact_ocaml_heap: bool, + check_invariants: bool, + inline_decorators: bool, + require_pyre_env: bool, + ignore_positions: bool, ) -> None: """ - Entry point function which checks if full_result.json is there, calls - functions from runner_lib to run pysa, parse full_result.json, and compare the output. + Entry point function to run a full end-to-end integration test. """ logging.basicConfig( level=logging.DEBUG, format="%(asctime)s %(levelname)s %(message)s", ) - full_result_file_path = run_directory / "full_result.json" - - if not os.path.isfile(full_result_file_path): - raise FileNotFoundError( - f"{full_result_file_path} containing expected issues is not found" + # Verify `PYRE_BINARY` is set + if require_pyre_env and "PYRE_BINARY" not in os.environ: + LOG.error( + "Required environment variable `PYRE_BINARY` is not set. " + "`make all` from this script's directory will " + "automatically set `PYRE_BINARY`" ) + sys.exit(1) - LOG.info("Running in `%s`", run_directory) - run_pysa_integration_test( - run_directory, + LOG.info("Running in `%s`", working_directory) + pysa_results = test_runner_lib.run_pysa( passthrough_args=passthrough_args, skip_model_verification=skip_model_verification, - filter_issues=filter_issues, run_from_source=run_from_source, save_results_to=save_results_to, + typeshed=typeshed, + compact_ocaml_heap=compact_ocaml_heap, + check_invariants=check_invariants, + inline_decorators=inline_decorators, + working_directory=working_directory, + ) + + test_result_directory = ( + save_results_to if save_results_to is not None else working_directory + ) + test_runner_lib.compare_to_expected_json( + actual_results=pysa_results, + expected_results_path=working_directory / "result.json", + test_result_directory=test_result_directory, + filter_issues=filter_issues, + ignore_positions=ignore_positions, ) if __name__ == "__main__": parser = argparse.ArgumentParser(description="Run integeration tests") - parser.add_argument("--run-directory", type=Path) - parser.add_argument("--filter-issues", action="store_true") - parser.add_argument("--skip-model-verification", action="store_true") - parser.add_argument("--run-from-source", action="store_true") - parser.add_argument("--passthrough-args", nargs="+") + parser.add_argument("--working-directory", type=Path, help="Test working directory") + parser.add_argument( + "--filter-issues", + action="store_true", + help="Filter out issues with a rule that is not present in the function name", + ) + parser.add_argument( + "--skip-model-verification", action="store_true", help="Skip model verification" + ) + parser.add_argument( + "--run-from-source", + action="store_true", + help="Run pysa from source with the open source setup", + ) + parser.add_argument( + "--require-pyre-env", + action="store_true", + help="Require the PYRE_BINARY environment variable to be set", + ) + parser.add_argument( + "--ignore-positions", + action="store_true", + help="Ignore positions when comparing expected results", + ) + parser.add_argument( + "--passthrough-args", + nargs="+", + help="Additional parameters to pass to `pyre analyze`", + ) parser.add_argument( "-s", "--save-results-to", type=Path, - help=("Directory to write analysis results to. Default: output is not saved"), + help="Directory to write analysis results to. Default: output is not saved", + ) + parser.add_argument( + "--compact-ocaml-heap", + action="store_true", + default=False, + help="Compact OCaml heap during the analysis to save memory", + ) + parser.add_argument( + "--check-invariants", + action="store_true", + default=False, + help="Check abstract domain invariants when running the analysis", + ) + parser.add_argument( + "--inline-decorators", + action="store_true", + default=False, + help="Inline decorators when running the analysis", + ) + parser.add_argument( + "--typeshed", + type=Path, + help="Path to the typeshed to use", ) parsed: argparse.Namespace = parser.parse_args() - - run_directory: Path = parsed.run_directory - if run_directory is None: - run_directory = Path(os.getcwd()) - filter_issues: bool = parsed.filter_issues - skip_model_verification: bool = parsed.skip_model_verification - run_from_source: bool = parsed.run_from_source - passthrough_args: List[str] = parsed.passthrough_args - if passthrough_args is None: - passthrough_args = [] - save_results_to: Optional[Path] = parsed.save_results_to - main( - run_directory, - filter_issues, - skip_model_verification, - run_from_source, - passthrough_args, - save_results_to, + working_directory=parsed.working_directory or Path(os.getcwd()), + filter_issues=parsed.filter_issues, + skip_model_verification=parsed.skip_model_verification, + run_from_source=parsed.run_from_source, + passthrough_args=parsed.passthrough_args, + save_results_to=parsed.save_results_to, + typeshed=parsed.typeshed, + compact_ocaml_heap=parsed.compact_ocaml_heap, + check_invariants=parsed.check_invariants, + inline_decorators=parsed.inline_decorators, + require_pyre_env=parsed.require_pyre_env, + ignore_positions=parsed.ignore_positions, ) diff --git a/tools/pysa_integration_tests/runner_lib.py b/tools/pysa_integration_tests/runner_lib.py index 3852495704c..01efc18e8a6 100644 --- a/tools/pysa_integration_tests/runner_lib.py +++ b/tools/pysa_integration_tests/runner_lib.py @@ -6,14 +6,18 @@ # pyre-strict """ -TODO(T132414938) Add a module-level docstring +Helper functions for integration test runners, +e.g running pysa and compare the results with the expectations. """ from __future__ import annotations +import ast import enum import json import logging +import os.path +import re import subprocess import sys from pathlib import Path @@ -31,26 +35,102 @@ class PyreErrorException(Exception): pass +@final +class TestConfigurationException(Exception): + pass + + class ExitCode(enum.IntEnum): # 1-29 reserved for pyre and pysa client, see client/commands/commands.py TEST_COMPARISON_DIFFERS = 30 TEST_MODEL_VERIFICATION_ERROR = 31 +def is_test_function(define: str, code: int) -> bool: + return f"test_{code}_" in define + + +def is_test_class_method(define: str, code: int) -> bool: + define_split = define.split(".") + if len(define_split) < 2: + return False + return f"Test{code}" in define_split[-2] + + +def validate_test_functions_and_class_names(current_directory: Path) -> None: + LOG.info( + "Ensure all functions and classes in test_XXX files meet the expected format" + ) + + test_function_pattern = re.compile(r"test_\d{4}(_no)?_flag_\w+") + test_class_pattern = re.compile(r"Test\d{4}\w+") + helper_class_pattern = re.compile(r"Helper\w+") + + test_paths = [ + path + for path in current_directory.iterdir() + if re.match(r"test(_\w+)?\.py$", path.name) + ] + + for test_path in test_paths: + parsed_ast = ast.parse(test_path.read_text()) + + functions = [ + node + for node in parsed_ast.body + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + ] + + functions = [] + classes = [] + for node in parsed_ast.body: + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + functions.append(node) + elif isinstance(node, ast.ClassDef): + classes.append(node) + + for function in functions: + function_name = function.name + LOG.debug(f"Validating function: {function_name}") + + if not test_function_pattern.match(function_name): + raise TestConfigurationException( + f"Expected test function {function_name} to have the " + + "format test_####_flag_XXXX or test_####_no_flag_XXXX, " + + "to indicate that issue #### is being tested" + ) + + for klass in classes: + class_name = klass.name + LOG.debug(f"Validating class: {class_name}") + + if not ( + test_class_pattern.match(class_name) + or helper_class_pattern.match(class_name) + ): + raise TestConfigurationException( + f"Expected test class {class_name} to have the " + + "format Test####XXXX to indicate that issue #### " + + "is being tested, or HelperXXXX to indicate it is " + + "an unrelated helper class." + ) + + def normalized_json_dump( - results: str, salient_keys_only: bool, filter_issues: bool + results: str, ignore_positions: bool, filter_issues: bool ) -> str: """ Returns a normalised JSON string from results keeping only essential items. Removes all keys that are not salient to determining if results have changed - when 'salient_keys_only' is true. Filters issues down to issues that have - the code we intend to test for if 'filter_issues' is true. + when `ignore_positions` is true. Filters issues down to issues that have + the code we intend to test for if `filter_issues` is true. """ normalized = json.loads(results) + if "errors" in normalized: pretty_error = json.dumps(normalized, sort_keys=True, indent=2) raise PyreErrorException( - f"Errors were found when processing results:\n{pretty_error}" + f"Errors were found when processing analysis results:\n{pretty_error}" ) if filter_issues: @@ -58,7 +138,10 @@ def normalized_json_dump( # test for. This prevents the introduction of new rules or false # positives from breaking existing tests. normalized = [ - issue for issue in normalized if f"test_{issue['code']}_" in issue["define"] + issue + for issue in normalized + if is_test_function(issue["define"], issue["code"]) + or is_test_class_method(issue["define"], issue["code"]) ] normalized = sorted( @@ -71,8 +154,8 @@ def normalized_json_dump( ), ) - if salient_keys_only: - salient_keys = {"code", "define", "description", "path"} + if ignore_positions: + salient_keys = {"code", "define", "description", "path", "name"} stripped_issues = [] for issue in normalized: stripped_issue = { @@ -89,33 +172,150 @@ def normalized_json_dump( return json.dumps(normalized, sort_keys=True, indent=2) + "\n" -def compare_results( +def run_pysa( + *, + save_results_to: Optional[Path] = None, + save_errors_to: Optional[Path] = None, + target: Optional[str] = None, + number_of_workers: Optional[int] = None, + skip_model_verification: bool = False, + isolation_prefix: Optional[str] = None, + repository_root: Optional[Path] = None, + excludes: Optional[Sequence[str]] = None, + run_from_source: bool = False, + typeshed: Optional[Path] = None, + compact_ocaml_heap: bool = False, + check_invariants: bool = False, + inline_decorators: bool = False, + maximum_trace_length: Optional[int] = None, + maximum_tito_depth: Optional[int] = None, + passthrough_args: Optional[Sequence[str]] = None, + working_directory: Optional[Path] = None, + silent: bool = False, + error_help: Optional[str] = None, +) -> str: + """Run pysa for the given test and produce a list of errors in JSON.""" + if run_from_source: + command = [ + "python", + "-m" "pyre-check.client.pyre", + ] + else: + command = ["pyre"] + + command.append("--noninteractive") + + if isolation_prefix is not None: + command.extend(["--isolation-prefix", isolation_prefix]) + + if number_of_workers is not None: + command.append(f"--number-of-workers={number_of_workers}") + + if typeshed is not None: + command.extend(["--typeshed", typeshed.absolute().as_posix()]) + + if target is not None: + command.append(f"--target={target}") + + if excludes is not None: + for exclude in excludes: + command.extend(["--exclude", exclude]) + + command.append("analyze") + + if skip_model_verification: + command.append("--no-verify") + + if repository_root is not None: + command.extend(["--repository-root", str(repository_root)]) + + if save_results_to is not None: + command.extend(["--save-results-to", str(save_results_to)]) + + if compact_ocaml_heap: + command.append("--compact-ocaml-heap") + + if check_invariants: + command.append("--check-invariants") + + if inline_decorators: + command.append("--inline-decorators") + + if maximum_trace_length is not None: + command.append(f"--maximum-trace-length={maximum_trace_length}") + + if maximum_tito_depth is not None: + command.append(f"--maximum-tito-depth={maximum_tito_depth }") + + if passthrough_args is not None: + command.extend(passthrough_args) + + LOG.info(f"Running `{' '.join(command)}`") + try: + process = subprocess.run( + command, + check=True, + text=True, + stdout=subprocess.PIPE, + stderr=(subprocess.DEVNULL if silent else None), + cwd=working_directory, + ) + except subprocess.CalledProcessError as exception: + LOG.error(f"`pyre analyze` failed with return code {exception.returncode}") + sys.stdout.write(exception.stdout) + if error_help is not None: + sys.stdout.write("\n") + sys.stdout.write(error_help) + sys.exit(exception.returncode) + + if save_results_to is not None: + errors = (save_results_to / "errors.json").read_text() + else: + errors = process.stdout + + if save_errors_to is not None: + save_errors_to.write_text(errors) + + return errors + + +def compare_to_expected_json( + *, actual_results: str, - expected_results: str, + expected_results_path: Path, test_result_directory: Path, filter_issues: bool, + ignore_positions: bool, + error_help: Optional[str] = None, ) -> None: + """ + Compare the errors from `run_pysa` to a set of expected + errors from a JSON file. + """ + if not os.path.isfile(expected_results_path): + raise FileNotFoundError( + f"Could NOT find expected result file `{expected_results_path}`" + ) + expected_results = expected_results_path.read_text() + normalized_pysa_results = normalized_json_dump( - actual_results, salient_keys_only=True, filter_issues=filter_issues + actual_results, ignore_positions=ignore_positions, filter_issues=filter_issues ) normalized_expected_results = normalized_json_dump( - expected_results, salient_keys_only=True, filter_issues=filter_issues + expected_results, ignore_positions=ignore_positions, filter_issues=filter_issues ) - if normalized_pysa_results != normalized_expected_results: - actual_full_results_path = test_result_directory / "full_result.actual" - actual_full_results_path.write_text( - normalized_json_dump( - actual_results, salient_keys_only=False, filter_issues=filter_issues - ) - ) + if normalized_pysa_results == normalized_expected_results: + LOG.info("Run produced expected results") + return - expected_full_results_path = test_result_directory / "expected_result.json" - expected_full_results_path.write_text( - normalized_json_dump( - expected_results, salient_keys_only=False, filter_issues=filter_issues - ) + (test_result_directory / "full_result.json").write_text(actual_results) + (test_result_directory / "result.actual").write_text( + normalized_json_dump( + actual_results, ignore_positions=False, filter_issues=filter_issues ) + ) + if ignore_positions: actual_invariant_results_path = ( test_result_directory / "position_invariant_result.actual" ) @@ -125,74 +325,24 @@ def compare_results( test_result_directory / "position_invariant_result.json" ) expected_invariant_results_path.write_text(normalized_expected_results) + else: + actual_invariant_results_path = test_result_directory / "result.actual" + expected_invariant_results_path = expected_results_path + if ignore_positions: sys.stdout.write("Output differs from expected:\n") - sys.stdout.flush() - subprocess.run( - [ - "diff", - "-u", - expected_invariant_results_path, - actual_invariant_results_path, - ] - ) - sys.exit(ExitCode.TEST_COMPARISON_DIFFERS.value) else: - LOG.info("Run produced expected results") - - -def run_pysa_integration_test( - current_directory: Path, - passthrough_args: Sequence[str], - skip_model_verification: bool, - filter_issues: bool, - save_results_to: Optional[Path], - run_from_source: bool = False, -) -> None: - """ - Runs pysa and compares the output to that in full_results.json. Creates - raw_results.json file that contains the output. Creates - position_invariant_result.json that contains position information to - compare using diff with position_invariant_result.actual before exiting if - there is a mismatch between the specified and detected issues. - """ - LOG.info("Running `pyre analyze`") - if run_from_source: - command = [ - "python", - "-m" "pyre-check.client.pyre", + sys.stdout.write("Output differs from expected (after stripping locations):\n") + sys.stdout.flush() + subprocess.run( + [ + "diff", + "-u", + expected_invariant_results_path, + actual_invariant_results_path, ] - else: - command = ["pyre"] - command.extend(["--noninteractive", "analyze"]) - - if save_results_to is not None: - command.extend(["--save-results-to", str(save_results_to)]) - - if skip_model_verification: - command.append("--no-verify") - - command.extend(passthrough_args) - LOG.debug(f"Using command: {command}") - pysa_results: str - try: - pysa_results = subprocess.check_output( - command, text=True, cwd=current_directory - ) - if save_results_to is not None: - pysa_results = (save_results_to / "errors.json").read_text() - except subprocess.CalledProcessError as exception: - LOG.error(f"`pyre analyze` failed with return code {exception.returncode}") - sys.stdout.write(exception.output) - sys.exit(exception.returncode) - - (current_directory / "raw_results.json").write_text(pysa_results) - - expected_results = (current_directory / "full_result.json").read_text() - - test_result_directory = ( - save_results_to if save_results_to is not None else current_directory - ) - compare_results( - pysa_results, expected_results, test_result_directory, filter_issues ) + if error_help is not None: + sys.stdout.write("\n") + sys.stdout.write(error_help) + sys.exit(ExitCode.TEST_COMPARISON_DIFFERS.value)