Refactor integration tests to use the test runner lib

Summary:
# Problem
We have many different integration tests across the codebase. Each of these run pysa and then compare the output json to an expected json file. Most of those re-implement most of the code, leading to code duplication.

One reason for this is that these scripts run from different environments:
* Some tests (e.g, the fbcode one) run through buck, hence they need proper buck targets
* Some tests are run from `make` and require `PYRE_BINARY` to be set
* Some tests are run from the open source version using `python -m pyre.client`

For some of these, we want to use annotations rather than an "expected file" to specify expected issues (see design doc https://docs.google.com/document/d/1SwOWt_1rO9i7paF8jVtTVJsw1W1p9G5ROEP7b-B63Xw/edit ). This is hard to do because of all the different scripts around the codebase.

# Solution
Let's move most of the logic in `runner_lib`, making it generic enough to support all tests.

Python scripts now forward the call to `tools/pysa_integration_tests/run.py`, which is a generic integration test runner script. This is less than ideal, but there are no good alternatives - we cannot just import it as a python module since parent directories don't have a `__init__.py`.

Buck scripts can just rely on the buck target `//tools/pyre/tools/pysa_integration_tests:runner_lib`.

Reviewed By: alexkassil

Differential Revision: D64179565

fbshipit-source-id: 44508520cfae73cac4ea0f51853a2ca434dbd820

documentation/deliberately_vulnerable_flask_app/.gitignore

@@ -1,6 +1,6 @@
-full_result.actual
+result.actual
 position_invariant_result.json
 position_invariant_result.actual
-raw_result.json
+full_result.json
 .pyre/
 .pyre_configuration

documentation/deliberately_vulnerable_flask_app/run_integration_tests.sh

@@ -4,17 +4,9 @@
 # This source code is licensed under the MIT license found in the
 # LICENSE file in the root directory of this source tree.
 
-set +e
-python3 ../../tools/pysa_integration_tests/run.py \
+cd "$(dirname "$0")" || exit
+exec python3 ../../tools/pysa_integration_tests/run.py \
     --skip-model-verification \
-    --run-from-source
-
-exit_code=$?
-
-if [[ "$exit_code" != "0" ]]; then
-    echo "--- raw_results.json --"
-    cat raw_results.json
-    echo "---"
-fi
-
-exit $exit_code
+    --run-from-source \
+    --ignore-positions \
+    "$@"

stubs/integration_test/run.py

@@ -4,103 +4,24 @@
 # This source code is licensed under the MIT license found in the
 # LICENSE file in the root directory of this source tree.
 
-import argparse
-import json
-import logging
 import os
-import subprocess
 import sys
-from pathlib import Path
 
-LOG: logging.Logger = logging.getLogger(__name__)
-
-
-def normalized_json_dump(input: str) -> str:
-    normalized = json.loads(input)
-
-    normalized = sorted(
-        normalized,
-        key=lambda issue: (
-            issue["path"],
-            issue["line"],
-            issue["column"],
-            issue["name"],
-        ),
-    )
-
-    return json.dumps(normalized, sort_keys=True, indent=2) + "\n"
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description="Run pysa stubs integration test")
-    parser.add_argument(
-        "-s",
-        "--save-results-to",
-        type=str,
-        help=("Directory to write analysis results to. Default: output is not saved"),
-    )
-    parser.add_argument(
-        "--compact-ocaml-heap",
-        action="store_true",
-        default=False,
-        help=("Compact OCaml heap during the analysis to save memory."),
-    )
-    arguments = parser.parse_args()
-
-    logging.basicConfig(
-        level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s"
-    )
-
-    # Verify `PYRE_BINARY` is set
-    if "PYRE_BINARY" not in os.environ:
-        LOG.error(
-            "Required environment variable `PYRE_BINARY` is not set. "
-            "`make all` from this script's directory will "
-            "automatically set `PYRE_BINARY`"
-        )
-        sys.exit(1)
-
-    # Switch to directory of this script.
-    os.chdir(os.path.dirname(os.path.abspath(__file__)))
-    LOG.info("Running in `%s`", os.getcwd())
-
-    typeshed_directory = Path("../typeshed/typeshed").absolute().as_posix()
-
-    LOG.info("Running `pyre analyze`")
-    try:
-        command = [
-            "pyre",
-            "--typeshed",
-            typeshed_directory,
-            "--noninteractive",
-            "analyze",
-            "--check-invariants",
-            "--inline-decorators",
-        ]
-        if arguments.save_results_to is not None:
-            command.extend(["--save-results-to", arguments.save_results_to])
-        if arguments.compact_ocaml_heap:
-            command.append("--compact-ocaml-heap")
-        output = subprocess.check_output(command).decode()
-
-        if arguments.save_results_to is not None:
-            with open(f"{arguments.save_results_to}/errors.json") as file:
-                output = file.read()
-
-    except subprocess.CalledProcessError as exception:
-        LOG.error(f"`pyre analyze` failed with return code {exception.returncode}")
-        sys.stdout.write(exception.output.decode())
-        sys.exit(exception.returncode)
-
-    expected = ""
-    with open("result.json") as file:
-        expected = file.read()
-
-    if normalized_json_dump(expected) != normalized_json_dump(output):
-        with open("result.actual", "w") as file:
-            file.write(normalized_json_dump(output))
-        sys.stdout.write("Output differs from expected:\n")
-        subprocess.run(["diff", "-u", "result.json", "result.actual"])
-        sys.exit(30)  # ExitCode.TEST_COMPARISON_DIFFERS
-
-    LOG.info("Run produced expected results")
+directory = os.path.dirname(sys.argv[0])
+script_name = os.path.basename(sys.argv[0])
+
+if directory != ".":
+    os.chdir(directory)
+
+os.execv(
+    sys.executable,
+    [
+        script_name,
+        "../../tools/pysa_integration_tests/run.py",
+        "--require-pyre-env",
+        "--check-invariants",
+        "--inline-decorators",
+        "--typeshed", "../typeshed/typeshed",
+    ]
+    + sys.argv[1:],
+)

tools/pysa_integration_tests/run.py

@@ -4,89 +4,157 @@
 # LICENSE file in the root directory of this source tree.
 
 """
-TODO(T132414938) Add a module-level docstring
+A generic script to run integration tests.
 """
 
+# pyre-strict
 import argparse
 import logging
 import os
+import sys
 from pathlib import Path
-from typing import List, Optional
+from typing import List, Optional, TYPE_CHECKING
 
-# pyre-ignore[21]
-from runner_lib import run_pysa_integration_test
+# This script is meant to be run from the command line.
+if TYPE_CHECKING:
+    import tools.pyre.tools.pysa_integration_tests.runner_lib as test_runner_lib
+else:
+    import runner_lib as test_runner_lib  # @manual=//tools/pyre/tools/pysa_integration_tests:runner_lib
 
 LOG: logging.Logger = logging.getLogger(__name__)
 
 
 def main(
-    run_directory: Path,
+    *,
+    working_directory: Path,
     filter_issues: bool,
     skip_model_verification: bool,
     run_from_source: bool,
-    passthrough_args: List[str],
+    passthrough_args: Optional[List[str]],
     save_results_to: Optional[Path],
+    typeshed: Optional[Path],
+    compact_ocaml_heap: bool,
+    check_invariants: bool,
+    inline_decorators: bool,
+    require_pyre_env: bool,
+    ignore_positions: bool,
 ) -> None:
     """
-    Entry point function which checks if full_result.json is there, calls
-    functions from runner_lib to run pysa, parse full_result.json, and compare the output.
+    Entry point function to run a full end-to-end integration test.
     """
     logging.basicConfig(
         level=logging.DEBUG,
         format="%(asctime)s %(levelname)s %(message)s",
     )
 
-    full_result_file_path = run_directory / "full_result.json"
-
-    if not os.path.isfile(full_result_file_path):
-        raise FileNotFoundError(
-            f"{full_result_file_path} containing expected issues is not found"
+    # Verify `PYRE_BINARY` is set
+    if require_pyre_env and "PYRE_BINARY" not in os.environ:
+        LOG.error(
+            "Required environment variable `PYRE_BINARY` is not set. "
+            "`make all` from this script's directory will "
+            "automatically set `PYRE_BINARY`"
         )
+        sys.exit(1)
 
-    LOG.info("Running in `%s`", run_directory)
-    run_pysa_integration_test(
-        run_directory,
+    LOG.info("Running in `%s`", working_directory)
+    pysa_results = test_runner_lib.run_pysa(
         passthrough_args=passthrough_args,
         skip_model_verification=skip_model_verification,
-        filter_issues=filter_issues,
         run_from_source=run_from_source,
         save_results_to=save_results_to,
+        typeshed=typeshed,
+        compact_ocaml_heap=compact_ocaml_heap,
+        check_invariants=check_invariants,
+        inline_decorators=inline_decorators,
+        working_directory=working_directory,
+    )
+
+    test_result_directory = (
+        save_results_to if save_results_to is not None else working_directory
+    )
+    test_runner_lib.compare_to_expected_json(
+        actual_results=pysa_results,
+        expected_results_path=working_directory / "result.json",
+        test_result_directory=test_result_directory,
+        filter_issues=filter_issues,
+        ignore_positions=ignore_positions,
     )
 
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="Run integeration tests")
 
-    parser.add_argument("--run-directory", type=Path)
-    parser.add_argument("--filter-issues", action="store_true")
-    parser.add_argument("--skip-model-verification", action="store_true")
-    parser.add_argument("--run-from-source", action="store_true")
-    parser.add_argument("--passthrough-args", nargs="+")
+    parser.add_argument("--working-directory", type=Path, help="Test working directory")
+    parser.add_argument(
+        "--filter-issues",
+        action="store_true",
+        help="Filter out issues with a rule that is not present in the function name",
+    )
+    parser.add_argument(
+        "--skip-model-verification", action="store_true", help="Skip model verification"
+    )
+    parser.add_argument(
+        "--run-from-source",
+        action="store_true",
+        help="Run pysa from source with the open source setup",
+    )
+    parser.add_argument(
+        "--require-pyre-env",
+        action="store_true",
+        help="Require the PYRE_BINARY environment variable to be set",
+    )
+    parser.add_argument(
+        "--ignore-positions",
+        action="store_true",
+        help="Ignore positions when comparing expected results",
+    )
+    parser.add_argument(
+        "--passthrough-args",
+        nargs="+",
+        help="Additional parameters to pass to `pyre analyze`",
+    )
     parser.add_argument(
         "-s",
         "--save-results-to",
         type=Path,
-        help=("Directory to write analysis results to. Default: output is not saved"),
+        help="Directory to write analysis results to. Default: output is not saved",
+    )
+    parser.add_argument(
+        "--compact-ocaml-heap",
+        action="store_true",
+        default=False,
+        help="Compact OCaml heap during the analysis to save memory",
+    )
+    parser.add_argument(
+        "--check-invariants",
+        action="store_true",
+        default=False,
+        help="Check abstract domain invariants when running the analysis",
+    )
+    parser.add_argument(
+        "--inline-decorators",
+        action="store_true",
+        default=False,
+        help="Inline decorators when running the analysis",
+    )
+    parser.add_argument(
+        "--typeshed",
+        type=Path,
+        help="Path to the typeshed to use",
     )
 
     parsed: argparse.Namespace = parser.parse_args()
-
-    run_directory: Path = parsed.run_directory
-    if run_directory is None:
-        run_directory = Path(os.getcwd())
-    filter_issues: bool = parsed.filter_issues
-    skip_model_verification: bool = parsed.skip_model_verification
-    run_from_source: bool = parsed.run_from_source
-    passthrough_args: List[str] = parsed.passthrough_args
-    if passthrough_args is None:
-        passthrough_args = []
-    save_results_to: Optional[Path] = parsed.save_results_to
-
     main(
-        run_directory,
-        filter_issues,
-        skip_model_verification,
-        run_from_source,
-        passthrough_args,
-        save_results_to,
+        working_directory=parsed.working_directory or Path(os.getcwd()),
+        filter_issues=parsed.filter_issues,
+        skip_model_verification=parsed.skip_model_verification,
+        run_from_source=parsed.run_from_source,
+        passthrough_args=parsed.passthrough_args,
+        save_results_to=parsed.save_results_to,
+        typeshed=parsed.typeshed,
+        compact_ocaml_heap=parsed.compact_ocaml_heap,
+        check_invariants=parsed.check_invariants,
+        inline_decorators=parsed.inline_decorators,
+        require_pyre_env=parsed.require_pyre_env,
+        ignore_positions=parsed.ignore_positions,
     )