Add a verification for users and CI (#4)

* Add initial code for a verification script

* Add a CI for verifying pipeline still works

* Add install line

* Add angr

* Update readme for verification script

.github/workflows/python-app.yml

@@ -0,0 +1,38 @@
+# This workflow will install Python dependencies, run tests and lint with a single version of Python
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: Core Tests
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python 3.11
+      uses: actions/setup-python@v2
+      with:
+        python-version: "3.11"
+    - name: Install Java 19
+      uses: actions/setup-java@v3
+      with:
+        distribution: 'temurin' # See 'Supported distributions' for available options
+        java-version: '19'
+    - name: Install dependencies
+      run: |
+        pip install --upgrade pip
+        sudo apt-get update && sudo apt-get install unzip openjdk-19-jdk graphviz-dev -y
+        pip3 install -e .
+        pip3 install angr
+    - name: Run verification tests
+      run: |
+        # a hack to allow for docker containers in github actions and ARM
+        export JAVA_OPTS="-Djdk.lang.Process.launchMechanism=vfork" 
+        ./scripts/ci_verify_pipeline.sh

Dockerfile

@@ -17,8 +17,8 @@ RUN apt-get update && apt-get -o APT::Immediate-Configure=0 install -y \
         libxslt1-dev git libffi-dev cmake libreadline-dev libtool debootstrap \
         debian-archive-keyring libglib2.0-dev libpixman-1-dev qtdeclarative5-dev \
         binutils-multiarch nasm libc6:i386 libgcc1:i386 libstdc++6:i386 \
-        libtinfo5:i386 zlib1g:i386 vim libssl-dev openjdk-8-jdk openjdk-11-jdk \
-        openjdk-17-jdk graphviz-dev \
+        libtinfo5:i386 zlib1g:i386 vim libssl-dev openjdk-8-jdk openjdk-17-jdk \
+        openjdk-19-jdk graphviz-dev python-is-python3 \
     && apt-get install --no-install-recommends -y apt-utils wget unzip tar clang-format \
     && rm -rf /var/lib/apt/lists/* \
     && mkdir SAILR && mkdir versioned_tools && mkdir tools
@@ -42,9 +42,6 @@ WORKDIR versioned_tools
 # angr decompiler (SAILR version)
 # ===========================================================
 
-# make it easier to install
-RUN apt update && apt install python-is-python3 -y
-
 # install angr, and all angr-packages, from the repos (for checkout ability)
 RUN git clone https://github.com/angr/angr-dev.git && ( \
     cd ./angr-dev && \
@@ -86,7 +83,10 @@ ENV FORCE_UNSAFE_CONFIGURE=1
 # ===========================================================
 
 # coreutils & compiler reqs
-RUN apt-get install -y apt-utils bison autopoint gettext gperf rsync texinfo libcap-dev libacl1-dev libssl-dev gcc-9 g++-9 clang flex
+RUN apt-get update && \
+    apt-get install -y \
+    apt-utils bison autopoint gettext gperf rsync texinfo libcap-dev libacl1-dev libssl-dev \
+    gcc-9 g++-9 clang flex
 
 #&& \
 #wget http://security.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2.16_amd64.deb && \

README.md

@@ -45,6 +45,24 @@ Run the setup script to install the dependencies:
 This will install the Python package locally and build the docker container. If you know you don't want to use the docker
 container, then you can directly install the Python package with `pip3 insatll .`. Note: you need `graphviz` on your system.
 
+Verify the installation by running:
+```bash
+./scripts/verify_pipeline.sh
+```
+
+If your installation is correct, you should see some final output like:
+```
+# Evaluation Data
+## Stats
+Layout: ('sum', 'mean', 'median')
+### O2
+Metric     | source      | angr_sailr  | angr_dream 
+---------- | ----------- | ----------- | -----------
+cfged      | 0/0/0.0     | 14/1.75/2.0 | 34/4.25/2.0
+...
+```
+
+
 ## Usage
 After installation, if you used the script normally (i.e. the docker install), than you can use the `docker-eval.sh` script
 which is a proxy to the `eval.py` script, but inside the container. 

docker-eval.sh

@@ -9,8 +9,8 @@ docker run \
   -it \
   --rm \
   -v $PWD:/SAILR \
-  sailr-eval "${@:1}"
+  sailr-eval "${@:1}" || exit 1
 
-echo "[+] Done! Remember to update the perms on your results: 'sudo chown \$USER:\$USER -R results'"
-#sudo chown $USER:$USER -R results/
+#echo "[+] Done! Remember to update the perms on your results: 'sudo chown \$USER:\$USER -R results'"
+sudo chown $USER:$USER -R results/
 exit 0

sailreval/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "1.0.1"
+__version__ = "1.1.0"
 
 # create loggers
 import logging

scripts/ci_verify_pipeline.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+PROJECT_DIR="$SCRIPT_DIR/.."
+EXAMPLE_PROJECT_DIR="$SCRIPT_DIR/../tests/example_project"
+RESULTS_DIR="$SCRIPT_DIR/../results/O2"
+CORES="$(nproc --all)"
+
+echo "Installing dependencies..."
+sudo apt-get update && sudo apt-get install -y gcc make strip
+
+echo "[+] Checking if results directory exists..."
+if [ ! -d "$RESULTS_DIR" ]; then
+    echo "[+] Creating results directory..."
+    mkdir -p "$RESULTS_DIR"
+fi
+
+echo "[+] Running pipeline in $PROJECT_DIR..."
+cd "$PROJECT_DIR" || exit 1
+
+echo "[+] Running compilation pipeline..." && \
+# runs on the default opts, which is O2
+./eval.py --compile example_project --cores "$CORES" && \
+echo "[+] Running decompilation pipeline (only for source)..." && \
+./eval.py --decompile example_project --cores "$CORES" --use-dec source && \
+echo "[+] Copying decompilation..." && \
+mkdir -p "$PROJECT_DIR"/results/O2/example_project/sailr_decompiled && \
+cp "$PROJECT_DIR"/tests/ci/angr* "$PROJECT_DIR"/results/O2/example_project/sailr_decompiled && \
+echo "[+] Running measurement pipeline (gotos, bools, calls, cfged)..." && \
+./eval.py --measure example_project --use-metric gotos cfged bools func_calls --use-dec source angr_sailr angr_dream --cores "$CORES" && \
+echo "[+] Running aggregation pipeline..." && \
+./eval.py --summarize-targets example_project --use-dec source angr_sailr angr_dream --use-metric gotos cfged bools func_calls --show-stats && \
+# cleanup
+rm -rf "$RESULTS_DIR/example_project" && \
+echo "[+] The pipeline has successfully finished!" || (echo "[!] Pipeline failed, check the last stage it was in to figure out where!" && exit 1)

scripts/verify_pipeline.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+PROJECT_DIR="$SCRIPT_DIR/.."
+EXAMPLE_PROJECT_DIR="$SCRIPT_DIR/../tests/example_project"
+RESULTS_DIR="$SCRIPT_DIR/../results/O2"
+CORES="$(nproc --all)"
+
+echo "[+] Checking if results directory exists..."
+if [ ! -d "$RESULTS_DIR" ]; then
+    echo "[+] Creating results directory..."
+    mkdir -p "$RESULTS_DIR"
+fi
+
+echo "[+] Running pipeline in $PROJECT_DIR..."
+cd "$PROJECT_DIR" || exit 1
+
+echo "[+] The following runs will require sudo for docker and file permissions..."
+echo "[+] Running compilation pipeline..." && \
+# runs on the default opts, which is O2
+sudo ./docker-eval.sh --compile example_project --cores "$CORES" && \
+echo "[+] Running decompilation pipeline (only for source)..." && \
+./eval.py --decompile example_project --cores "$CORES" --use-dec source && \
+echo "[+] Running decompilation pipeline (for other decompilers)..." && \
+sudo ./docker-eval.sh --decompile example_project --use-dec angr_sailr angr_dream --cores "$CORES" && \
+echo "[+] Running measurement pipeline (gotos, bools, calls, cfged)..." && \
+./eval.py --measure example_project --use-metric gotos cfged bools func_calls --use-dec source angr_sailr angr_dream --cores "$CORES" && \
+echo "[+] Running aggregation pipeline..." && \
+./eval.py --summarize-targets example_project --use-dec source angr_sailr angr_dream --use-metric gotos cfged bools func_calls --show-stats && \
+# cleanup
+#rm -rf "$RESULTS_DIR/example_project" && \
+echo "[+] The pipeline has successfully finished!" || (echo "[!] Pipeline failed, check the last stage it was in to figure out where!" && exit 1)

targets/example_project.toml

@@ -0,0 +1,17 @@
+package_name = "example_project"
+# path is relative to ./results/{opt_level}/ directory, so usually should start with ../../
+source_remote = "../../tests/example_project"
+# see other targets for real remote types
+remote_type = "local"
+download = true
+# post_download_cmds are run only once after first download
+# for commands that run before every make, use pre_make_cmds
+post_download_cmds = ["./bootstrap.sh"]
+# useful for git repos (will checkout)
+version = "v0.0.0"
+# the directory name of the entire project
+package_dir = "example_project"
+make_cmd = "make"
+post_make_cmds = []
+# location (inside package_dir) that has the source code
+source_dir = "."

tests/ci/angr_dream_example.c

@@ -0,0 +1,64 @@
+void complete_job() {
+    unsigned long long v1;  // rax
+
+    v1 = puts("checking...");
+    return;
+}
+
+void next_job() {
+    unsigned long long v1;  // rax
+
+    puts("next_job");
+    v1 = 1;
+    return;
+}
+
+void refresh_jobs() {
+    unsigned long long v1;  // rax
+
+    puts("refresh_jobs");
+    v1 = 2;
+    return;
+}
+
+void fast_unlock() {
+    unsigned long long v1;  // rax
+
+    puts("fast_unlock");
+    v1 = 4;
+    return;
+}
+
+void log_workers() {
+    unsigned long long v1;  // rax
+
+    v1 = puts("log_workers");
+    return;
+}
+
+long long schedule_job(unsigned long a0, unsigned long long a1, unsigned long a2) {
+    if (a0 && a1) {
+        complete_job();
+        if (EARLY_EXIT != a2) {
+            next_job();
+            refresh_jobs();
+        }
+    }
+    if (!a0 || !a1)
+        refresh_jobs();
+    if (a1 || EARLY_EXIT != a2)
+        fast_unlock();
+    complete_job();
+    log_workers();
+    return job_status(a1);
+}
+
+long long job_status(unsigned long a0) {
+    puts("job_status");
+    return a0;
+}
+
+int main(unsigned long a0, struct struct_0 *a1[3]) {
+    return schedule_job(a1[0]->field_0, a1[1]->field_0, a1[2]->field_0);
+}
+

tests/ci/angr_dream_example.linemaps

@@ -0,0 +1,60 @@
+[complete_job]
+1 = [ 96,]
+2 = [ 96,]
+3 = [ 96,]
+4 = [ 96, 100, 107,]
+5 = [ 96, 107, 100,]
+
+[next_job]
+1 = [ 0,]
+2 = [ 0,]
+3 = [ 0,]
+4 = [ 0, 4, 8, 15,]
+5 = [ 20,]
+6 = [ 25, 29,]
+
+[refresh_jobs]
+1 = [ 32,]
+2 = [ 32,]
+3 = [ 32,]
+4 = [ 32, 36, 40, 47,]
+5 = [ 52,]
+6 = [ 57, 61,]
+
+[fast_unlock]
+1 = [ 64,]
+2 = [ 64,]
+3 = [ 64,]
+4 = [ 64, 68, 72, 79,]
+5 = [ 84,]
+6 = [ 89, 93,]
+
+[log_workers]
+1 = [ 112,]
+2 = [ 112,]
+3 = [ 112,]
+4 = [ 112, 116, 123,]
+5 = [ 112, 123, 116,]
+
+[schedule_job]
+1 = [ 160,]
+2 = [ 160, 164, 165, 167, 168, 172, 174,]
+3 = [ 180, 182, 184,]
+4 = [ 195, 189,]
+5 = [ 197, 199,]
+6 = [ 204, 206,]
+10 = [ 224, 226,]
+11 = [ 195, 189,]
+12 = [ 211, 213,]
+13 = [ 235, 237,]
+14 = [ 242, 244,]
+15 = [ 257,]
+
+[job_status]
+1 = [ 128,]
+2 = [ 128, 132, 134, 137, 144,]
+3 = [ 149, 152, 154,]
+
+[main]
+1 = [ 352,]
+2 = [ 380,]