Merge branch 'develop' into feature/task_id

.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,13 +1,13 @@
 ---
-name: "\U0001F41B Bug report"
+name: "Bug report"
 about: Create a report to help us improve
 title: "[BUG] "
 labels: bug
 assignees: ''
 
 ---
 
-## 🐛 Bug Report
+## Bug Report
 
 **Describe the bug**
 A clear and concise description of what the bug is.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -1,5 +1,5 @@
 ---
-name: "\U0001F680 Feature request"
+name: "Feature request"
 about: Suggest an idea for Merlin
 title: "[FEAT] "
 labels: enhancement
@@ -9,7 +9,7 @@ assignees: ''
 
 <!-- NOTE: If your feature is related to a bug, please file a Bug report instead! -->
 
-## 🚀 Feature Request
+## Feature Request
 
 **What problem is this feature looking to solve?**
 A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]

.github/ISSUE_TEMPLATE/question.md

@@ -1,9 +1,9 @@
 ---
-name: 🤓 General question
+name: General question
 labels: 'question'
 title: '[Q/A] '
 about: Ask, discuss, debate with the Merlin team
 ---
 
-## 🤓 Question
+## Question
 <!-- (Have you searched the issues page before asking?) -->

.github/workflows/push-pr_workflow.yml

@@ -3,12 +3,96 @@ name: Python CI
 on: [push, pull_request]
 
 jobs:
-  build:
+  Changelog:
+    name: CHANGELOG.md updated
+    runs-on: ubuntu-latest
+    if: github.event_name == 'pull_request'
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v2
+      with:
+        fetch-depth: 0 # Checkout the whole history, in case the target is way far behind
+
+    - name: Check if target branch has been merged
+      run: |
+        if git merge-base --is-ancestor ${{ github.event.pull_request.base.sha }} ${{ github.sha }}; then
+          echo "Target branch has been merged into the source branch."
+        else
+          echo "Target branch has not been merged into the source branch. Please merge in target first."
+          exit 1
+        fi
+
+    - name: Check that CHANGELOG has been updated
+      run: |
+        # If this step fails, this means you haven't updated the CHANGELOG.md file with notes on your contribution.
+        if git diff --name-only ${{ github.event.pull_request.base.sha }} ${{ github.sha }} | grep -q '^CHANGELOG.md$'; then
+          echo "Thanks for helping keep our CHANGELOG up-to-date!"
+        else
+          echo "Please update the CHANGELOG.md file with notes on your contribution."
+          exit 1
+        fi
+
+  Lint:
+    runs-on: ubuntu-latest
+    env:
+      MAX_LINE_LENGTH: 127
+      MAX_COMPLEXITY: 15
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.x'
+
+    - name: Check cache
+      uses: actions/cache@v2
+      with:
+        path: ~/.cache/pip
+        key: ${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }}
+
+    - name: Install dependencies
+      run: |
+        python3 -m pip install --upgrade pip
+        if [ -f requirements.txt ]; then pip install --upgrade -r requirements.txt; fi
+        pip3 install --upgrade -r requirements/dev.txt
+
+    - name: Lint with flake8
+      run: |
+        # stop the build if there are Python syntax errors or undefined names
+        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+        flake8 . --count --max-complexity=$MAX_COMPLEXITY --statistics --max-line-length=$MAX_LINE_LENGTH
+
+    - name: Lint with isort
+      run: |
+        python3 -m isort --check --line-length $MAX_LINE_LENGTH merlin
+        python3 -m isort --check --line-length $MAX_LINE_LENGTH tests
+        python3 -m isort --check --line-length $MAX_LINE_LENGTH *.py
 
+    - name: Lint with Black
+      run: |
+        python3 -m black --check --line-length $MAX_LINE_LENGTH --target-version py38 merlin
+        python3 -m black --check --line-length $MAX_LINE_LENGTH --target-version py38 tests
+        python3 -m black --check --line-length $MAX_LINE_LENGTH --target-version py38 *.py
+
+    - name: Lint with PyLint
+      run: |
+        python3 -m pylint merlin --rcfile=setup.cfg --exit-zero
+        python3 -m pylint tests --rcfile=setup.cfg --exit-zero
+
+  Local-test-suite:
     runs-on: ubuntu-latest
+    env: 
+      GO_VERSION: 1.18.1
+      SINGULARITY_VERSION: 3.9.9
+      OS: linux
+      ARCH: amd64
+
     strategy:
       matrix:
-        python-version: [3.6, 3.7, 3.8, 3.9]
+        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11']
 
     steps:
     - uses: actions/checkout@v2
@@ -17,11 +101,39 @@ jobs:
       with:
         python-version: ${{ matrix.python-version }}
 
+    - name: Check cache
+      uses: actions/cache@v2
+      with:
+        path: ${{ env.pythonLocation }}
+        key: ${{ env.pythonLocation }}-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }}
+
     - name: Install dependencies
       run: |
         python3 -m pip install --upgrade pip
         if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
         pip3 install -r requirements/dev.txt
+        pip freeze
+    
+    - name: Install singularity
+      run: |
+        sudo apt-get update && sudo apt-get install -y \
+          build-essential \
+          libssl-dev \
+          uuid-dev \
+          libgpgme11-dev \
+          squashfs-tools \
+          libseccomp-dev \
+          pkg-config
+        wget https://go.dev/dl/go$GO_VERSION.$OS-$ARCH.tar.gz
+        sudo tar -C /usr/local -xzf go$GO_VERSION.$OS-$ARCH.tar.gz
+        rm go$GO_VERSION.$OS-$ARCH.tar.gz
+        export PATH=$PATH:/usr/local/go/bin
+        wget https://github.com/sylabs/singularity/releases/download/v$SINGULARITY_VERSION/singularity-ce-$SINGULARITY_VERSION.tar.gz
+        tar -xzf singularity-ce-$SINGULARITY_VERSION.tar.gz
+        cd singularity-ce-$SINGULARITY_VERSION
+        ./mconfig && \
+          make -C ./builddir && \
+          sudo make -C ./builddir install
 
     - name: Install merlin to run unit tests
       run: |
@@ -33,17 +145,85 @@ jobs:
         merlin example feature_demo
         pip3 install -r feature_demo/requirements.txt
 
-    - name: Lint with flake8
-      run: |
-        # stop the build if there are Python syntax errors or undefined names
-        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
-        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
-        flake8 . --count --exit-zero --max-complexity=15 --statistics --max-line-length=88
-
     - name: Run pytest over unit test suite
       run: |
-        python3 -m pytest tests/unit/
+        python3 -m pytest -v --order-scope=module tests/unit/
 
-    - name: Run integration test suite, locally
+    - name: Run integration test suite for local tests
       run: |
         python3 tests/integration/run_tests.py --verbose --local
+
+  Distributed-test-suite:
+    runs-on: ubuntu-latest
+    services:
+      # rabbitmq:
+      #   image: rabbitmq:latest
+      #   ports:
+      #     - 5672:5672
+      #   options: --health-cmd "rabbitmqctl node_health_check" --health-interval 10s --health-timeout 5s --health-retries 5
+      # Label used to access the service container
+      redis:
+        # Docker Hub image
+        image: redis
+        # Set health checks to wait until redis has started
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 6379:6379
+
+    strategy:
+      matrix:
+        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11']
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Check cache
+      uses: actions/cache@v2
+      with:
+        path: ${{ env.pythonLocation }}
+        key: ${{ env.pythonLocation }}-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }}
+
+    - name: Install dependencies
+      run: |
+        python3 -m pip install --upgrade pip
+        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+        pip3 install -r requirements/dev.txt
+
+    - name: Install merlin and setup redis as the broker
+      run: |
+        pip3 install -e .
+        merlin config --broker redis
+
+    - name: Install CLI task dependencies generated from the 'feature demo' workflow
+      run: |
+        merlin example feature_demo
+        pip3 install -r feature_demo/requirements.txt
+
+    - name: Run integration test suite for distributed tests
+      env:
+        REDIS_HOST: redis
+        REDIS_PORT: 6379
+      run: |
+        python3 tests/integration/run_tests.py --verbose --distributed
+
+    # - name: Setup rabbitmq config
+    #   run: |
+    #     merlin config --test rabbitmq
+
+    # - name: Run integration test suite for rabbitmq
+    #   env:
+    #     AMQP_URL: amqp://localhost:${{ job.services.rabbitmq.ports[5672] }}
+    #     RABBITMQ_USER: Jimmy_Space
+    #     RABBITMQ_PASS: Alexander_Rules
+    #   ports:
+    #     - ${{ job.services.rabbitmq.ports['5672'] }}
+    #   run: |
+    #     python3 tests/integration/run_tests.py --verbose --ids 31 32

.github/workflows/python-publish.yml

@@ -0,0 +1,36 @@
+# This workflow will upload a Python Package using Twine when a release is created
+# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
+
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+
+name: Upload Python Package
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  deploy:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.x'
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install build
+    - name: Build package
+      run: python -m build
+    - name: Publish package
+      uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29
+      with:
+        user: __token__
+        password: ${{ secrets.PYPI_API_TOKEN }}

.gitignore

@@ -30,6 +30,9 @@ ARCHIVE_DIR
 *_OUTPUT/
 *_OUTPUT_D/
 *_ensemble_*/
+studies/
+appendonlydir/
+cli_test_studies/
 
 # Scheduler logs
 flux.out
@@ -66,3 +69,4 @@ jupyter/testDistributedSamples.py
 dist/
 build/
 .DS_Store
+.vscode/

.lgtm.yml

@@ -0,0 +1,35 @@
+##########################################################################################
+# Customize file classifications.                                                        #
+# Results from files under any classifier will be excluded from LGTM                     #
+# statistics.                                                                            #
+##########################################################################################
+
+##########################################################################################
+# Use the `path_classifiers` block to define changes to the default classification of    #
+# files.                                                                                 #
+##########################################################################################
+
+path_classifiers:
+  test:
+    # Exclude all files to ovveride lgtm defaults
+    - exclude: /
+    # Classify all files in the top-level directories tests/ 
+    # and merlin/examples as test code.
+    - tests
+    - merlin/examples
+
+#########################################################################################
+# Use the `queries` block to change the default display of query results.               #
+# The py/clear-text-logging-sensitive-data exclusion is due to cert and password        #
+# keywords being flagged as security leaks.                                             #
+#########################################################################################
+
+queries:
+  - include: "*"
+  - exclude: "py/clear-text-logging-sensitive-data"
+
+extraction:
+  python:
+    python_setup:
+      version: "3"
+