Merge branch 'release-0.4.0'

.bettercodehub.yml

@@ -0,0 +1,25 @@
+component_depth: 1
+languages:
+  - name: python
+    production:
+      exclude:
+        - /muscle_manager_protocol/.*\.py
+        - /scripts/.*\.py
+        - /setup.py
+        - /docs/.*
+        - .*/test/.*\.py
+    test:
+      include:
+        - .*/test/.*\.py
+        - /integration_test/.*\.py
+  - name: cpp
+    production:
+      exclude:
+        - /libmuscle/cpp/src/muscle_manager_protocol/.*
+        - /libmuscle/cpp/src/libmuscle/bindings/.*
+        - /libmuscle/cpp/src/ymmsl/bindings/.*
+        - .*/tests/.*\.?pp
+        - /docs/.*
+    test:
+      include:
+        - .*/tests/.*\.?pp

.github/workflows/ci.yaml

@@ -18,7 +18,7 @@ jobs:
         sudo apt-get update
         sudo apt-get install -y build-essential cmake gfortran libopenmpi-dev pkg-config wget valgrind
         sudo apt-get install -y libssl-dev zlib1g-dev
-        pip install ymmsl==0.10.1
+        pip install ymmsl==0.11.0
 
     - name: Build and run the test suite
       env:

.github/workflows/ci_python3.5.1.yaml

@@ -13,7 +13,7 @@ jobs:
       uses: actions/cache@v1
       with:
         path: ${{ github.workspace }}/.eggs
-        key: python-compatibility-3.5.1-eggs
+        key: python-compatibility-3.5.1-eggs2
 
     - name: Run Python tests on 3.5.1
-      run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.5.1 /bin/bash -c 'cd /home/muscle3 && pip install -U pip setuptools wheel "numpy<1.19" ymmsl==0.10.1 && make test_python_only'
+      run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.5.1 /bin/bash -c 'cd /home/muscle3 && pip install -U pip setuptools wheel "numpy<1.19" ymmsl==0.11.0 && make test_python_only'

.github/workflows/ci_python3.5.yaml

@@ -16,4 +16,4 @@ jobs:
         key: python-compatibility-3.5-eggs
 
     - name: Run Python tests on 3.5 latest
-      run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.5 /bin/bash -c 'cd /home/muscle3 && pip install numpy ymmsl==0.10.1 && make test_python_only'
+      run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.5 /bin/bash -c 'cd /home/muscle3 && pip install numpy ymmsl==0.11.0 && make test_python_only'

.github/workflows/ci_python3.6.1.yaml

@@ -16,4 +16,4 @@ jobs:
         key: python-compatibility-3.6.1-eggs
 
     - name: Run Python tests on 3.6.1 latest
-      run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.6.1 /bin/bash -c 'cd /home/muscle3 && pip install -U pip setuptools wheel ymmsl==0.10.1 && make test_python_only'
+      run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.6.1 /bin/bash -c 'cd /home/muscle3 && pip install -U pip setuptools wheel ymmsl==0.11.0 && make test_python_only'

.github/workflows/ci_python3.6.yaml

@@ -16,4 +16,4 @@ jobs:
         key: python-compatibility-3.6-eggs
 
     - name: Run Python tests on 3.6 latest
-      run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.6 /bin/bash -c 'cd /home/muscle3 && pip install ymmsl==0.10.1 && make test_python_only'
+      run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.6 /bin/bash -c 'cd /home/muscle3 && pip install ymmsl==0.11.0 && make test_python_only'

.github/workflows/ci_python3.7.yaml

@@ -16,4 +16,4 @@ jobs:
         key: python-compatibility-3.7-eggs
 
     - name: Run Python tests on 3.7 latest
-      run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.7 /bin/bash -c 'cd /home/muscle3 && pip install ymmsl==0.10.1 && make test_python_only'
+      run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.7 /bin/bash -c 'cd /home/muscle3 && pip install ymmsl==0.11.0 && make test_python_only'

.github/workflows/ci_python3.8.yaml

@@ -16,4 +16,4 @@ jobs:
         key: python-compatibility-3.8-eggs
 
     - name: Run Python tests on 3.8 latest
-      run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.8 /bin/bash -c 'cd /home/muscle3 && pip install ymmsl==0.10.1 && make test_python_only'
+      run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.8 /bin/bash -c 'cd /home/muscle3 && pip install ymmsl==0.11.0 && make test_python_only'

.github/workflows/ci_ubuntu16.04.yaml

@@ -15,4 +15,4 @@ jobs:
     - uses: actions/checkout@v2
 
     - name: Run tests on Ubuntu 16.04
-      run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:16.04 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran valgrind libopenmpi-dev pkg-config python3 python3-pip python3-venv curl && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && pip3 install ymmsl==0.10.1 && make test_examples'
+      run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:16.04 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran valgrind libopenmpi-dev pkg-config python3 python3-pip python3-venv curl && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && pip3 install ymmsl==0.11.0 && make test_examples'

.github/workflows/ci_ubuntu18.04.yaml

@@ -15,4 +15,4 @@ jobs:
     - uses: actions/checkout@v2
 
     - name: Run tests on Ubuntu 18.04
-      run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:18.04 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran valgrind libopenmpi-dev pkg-config python3 python3-pip python3-venv curl && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && pip3 install ymmsl==0.10.1 && make test_examples'
+      run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:18.04 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran valgrind libopenmpi-dev pkg-config python3 python3-pip python3-venv curl && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && pip3 install ymmsl==0.11.0 && make test_examples'

.github/workflows/ci_ubuntu20.04.yaml

@@ -15,4 +15,4 @@ jobs:
     - uses: actions/checkout@v2
 
     - name: Run tests on Ubuntu 20.04
-      run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:20.04 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran valgrind libopenmpi-dev pkg-config python3 python3-pip python3-venv curl && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && pip3 install ymmsl==0.10.1 && make test_examples'
+      run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:20.04 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran valgrind libopenmpi-dev pkg-config python3 python3-pip python3-venv curl && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && pip3 install ymmsl==0.11.0 && make test_examples'

.github/workflows/ci_ubuntu20.10.yaml

@@ -15,4 +15,4 @@ jobs:
     - uses: actions/checkout@v2
 
     - name: Run tests on Ubuntu 20.10
-      run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:20.10 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran valgrind libopenmpi-dev pkg-config python3 python3-pip python3-venv curl && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && pip3 install ymmsl==0.10.1 && make test_examples'
+      run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:20.10 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran valgrind libopenmpi-dev pkg-config python3 python3-pip python3-venv curl && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && pip3 install ymmsl==0.11.0 && make test_examples'

CHANGELOG.rst

@@ -5,6 +5,28 @@ Change Log
 All notable changes to this project will be documented in this file.
 This project adheres to `Semantic Versioning <http://semver.org/>`_.
 
+0.4.0
+*****
+
+Incompatible changes
+--------------------
+
+* `compute_elements` are now called `components` in .ymmsl files
+
+Improved
+--------
+
+* Use latest OpenSSL library when installing it automatically
+
+Fixed
+-----
+
+* Handling of non-contiguous and F-order numpy arrays
+* C++ memory usage for large dicts/lists now more reasonable
+* Improved shutdown when Python submodel crashes
+* Logging warning message
+
+
 0.3.2
 *****
 

CITATION.cff

@@ -1,18 +1,57 @@
 # YAML 1.2
 ---
+cff-version: "1.1.0"
+
+title: "MUSCLE 3: The Multiscale Coupling Library and Environment"
+doi: "10.5281/zenodo.3258864"
+repository-code: "https://github.com/multiscale/muscle3"
+
+message: "If you use MUSCLE 3, please cite it using these metadata."
 authors:
   -
     family-names: Veen
     given-names: Lourens
     orcid: "https://orcid.org/0000-0002-6311-1168"
-cff-version: "1.1.0"
-doi: "10.5281/zenodo.3258864"
+
 keywords:
   - multiscale
   - coupling
   - MUSCLE
+
 license: "Apache-2.0"
-message: "If you use MUSCLE 3, please cite it using these metadata."
-repository-code: "https://github.com/multiscale/muscle3"
-title: "MUSCLE 3: The Multiscale Coupling Library and Environment"
+
+references:
+  - type: conference-paper
+    doi: "10.1007/978-3-030-50433-5_33"
+    authors:
+      - family-names: Veen
+        given-names: Lourens E.
+      - family-names: Hoekstra
+        given-names: Alfons G.
+    title: Easing Multiscale Model Design and Coupling with MUSCLE 3
+    year: 2020
+
+    collection-title: Lecture Notes in Computer Science
+    volume: 12142
+    volume-title: Computational Science -- ICCS 2020
+    editors:
+      - family-names: Krzhizhanovskaya
+        given-names: Valeria V.
+      - family-names: Závodszky
+        given-names: Gábor
+      - family-names: Lees
+        given-names: Michael H.
+      - family-names: Dongarra
+        given-names: Jack J.
+      - family-names: Sloot
+        given-names: Peter M. A.
+      - family-names: Brissos
+        given-names: Sèrgio
+      - family-names: Teixeira
+        given-names: João
+    start: 425
+    end: 438
+    publisher:
+      name: Springer International Publishing
+      city: Cham
 ...

VERSION

@@ -1 +1 @@
-0.3.2
+0.4.0

docs/source/cplusplus.rst

@@ -334,25 +334,37 @@ objects containing different kinds of data:
             "existing_object", d4
             );
 
+    // grid
+    // +-----------------+
+    // | 1.0 | 2.0 | 3.0 |
+    // +-----------------+
+    // | 4.0 | 5.0 | 6.0 |
+    // +-----------------+
+    std::vector<double> array = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0};
+    auto d11 = Data::grid(array.data(), {2, 3}, {"row", "col"});
+
     // byte array
     std::vector<char> bytes;
-    auto d11 = Data::byte_array(bytes.data(), bytes.size());
+    auto d12 = Data::byte_array(bytes.data(), bytes.size());
 
     // simple types are created automatically when making a Message
     instance.send("output_port", Message(t_cur, 1.0));
 
     std::string text("A text message");
     instance.send("output_port", Message(t_cur, text));
 
-    // but you have to name dicts, lists, and byte arrays
+    // but you have to name dicts, lists, grids and byte arrays
     instance.send("output_port", Message(t_cur, Data::list("a", 10)));
 
 
 As you can see, sending complex data types with MUSCLE is almost as easy in C++
-as it is in Python. One thing that's still missing is sending and receiving
-multidimensional arrays of numbers. We hope to add that in a future version; for
-now you'll need to put them into a list of numbers manually.
-
+as it is in Python. There is however a bit of a cost to this ease-of-use in the
+form of extra memory usage. If you're sending large amounts of data, then it is
+best to use as many grids as you can, as those are much more efficient than
+dictionaries and lists. In particular, a set of objects (agents for example) is
+much more efficiently sent as a dictionary-of-grids (with one 1D-grid for each
+attribute) than as a list of dictionaries. (The latter will work up to a million
+objects or so, but it will be slower.)
 
 .. code-block:: cpp
 

docs/source/examples/rd_settings.ymmsl

@@ -0,0 +1,14 @@
+ymmsl_version: v0.1
+
+model:
+  name: reaction_diffusion
+
+settings:
+  micro.t_max: 2.469136e-06
+  micro.dt: 2.469136e-08
+  macro.t_max: 0.0001234568
+  macro.dt: 2.469136e-06
+  x_max: 1.0
+  dx: 0.01
+  k: -40500.0
+  d: 0.0405

docs/source/examples/reaction_diffusion.ymmsl

@@ -2,19 +2,9 @@ ymmsl_version: v0.1
 
 model:
   name: reaction_diffusion
-  compute_elements:
+  components:
     macro: diffusion
     micro: reaction
   conduits:
     macro.state_out: micro.initial_state
     micro.final_state: macro.state_in
-
-settings:
-  micro.t_max: 2.469136e-06
-  micro.dt: 2.469136e-08
-  macro.t_max: 0.0001234568
-  macro.dt: 2.469136e-06
-  x_max: 1.0
-  dx: 0.01
-  k: -40500.0
-  d: 0.0405

docs/source/examples/reaction_diffusion_cpp.sh

@@ -9,7 +9,7 @@ fi
 echo 'Running reaction-diffusion in C++'
 
 . python/build/venv/bin/activate
-muscle_manager reaction_diffusion.ymmsl &
+muscle_manager reaction_diffusion.ymmsl rd_settings.ymmsl &
 
 manager_pid=$!
 

docs/source/examples/reaction_diffusion_fortran.sh

@@ -9,7 +9,7 @@ fi
 echo 'Running reaction-diffusion in Fortran'
 
 . python/build/venv/bin/activate
-muscle_manager reaction_diffusion.ymmsl &
+muscle_manager reaction_diffusion.ymmsl rd_settings.ymmsl &
 
 manager_pid=$!
 

docs/source/examples/reaction_diffusion_mc.ymmsl

@@ -2,7 +2,7 @@ ymmsl_version: v0.1
 
 model:
   name: reaction_diffusion_mc
-  compute_elements:
+  components:
     mc: mc_driver
     rr: load_balancer
     macro:

docs/source/examples/reaction_diffusion_mpi_cpp.sh

@@ -9,7 +9,7 @@ fi
 echo 'Running reaction-diffusion with MPI in C++'
 
 . python/build/venv/bin/activate
-muscle_manager reaction_diffusion.ymmsl &
+muscle_manager reaction_diffusion.ymmsl rd_settings.ymmsl &
 
 manager_pid=$!
 

docs/source/examples/reaction_diffusion_mpi_fortran.sh

@@ -9,7 +9,7 @@ fi
 echo 'Running reaction-diffusion MPI in Fortran'
 
 . python/build/venv/bin/activate
-muscle_manager reaction_diffusion.ymmsl &
+muscle_manager reaction_diffusion.ymmsl rd_settings.ymmsl &
 
 manager_pid=$!
 

docs/source/examples/reaction_diffusion_python.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 . python/build/venv/bin/activate
-muscle_manager reaction_diffusion.ymmsl &
+muscle_manager reaction_diffusion.ymmsl rd_settings.ymmsl &
 
 manager_pid=$!
 

docs/source/examples/reaction_diffusion_python_cpp.sh

@@ -9,7 +9,7 @@ fi
 echo 'Running reaction-diffusion in Python and C++'
 
 . python/build/venv/bin/activate
-muscle_manager reaction_diffusion.ymmsl &
+muscle_manager reaction_diffusion.ymmsl rd_settings.ymmsl &
 
 manager_pid=$!
 

docs/source/examples/reaction_diffusion_python_fortran.sh

@@ -9,7 +9,7 @@ fi
 echo 'Running reaction-diffusion in Python and Fortran'
 
 . python/build/venv/bin/activate
-muscle_manager reaction_diffusion.ymmsl &
+muscle_manager reaction_diffusion.ymmsl rd_settings.ymmsl &
 
 manager_pid=$!