diff --git a/.bettercodehub.yml b/.bettercodehub.yml
new file mode 100644
index 00000000..cfb0713d
--- /dev/null
+++ b/.bettercodehub.yml
@@ -0,0 +1,25 @@
+component_depth: 1
+languages:
+ - name: python
+ production:
+ exclude:
+ - /muscle_manager_protocol/.*\.py
+ - /scripts/.*\.py
+ - /setup.py
+ - /docs/.*
+ - .*/test/.*\.py
+ test:
+ include:
+ - .*/test/.*\.py
+ - /integration_test/.*\.py
+ - name: cpp
+ production:
+ exclude:
+ - /libmuscle/cpp/src/muscle_manager_protocol/.*
+ - /libmuscle/cpp/src/libmuscle/bindings/.*
+ - /libmuscle/cpp/src/ymmsl/bindings/.*
+ - .*/tests/.*\.?pp
+ - /docs/.*
+ test:
+ include:
+ - .*/tests/.*\.?pp
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 1f2527ba..45d2b554 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -18,7 +18,7 @@ jobs:
sudo apt-get update
sudo apt-get install -y build-essential cmake gfortran libopenmpi-dev pkg-config wget valgrind
sudo apt-get install -y libssl-dev zlib1g-dev
- pip install ymmsl==0.10.1
+ pip install ymmsl==0.11.0
- name: Build and run the test suite
env:
diff --git a/.github/workflows/ci_python3.5.1.yaml b/.github/workflows/ci_python3.5.1.yaml
index 414110fb..32c4b1d7 100644
--- a/.github/workflows/ci_python3.5.1.yaml
+++ b/.github/workflows/ci_python3.5.1.yaml
@@ -13,7 +13,7 @@ jobs:
uses: actions/cache@v1
with:
path: ${{ github.workspace }}/.eggs
- key: python-compatibility-3.5.1-eggs
+ key: python-compatibility-3.5.1-eggs2
- name: Run Python tests on 3.5.1
- run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.5.1 /bin/bash -c 'cd /home/muscle3 && pip install -U pip setuptools wheel "numpy<1.19" ymmsl==0.10.1 && make test_python_only'
+ run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.5.1 /bin/bash -c 'cd /home/muscle3 && pip install -U pip setuptools wheel "numpy<1.19" ymmsl==0.11.0 && make test_python_only'
diff --git a/.github/workflows/ci_python3.5.yaml b/.github/workflows/ci_python3.5.yaml
index 3d79cf5f..5f310a8e 100644
--- a/.github/workflows/ci_python3.5.yaml
+++ b/.github/workflows/ci_python3.5.yaml
@@ -16,4 +16,4 @@ jobs:
key: python-compatibility-3.5-eggs
- name: Run Python tests on 3.5 latest
- run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.5 /bin/bash -c 'cd /home/muscle3 && pip install numpy ymmsl==0.10.1 && make test_python_only'
+ run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.5 /bin/bash -c 'cd /home/muscle3 && pip install numpy ymmsl==0.11.0 && make test_python_only'
diff --git a/.github/workflows/ci_python3.6.1.yaml b/.github/workflows/ci_python3.6.1.yaml
index 75fe0a25..da7822a4 100644
--- a/.github/workflows/ci_python3.6.1.yaml
+++ b/.github/workflows/ci_python3.6.1.yaml
@@ -16,4 +16,4 @@ jobs:
key: python-compatibility-3.6.1-eggs
- name: Run Python tests on 3.6.1 latest
- run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.6.1 /bin/bash -c 'cd /home/muscle3 && pip install -U pip setuptools wheel ymmsl==0.10.1 && make test_python_only'
+ run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.6.1 /bin/bash -c 'cd /home/muscle3 && pip install -U pip setuptools wheel ymmsl==0.11.0 && make test_python_only'
diff --git a/.github/workflows/ci_python3.6.yaml b/.github/workflows/ci_python3.6.yaml
index eab001e9..09478248 100644
--- a/.github/workflows/ci_python3.6.yaml
+++ b/.github/workflows/ci_python3.6.yaml
@@ -16,4 +16,4 @@ jobs:
key: python-compatibility-3.6-eggs
- name: Run Python tests on 3.6 latest
- run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.6 /bin/bash -c 'cd /home/muscle3 && pip install ymmsl==0.10.1 && make test_python_only'
+ run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.6 /bin/bash -c 'cd /home/muscle3 && pip install ymmsl==0.11.0 && make test_python_only'
diff --git a/.github/workflows/ci_python3.7.yaml b/.github/workflows/ci_python3.7.yaml
index 88fbd066..b110f88f 100644
--- a/.github/workflows/ci_python3.7.yaml
+++ b/.github/workflows/ci_python3.7.yaml
@@ -16,4 +16,4 @@ jobs:
key: python-compatibility-3.7-eggs
- name: Run Python tests on 3.7 latest
- run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.7 /bin/bash -c 'cd /home/muscle3 && pip install ymmsl==0.10.1 && make test_python_only'
+ run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.7 /bin/bash -c 'cd /home/muscle3 && pip install ymmsl==0.11.0 && make test_python_only'
diff --git a/.github/workflows/ci_python3.8.yaml b/.github/workflows/ci_python3.8.yaml
index 549b2d38..93756697 100644
--- a/.github/workflows/ci_python3.8.yaml
+++ b/.github/workflows/ci_python3.8.yaml
@@ -16,4 +16,4 @@ jobs:
key: python-compatibility-3.8-eggs
- name: Run Python tests on 3.8 latest
- run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.8 /bin/bash -c 'cd /home/muscle3 && pip install ymmsl==0.10.1 && make test_python_only'
+ run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.8 /bin/bash -c 'cd /home/muscle3 && pip install ymmsl==0.11.0 && make test_python_only'
diff --git a/.github/workflows/ci_ubuntu16.04.yaml b/.github/workflows/ci_ubuntu16.04.yaml
index 7d783789..716bb931 100644
--- a/.github/workflows/ci_ubuntu16.04.yaml
+++ b/.github/workflows/ci_ubuntu16.04.yaml
@@ -15,4 +15,4 @@ jobs:
- uses: actions/checkout@v2
- name: Run tests on Ubuntu 16.04
- run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:16.04 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran valgrind libopenmpi-dev pkg-config python3 python3-pip python3-venv curl && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && pip3 install ymmsl==0.10.1 && make test_examples'
+ run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:16.04 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran valgrind libopenmpi-dev pkg-config python3 python3-pip python3-venv curl && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && pip3 install ymmsl==0.11.0 && make test_examples'
diff --git a/.github/workflows/ci_ubuntu18.04.yaml b/.github/workflows/ci_ubuntu18.04.yaml
index 3d9b313f..0a226a83 100644
--- a/.github/workflows/ci_ubuntu18.04.yaml
+++ b/.github/workflows/ci_ubuntu18.04.yaml
@@ -15,4 +15,4 @@ jobs:
- uses: actions/checkout@v2
- name: Run tests on Ubuntu 18.04
- run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:18.04 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran valgrind libopenmpi-dev pkg-config python3 python3-pip python3-venv curl && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && pip3 install ymmsl==0.10.1 && make test_examples'
+ run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:18.04 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran valgrind libopenmpi-dev pkg-config python3 python3-pip python3-venv curl && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && pip3 install ymmsl==0.11.0 && make test_examples'
diff --git a/.github/workflows/ci_ubuntu19.10.yaml b/.github/workflows/ci_ubuntu19.10.yaml
deleted file mode 100644
index 560d5284..00000000
--- a/.github/workflows/ci_ubuntu19.10.yaml
+++ /dev/null
@@ -1,18 +0,0 @@
-# Run Continuous Integration for the latest Ubuntu release
-# This mainly checks for issues/regressions in the native build
-name: native_compatibility_ubuntu19.10
-on:
- schedule:
- - cron: '0 3 * * 0'
- push:
- branches:
- - 'release-*'
-jobs:
- build:
- runs-on: ubuntu-latest
-
- steps:
- - uses: actions/checkout@v2
-
- - name: Run tests on Ubuntu 19.10
- run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:19.10 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran valgrind libopenmpi-dev pkg-config python3 python3-pip python3-venv curl && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && pip3 install ymmsl==0.10.1 && make test_examples'
diff --git a/.github/workflows/ci_ubuntu20.04.yaml b/.github/workflows/ci_ubuntu20.04.yaml
index ea711557..0b4cb0da 100644
--- a/.github/workflows/ci_ubuntu20.04.yaml
+++ b/.github/workflows/ci_ubuntu20.04.yaml
@@ -15,4 +15,4 @@ jobs:
- uses: actions/checkout@v2
- name: Run tests on Ubuntu 20.04
- run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:20.04 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran valgrind libopenmpi-dev pkg-config python3 python3-pip python3-venv curl && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && pip3 install ymmsl==0.10.1 && make test_examples'
+ run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:20.04 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran valgrind libopenmpi-dev pkg-config python3 python3-pip python3-venv curl && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && pip3 install ymmsl==0.11.0 && make test_examples'
diff --git a/.github/workflows/ci_ubuntu20.10.yaml b/.github/workflows/ci_ubuntu20.10.yaml
index 285eb940..5c2db62c 100644
--- a/.github/workflows/ci_ubuntu20.10.yaml
+++ b/.github/workflows/ci_ubuntu20.10.yaml
@@ -15,4 +15,4 @@ jobs:
- uses: actions/checkout@v2
- name: Run tests on Ubuntu 20.10
- run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:20.10 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran valgrind libopenmpi-dev pkg-config python3 python3-pip python3-venv curl && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && pip3 install ymmsl==0.10.1 && make test_examples'
+ run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:20.10 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran valgrind libopenmpi-dev pkg-config python3 python3-pip python3-venv curl && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && pip3 install ymmsl==0.11.0 && make test_examples'
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
index 2994d239..ed1b6927 100644
--- a/CHANGELOG.rst
+++ b/CHANGELOG.rst
@@ -5,6 +5,28 @@ Change Log
All notable changes to this project will be documented in this file.
This project adheres to `Semantic Versioning `_.
+0.4.0
+*****
+
+Incompatible changes
+--------------------
+
+* `compute_elements` are now called `components` in .ymmsl files
+
+Improved
+--------
+
+* Use latest OpenSSL library when installing it automatically
+
+Fixed
+-----
+
+* Handling of non-contiguous and F-order numpy arrays
+* C++ memory usage for large dicts/lists now more reasonable
+* Improved shutdown when Python submodel crashes
+* Logging warning message
+
+
0.3.2
*****
diff --git a/CITATION.cff b/CITATION.cff
index d576d1a0..9b2c5304 100644
--- a/CITATION.cff
+++ b/CITATION.cff
@@ -1,18 +1,57 @@
# YAML 1.2
---
+cff-version: "1.1.0"
+
+title: "MUSCLE 3: The Multiscale Coupling Library and Environment"
+doi: "10.5281/zenodo.3258864"
+repository-code: "https://github.com/multiscale/muscle3"
+
+message: "If you use MUSCLE 3, please cite it using these metadata."
authors:
-
family-names: Veen
given-names: Lourens
orcid: "https://orcid.org/0000-0002-6311-1168"
-cff-version: "1.1.0"
-doi: "10.5281/zenodo.3258864"
+
keywords:
- multiscale
- coupling
- MUSCLE
+
license: "Apache-2.0"
-message: "If you use MUSCLE 3, please cite it using these metadata."
-repository-code: "https://github.com/multiscale/muscle3"
-title: "MUSCLE 3: The Multiscale Coupling Library and Environment"
+
+references:
+ - type: conference-paper
+ doi: "10.1007/978-3-030-50433-5_33"
+ authors:
+ - family-names: Veen
+ given-names: Lourens E.
+ - family-names: Hoekstra
+ given-names: Alfons G.
+ title: Easing Multiscale Model Design and Coupling with MUSCLE 3
+ year: 2020
+
+ collection-title: Lecture Notes in Computer Science
+ volume: 12142
+ volume-title: Computational Science -- ICCS 2020
+ editors:
+ - family-names: Krzhizhanovskaya
+ given-names: Valeria V.
+ - family-names: Závodszky
+ given-names: Gábor
+ - family-names: Lees
+ given-names: Michael H.
+ - family-names: Dongarra
+ given-names: Jack J.
+ - family-names: Sloot
+ given-names: Peter M. A.
+ - family-names: Brissos
+ given-names: Sèrgio
+ - family-names: Teixeira
+ given-names: João
+ start: 425
+ end: 438
+ publisher:
+ name: Springer International Publishing
+ city: Cham
...
diff --git a/VERSION b/VERSION
index d15723fb..1d0ba9ea 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-0.3.2
+0.4.0
diff --git a/docs/source/cplusplus.rst b/docs/source/cplusplus.rst
index 6b49f09f..aa83b4a4 100644
--- a/docs/source/cplusplus.rst
+++ b/docs/source/cplusplus.rst
@@ -334,9 +334,18 @@ objects containing different kinds of data:
"existing_object", d4
);
+ // grid
+ // +-----------------+
+ // | 1.0 | 2.0 | 3.0 |
+ // +-----------------+
+ // | 4.0 | 5.0 | 6.0 |
+ // +-----------------+
+ std::vector array = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0};
+ auto d11 = Data::grid(array.data(), {2, 3}, {"row", "col"});
+
// byte array
std::vector bytes;
- auto d11 = Data::byte_array(bytes.data(), bytes.size());
+ auto d12 = Data::byte_array(bytes.data(), bytes.size());
// simple types are created automatically when making a Message
instance.send("output_port", Message(t_cur, 1.0));
@@ -344,15 +353,18 @@ objects containing different kinds of data:
std::string text("A text message");
instance.send("output_port", Message(t_cur, text));
- // but you have to name dicts, lists, and byte arrays
+ // but you have to name dicts, lists, grids and byte arrays
instance.send("output_port", Message(t_cur, Data::list("a", 10)));
As you can see, sending complex data types with MUSCLE is almost as easy in C++
-as it is in Python. One thing that's still missing is sending and receiving
-multidimensional arrays of numbers. We hope to add that in a future version; for
-now you'll need to put them into a list of numbers manually.
-
+as it is in Python. There is however a bit of a cost to this ease-of-use in the
+form of extra memory usage. If you're sending large amounts of data, then it is
+best to use as many grids as you can, as those are much more efficient than
+dictionaries and lists. In particular, a set of objects (agents for example) is
+much more efficiently sent as a dictionary-of-grids (with one 1D-grid for each
+attribute) than as a list of dictionaries. (The latter will work up to a million
+objects or so, but it will be slower.)
.. code-block:: cpp
diff --git a/docs/source/examples/rd_settings.ymmsl b/docs/source/examples/rd_settings.ymmsl
new file mode 100644
index 00000000..3aefddde
--- /dev/null
+++ b/docs/source/examples/rd_settings.ymmsl
@@ -0,0 +1,14 @@
+ymmsl_version: v0.1
+
+model:
+ name: reaction_diffusion
+
+settings:
+ micro.t_max: 2.469136e-06
+ micro.dt: 2.469136e-08
+ macro.t_max: 0.0001234568
+ macro.dt: 2.469136e-06
+ x_max: 1.0
+ dx: 0.01
+ k: -40500.0
+ d: 0.0405
diff --git a/docs/source/examples/reaction_diffusion.ymmsl b/docs/source/examples/reaction_diffusion.ymmsl
index 5cf3468b..c75ea225 100644
--- a/docs/source/examples/reaction_diffusion.ymmsl
+++ b/docs/source/examples/reaction_diffusion.ymmsl
@@ -2,19 +2,9 @@ ymmsl_version: v0.1
model:
name: reaction_diffusion
- compute_elements:
+ components:
macro: diffusion
micro: reaction
conduits:
macro.state_out: micro.initial_state
micro.final_state: macro.state_in
-
-settings:
- micro.t_max: 2.469136e-06
- micro.dt: 2.469136e-08
- macro.t_max: 0.0001234568
- macro.dt: 2.469136e-06
- x_max: 1.0
- dx: 0.01
- k: -40500.0
- d: 0.0405
diff --git a/docs/source/examples/reaction_diffusion_cpp.sh b/docs/source/examples/reaction_diffusion_cpp.sh
index 4e421527..f984a421 100755
--- a/docs/source/examples/reaction_diffusion_cpp.sh
+++ b/docs/source/examples/reaction_diffusion_cpp.sh
@@ -9,7 +9,7 @@ fi
echo 'Running reaction-diffusion in C++'
. python/build/venv/bin/activate
-muscle_manager reaction_diffusion.ymmsl &
+muscle_manager reaction_diffusion.ymmsl rd_settings.ymmsl &
manager_pid=$!
diff --git a/docs/source/examples/reaction_diffusion_fortran.sh b/docs/source/examples/reaction_diffusion_fortran.sh
index bdce14eb..6792da11 100755
--- a/docs/source/examples/reaction_diffusion_fortran.sh
+++ b/docs/source/examples/reaction_diffusion_fortran.sh
@@ -9,7 +9,7 @@ fi
echo 'Running reaction-diffusion in Fortran'
. python/build/venv/bin/activate
-muscle_manager reaction_diffusion.ymmsl &
+muscle_manager reaction_diffusion.ymmsl rd_settings.ymmsl &
manager_pid=$!
diff --git a/docs/source/examples/reaction_diffusion_mc.ymmsl b/docs/source/examples/reaction_diffusion_mc.ymmsl
index 4e50db94..ccfa2f6b 100644
--- a/docs/source/examples/reaction_diffusion_mc.ymmsl
+++ b/docs/source/examples/reaction_diffusion_mc.ymmsl
@@ -2,7 +2,7 @@ ymmsl_version: v0.1
model:
name: reaction_diffusion_mc
- compute_elements:
+ components:
mc: mc_driver
rr: load_balancer
macro:
diff --git a/docs/source/examples/reaction_diffusion_mpi_cpp.sh b/docs/source/examples/reaction_diffusion_mpi_cpp.sh
index dcc18da1..338ae0b8 100755
--- a/docs/source/examples/reaction_diffusion_mpi_cpp.sh
+++ b/docs/source/examples/reaction_diffusion_mpi_cpp.sh
@@ -9,7 +9,7 @@ fi
echo 'Running reaction-diffusion with MPI in C++'
. python/build/venv/bin/activate
-muscle_manager reaction_diffusion.ymmsl &
+muscle_manager reaction_diffusion.ymmsl rd_settings.ymmsl &
manager_pid=$!
diff --git a/docs/source/examples/reaction_diffusion_mpi_fortran.sh b/docs/source/examples/reaction_diffusion_mpi_fortran.sh
index acdd6566..394b74d4 100755
--- a/docs/source/examples/reaction_diffusion_mpi_fortran.sh
+++ b/docs/source/examples/reaction_diffusion_mpi_fortran.sh
@@ -9,7 +9,7 @@ fi
echo 'Running reaction-diffusion MPI in Fortran'
. python/build/venv/bin/activate
-muscle_manager reaction_diffusion.ymmsl &
+muscle_manager reaction_diffusion.ymmsl rd_settings.ymmsl &
manager_pid=$!
diff --git a/docs/source/examples/reaction_diffusion_python.sh b/docs/source/examples/reaction_diffusion_python.sh
index 843dbf11..7f9ff37d 100755
--- a/docs/source/examples/reaction_diffusion_python.sh
+++ b/docs/source/examples/reaction_diffusion_python.sh
@@ -1,7 +1,7 @@
#!/bin/bash
. python/build/venv/bin/activate
-muscle_manager reaction_diffusion.ymmsl &
+muscle_manager reaction_diffusion.ymmsl rd_settings.ymmsl &
manager_pid=$!
diff --git a/docs/source/examples/reaction_diffusion_python_cpp.sh b/docs/source/examples/reaction_diffusion_python_cpp.sh
index f04c0276..bbff9fab 100755
--- a/docs/source/examples/reaction_diffusion_python_cpp.sh
+++ b/docs/source/examples/reaction_diffusion_python_cpp.sh
@@ -9,7 +9,7 @@ fi
echo 'Running reaction-diffusion in Python and C++'
. python/build/venv/bin/activate
-muscle_manager reaction_diffusion.ymmsl &
+muscle_manager reaction_diffusion.ymmsl rd_settings.ymmsl &
manager_pid=$!
diff --git a/docs/source/examples/reaction_diffusion_python_fortran.sh b/docs/source/examples/reaction_diffusion_python_fortran.sh
index 5aef427a..60b41469 100755
--- a/docs/source/examples/reaction_diffusion_python_fortran.sh
+++ b/docs/source/examples/reaction_diffusion_python_fortran.sh
@@ -9,7 +9,7 @@ fi
echo 'Running reaction-diffusion in Python and Fortran'
. python/build/venv/bin/activate
-muscle_manager reaction_diffusion.ymmsl &
+muscle_manager reaction_diffusion.ymmsl rd_settings.ymmsl &
manager_pid=$!
diff --git a/docs/source/fortran.rst b/docs/source/fortran.rst
index cd817464..9ffd4575 100644
--- a/docs/source/fortran.rst
+++ b/docs/source/fortran.rst
@@ -346,7 +346,8 @@ your model!):
.. code-block:: fortran
- type(LIBMUSCLE_Data) :: d1, d2, d3, d4, d5, d6, d7, d8, d9, d10, d11
+ type(LIBMUSCLE_Data) :: d1, d2, d3, d4, d5, d6, d7, d8, d9, d10, d11, d12
+ integer (LIBMUSCLE_int8), dimension(2, 3) :: ar
character(len=1), dimension(1024) :: bytes
! create a Data containing a nil value (None in Python)
@@ -377,13 +378,24 @@ your model!):
call LIBMUSCLE_Data_set_item(d10, 'note', 'Keys must be strings')
call LIBMUSCLE_Data_set_item(d10, 'nest all you want', d9)
+ ! grid
+ ar = reshape(spread((/1_LIBMUSCLE_int8/), 1, 6), (/2, 3/))
+ d12 = LIBMUSCLE_Data_create_grid(ar)
+
! byte array
- d11 = LIBMUSCLE_Data_create_byte_array(bytes)
+ d12 = LIBMUSCLE_Data_create_byte_array(bytes)
As you can see, sending complex data types with MUSCLE is a bit more difficult
in Fortran than in Python, but it is not too burdensome.
+If you want to send large amounts of data, then use grids (arrays) as much as
+possible. Lists and dicts are more flexible, but they are also slower and take
+up much more memory. In particular, a set of objects (agents for example) is
+better sent as a dict of arrays (with one 1D array for each attribute) than as a
+list of dicts. (The latter will probably work up to 1 million objects or so, but
+it will still be slower.)
+
Handling errors
```````````````
diff --git a/docs/source/index.rst b/docs/source/index.rst
index d584d604..45291e2d 100644
--- a/docs/source/index.rst
+++ b/docs/source/index.rst
@@ -9,6 +9,21 @@ Welcome to MUSCLE 3's documentation!
MUSCLE 3 is the third incarnation of the Multiscale Coupling Library and
Environment.
+With MUSCLE 3, you can connect multiple simulation models together into a
+multiscale simulation. Simulation models can be as simple as a single Python
+file, or as complex as a combination of multiple separate simulation codes
+written in C++ or Fortran, and running on an HPC machine.
+
+If you use MUSCLE 3 for scientific work, please `cite
+the version of the MUSCLE 3 software you used
+`_ and the following paper:
+
+Veen L.E., Hoekstra A.G. (2020) Easing Multiscale Model Design and Coupling
+with MUSCLE 3. In: Krzhizhanovskaya V. et al. (eds) Computational Science –
+ICCS 2020. ICCS 2020. Lecture Notes in Computer Science, vol 12142. Springer,
+Cham. ``_
+
+
.. toctree::
:maxdepth: 4
:caption: Contents:
diff --git a/docs/source/installing.rst.in b/docs/source/installing.rst.in
index d57974c1..0560b3c9 100644
--- a/docs/source/installing.rst.in
+++ b/docs/source/installing.rst.in
@@ -109,7 +109,7 @@ The dependencies are:
If your model uses any of these dependencies directly, then it's best to install
that dependency on your system, either via the package manager or from source,
and then link both your library and MUSCLE 3 to the dependency. (See below for
-how t opoint the build to your installation.) This avoids having two different
+how to point the build to your installation.) This avoids having two different
versions around and active at the same time. Otherwise, it's easier to rely on
the automatic installation. Note that the gRPC and Protobuf dependencies are
exact; getting them to install and work correctly on all systems is
diff --git a/docs/source/tutorial.rst b/docs/source/tutorial.rst
index 681189de..1590ba5b 100644
--- a/docs/source/tutorial.rst
+++ b/docs/source/tutorial.rst
@@ -248,8 +248,9 @@ and is set to ``None`` here.
MUSCLE 3 uses `MessagePack `_ to encode messages between
models. MessagePack is a binary encoding format which can be thought of as a
binary version of JSON. That means that the message can be an integer, float,
-bool, string, or a list or dictionary containing such. Like with JSON, these can
-be nested, so you can send a dictionary containing lists of floats for example.
+bool, string, or a list or dictionary containing such, and MUSCLE 3 also
+supports NumPy arrays and byte arrays. Like with JSON, these can be nested, so
+you can send a dictionary containing lists of floats for example.
MessagePack is self-describing, so you can inspect the received message to find
out what you were sent. In most cases, all data you send on a particular
@@ -268,6 +269,13 @@ name of the dimensions. In this case there's only one, and ``x`` is not very
descriptive, so we could have also passed ``U`` directly, in which case MUSCLE
would have sent a :class:`libmuscle.Grid` without index names automatically.
+Note that grids (and NumPy arrays) are much more efficient than lists and
+dictionaries. If you have a lot of data to send, then you should use those as
+much as possible. For example, a set of agents is best sent as a dictionary of
+1D NumPy arrays, with one array/grid for each attribute. A list of dictionaries
+will be quite a bit slower and use a lot more memory, but it should work up to
+a million or so objects.
+
Finally, if you want to use your own encoding, you can just send a ``bytes``
object, which will be transmitted as-is, with minimal overhead.
diff --git a/integration_test/conftest.py b/integration_test/conftest.py
index a64b0717..b2ae93f3 100644
--- a/integration_test/conftest.py
+++ b/integration_test/conftest.py
@@ -67,7 +67,7 @@ def mmp_server_process(yatiml_log_warning):
'ymmsl_version: v0.1\n'
'model:\n'
' name: test_model\n'
- ' compute_elements:\n'
+ ' components:\n'
' macro: macro_implementation\n'
' micro:\n'
' implementation: micro_implementation\n'
@@ -84,6 +84,9 @@ def mmp_server_process(yatiml_log_warning):
' test6:\n'
' - [1.0, 2.0]\n'
' - [3.0, 1.0]\n'
+ 'implementations:\n'
+ ' macro_implementation: macro.py\n'
+ ' micro_implementation: micro.py\n'
)
ymmsl_doc = ymmsl.load(ymmsl_text)
@@ -96,7 +99,7 @@ def mmp_server_process_simple(yatiml_log_warning):
'ymmsl_version: v0.1\n'
'model:\n'
' name: test_model\n'
- ' compute_elements:\n'
+ ' components:\n'
' macro: macro_implementation\n'
' micro: micro_implementation\n'
' conduits:\n'
@@ -123,7 +126,7 @@ def mmp_server(yatiml_log_warning):
'ymmsl_version: v0.1\n'
'model:\n'
' name: test_model\n'
- ' compute_elements:\n'
+ ' components:\n'
' macro: macro_implementation\n'
' micro:\n'
' implementation: micro_implementation\n'
diff --git a/integration_test/test_all.py b/integration_test/test_all.py
index f314396a..8764f3a4 100644
--- a/integration_test/test_all.py
+++ b/integration_test/test_all.py
@@ -1,7 +1,7 @@
from collections import OrderedDict
import numpy as np
-from ymmsl import (ComputeElement, Conduit, Configuration, Model, Operator,
+from ymmsl import (Component, Conduit, Configuration, Model, Operator,
Settings)
from libmuscle import Grid, Instance, Message
@@ -63,8 +63,8 @@ def test_all(log_file_in_tmpdir):
"""A positive all-up test of everything.
"""
elements = [
- ComputeElement('macro', 'macro_impl'),
- ComputeElement('micro', 'micro_impl', [10])]
+ Component('macro', 'macro_impl'),
+ Component('micro', 'micro_impl', [10])]
conduits = [
Conduit('macro.out', 'micro.in'),
diff --git a/integration_test/test_cpp_mmp_client.py b/integration_test/test_cpp_mmp_client.py
index 1c51de0a..54faaafe 100644
--- a/integration_test/test_cpp_mmp_client.py
+++ b/integration_test/test_cpp_mmp_client.py
@@ -20,7 +20,7 @@ def do_mmp_client_test(caplog):
'ymmsl_version: v0.1\n'
'model:\n'
' name: test_model\n'
- ' compute_elements:\n'
+ ' components:\n'
' macro: macro_implementation\n'
' micro:\n'
' implementation: micro_implementation\n'
@@ -83,10 +83,12 @@ def mock_remove(name: Reference):
assert result.returncode == 0
# check submit_log_message
- assert caplog.records[0].name == 'instances.test_logging'
- assert caplog.records[0].time_stamp == '1970-01-01T00:00:02Z'
- assert caplog.records[0].levelname == 'CRITICAL'
- assert caplog.records[0].message == 'Integration testing'
+ for rec in caplog.records:
+ if rec.name == 'instances.test_logging':
+ assert rec.time_stamp == '1970-01-01T00:00:02Z'
+ assert rec.levelname == 'CRITICAL'
+ assert rec.message == 'Integration testing'
+ break
# check register_instance
assert (instance_registry.get_locations('micro[3]') ==
diff --git a/integration_test/test_duplication_mapper.py b/integration_test/test_duplication_mapper.py
index 36dbf3ba..9717ad36 100644
--- a/integration_test/test_duplication_mapper.py
+++ b/integration_test/test_duplication_mapper.py
@@ -1,4 +1,4 @@
-from ymmsl import (ComputeElement, Conduit, Configuration, Operator, Model,
+from ymmsl import (Component, Conduit, Configuration, Operator, Model,
Settings)
from libmuscle import Instance, Message
@@ -36,9 +36,9 @@ def test_duplication_mapper(log_file_in_tmpdir):
This is an acyclic workflow.
"""
elements = [
- ComputeElement('dm', 'muscle.duplication_mapper'),
- ComputeElement('first', 'receiver'),
- ComputeElement('second', 'receiver')]
+ Component('dm', 'muscle.duplication_mapper'),
+ Component('first', 'receiver'),
+ Component('second', 'receiver')]
conduits = [
Conduit('dm.out', 'first.in'),
diff --git a/integration_test/test_logging.py b/integration_test/test_logging.py
index 1c92a694..c5ced013 100644
--- a/integration_test/test_logging.py
+++ b/integration_test/test_logging.py
@@ -16,7 +16,7 @@ def do_logging_test(caplog):
'ymmsl_version: v0.1\n'
'model:\n'
' name: test_model\n'
- ' compute_elements:\n'
+ ' components:\n'
' macro: macro_implementation\n'
' micro:\n'
' implementation: micro_implementation\n'
@@ -54,10 +54,12 @@ def do_logging_test(caplog):
# log and check
client.submit_log_message(message)
- assert caplog.records[0].name == 'instances.test_logging'
- assert caplog.records[0].time_stamp == '1970-01-01T00:00:02Z'
- assert caplog.records[0].levelname == 'DEBUG'
- assert caplog.records[0].message == 'Integration testing'
+ for rec in caplog.records:
+ if rec.name == 'instances.test_logging':
+ assert rec.time_stamp == '1970-01-01T00:00:02Z'
+ assert rec.levelname == 'DEBUG'
+ assert rec.message == 'Integration testing'
+ break
server.stop()
diff --git a/integration_test/test_parameter_overlays.py b/integration_test/test_parameter_overlays.py
index a4211cef..cf091193 100644
--- a/integration_test/test_parameter_overlays.py
+++ b/integration_test/test_parameter_overlays.py
@@ -1,6 +1,6 @@
from collections import OrderedDict
-from ymmsl import (ComputeElement, Conduit, Configuration, Model, Operator,
+from ymmsl import (Component, Conduit, Configuration, Model, Operator,
Settings)
from libmuscle import Instance, Message
@@ -94,11 +94,11 @@ def test_settings_overlays(log_file_in_tmpdir):
"""A positive all-up test of settings overlays.
"""
elements = [
- ComputeElement('qmc', 'qmc'),
- ComputeElement('macro', 'macro', [10]),
- ComputeElement('relay', 'explicit_relay'),
- ComputeElement('relay2', 'explicit_relay'),
- ComputeElement('micro', 'micro', [10])]
+ Component('qmc', 'qmc'),
+ Component('macro', 'macro', [10]),
+ Component('relay', 'explicit_relay'),
+ Component('relay2', 'explicit_relay'),
+ Component('micro', 'micro', [10])]
conduits = [
Conduit('qmc.settings_out', 'macro.muscle_settings_in'),
diff --git a/integration_test/test_registration.py b/integration_test/test_registration.py
index 77cc81e9..3ed68d96 100644
--- a/integration_test/test_registration.py
+++ b/integration_test/test_registration.py
@@ -14,7 +14,7 @@ def test_registration(log_file_in_tmpdir, mmp_server):
client.register_instance(instance_name, ['tcp://localhost:10000'],
[port])
- servicer = mmp_server._MMPServer__servicer
+ servicer = mmp_server._servicer
registry = servicer._MMPServicer__instance_registry
assert registry.get_locations(instance_name) == ['tcp://localhost:10000']
diff --git a/libmuscle/cpp/build/grpc/Makefile b/libmuscle/cpp/build/grpc/Makefile
index 5c688488..ad61c0d1 100644
--- a/libmuscle/cpp/build/grpc/Makefile
+++ b/libmuscle/cpp/build/grpc/Makefile
@@ -40,7 +40,7 @@ include $(TOOLDIR)/make_available.make
dep_name := openssl
dep_version_constraint := >= 1.0.2
-dep_version := 1.1.1f
+dep_version := 1.1.1i
dep_pkgconfig_name := openssl
dep_install := 1
include $(TOOLDIR)/make_available.make
diff --git a/libmuscle/cpp/build/libmuscle/tests/Makefile b/libmuscle/cpp/build/libmuscle/tests/Makefile
index e285659a..45304a4e 100644
--- a/libmuscle/cpp/build/libmuscle/tests/Makefile
+++ b/libmuscle/cpp/build/libmuscle/tests/Makefile
@@ -95,3 +95,7 @@ test_dep_lib_paths := $(subst $(space),:,$(foreach DIR,$(DEP_DIRS),$(DIR)/lib))
run_test%: test%
export LD_LIBRARY_PATH=$(test_dep_lib_paths) ; $(VALGRIND) ./$<
+# This uses setrlimit, which is not compatible with Valgrind, so run without
+.PHONY: run_test_data_memory_use
+run_test_data_memory_use: test_data_memory_use
+ export LD_LIBRARY_PATH=$(test_dep_lib_paths) ; ./$<
diff --git a/libmuscle/cpp/src/libmuscle/data.cpp b/libmuscle/cpp/src/libmuscle/data.cpp
index c7b0245c..dee33ef6 100644
--- a/libmuscle/cpp/src/libmuscle/data.cpp
+++ b/libmuscle/cpp/src/libmuscle/data.cpp
@@ -97,7 +97,6 @@ DataConstRef DataConstRef::grid_data_(
DataConstRef::DataConstRef()
: mp_zones_(new std::vector>())
{
- mp_zones_->push_back(std::make_shared());
mp_obj_ = zone_alloc_();
mp_obj_->type = msgpack::type::NIL;
}
@@ -846,17 +845,23 @@ Data Data::byte_array(char const * buf, uint32_t size) {
Data & Data::operator=(Data const & rhs) {
if (mp_obj_ != rhs.mp_obj_) {
*mp_obj_ = *rhs.mp_obj_;
-
- // We can't overwrite mp_zones_ here, because mp_obj_ is allocated on
- // one of them, and we don't know which. So we just append, which is
- // suboptimal because it may keep objects alive that are no longer
- // reachable. Consider a separate shared_ptr to the zone that mp_obj_
- // is on (in a separate member), so that we can safely overwrite
- // mp_zones_.
- if (mp_zones_ != rhs.mp_zones_)
- mp_zones_->insert(mp_zones_->end(),
- rhs.mp_zones_->cbegin(), rhs.mp_zones_->cend());
obj_cache_ = rhs.obj_cache_;
+
+ if (
+ rhs.mp_obj_->type == msgpack::type::STR ||
+ rhs.mp_obj_->type == msgpack::type::BIN ||
+ rhs.mp_obj_->type == msgpack::type::ARRAY ||
+ rhs.mp_obj_->type == msgpack::type::MAP ||
+ rhs.mp_obj_->type == msgpack::type::EXT)
+ {
+ // The above assignment will only copy the pointer for these
+ // types. So we need to add the source zones to our own to
+ // ensure that the data structure pointed to by that pointer
+ // continues to exist for as long as we do.
+ if (mp_zones_ != rhs.mp_zones_)
+ mp_zones_->insert(mp_zones_->end(),
+ rhs.mp_zones_->cbegin(), rhs.mp_zones_->cend());
+ }
}
return *this;
}
diff --git a/libmuscle/cpp/src/libmuscle/data.tpp b/libmuscle/cpp/src/libmuscle/data.tpp
index 3276fb1b..c2a4d5ec 100644
--- a/libmuscle/cpp/src/libmuscle/data.tpp
+++ b/libmuscle/cpp/src/libmuscle/data.tpp
@@ -30,6 +30,8 @@ T DataConstRef::as() const {
template
T * DataConstRef::zone_alloc_(uint32_t size) {
+ if (mp_zones_->empty())
+ mp_zones_->push_back(std::make_shared(24));
auto num_bytes = sizeof(T) * size;
return static_cast((*mp_zones_)[0]->allocate_align(
num_bytes, MSGPACK_ZONE_ALIGNOF(T)));
diff --git a/libmuscle/cpp/src/libmuscle/tests/test_data_memory_use.cpp b/libmuscle/cpp/src/libmuscle/tests/test_data_memory_use.cpp
new file mode 100644
index 00000000..e4913fcd
--- /dev/null
+++ b/libmuscle/cpp/src/libmuscle/tests/test_data_memory_use.cpp
@@ -0,0 +1,98 @@
+#include "libmuscle/data.hpp"
+#include "libmuscle/mcp/data_pack.hpp"
+
+#include
+#include
+#include
+#include
+
+#include
+#include
+#include
+
+#include
+
+
+using libmuscle::impl::Data;
+
+
+int main(int argc, char *argv[]) {
+ ::testing::InitGoogleTest(&argc, argv);
+ return RUN_ALL_TESTS();
+}
+
+TEST(libmuscle_mcp_data, large_structure) {
+ // produce a large with items in it
+ // regression test for excess memory usage
+
+ // at default malloc settings, we can measure to about 128kB accuracy
+ const std::size_t granularity = 128u * 1024u;
+
+ // we want to measure to 1% accuracy
+ const std::size_t data_size = granularity * 100u;
+
+ // we set max overhead at 110%, because msgpack's allocator allocates
+ // in powers of 2 so we need 100% for that, and a bit of overhead for
+ // the list itself and overhead.
+ const std::size_t overhead = data_size * 110 / 100;
+
+ // data segment before allocating is about 200kB
+ const std::size_t base_size = 200 * 1024;
+
+ const std::size_t max_size = base_size + data_size + overhead;
+
+ // tell the kernel to kill us if we're using too much memory
+ struct rlimit old_limits, limits;
+ getrlimit(RLIMIT_DATA, &old_limits);
+ getrlimit(RLIMIT_DATA, &limits);
+ limits.rlim_cur = max_size;
+ setrlimit(RLIMIT_DATA, &limits);
+
+ // This many items are needed to reach the given data size
+ // A MsgPack object is 24 bytes
+ const std::size_t num_items = data_size / 24;
+
+ malloc_info(0, stderr);
+
+ Data list = Data::nils(num_items);
+ malloc_info(0, stderr);
+ for (std::size_t i = 0; i < num_items; ++i)
+ list[i] = Data(42424242);
+
+ malloc_info(0, stderr);
+
+ // reset memory limit
+ setrlimit(RLIMIT_DATA, &old_limits);
+}
+
+TEST(libmuscle_mcp_data, nested_structure) {
+ // produce a large list with nested structures in it
+ // regression test for excess memory usage
+
+ // Originally reported with a 10M item list, which would try to use
+ // 600GB. Scaled down here to speed things up, and we want to do at least
+ // 100x better than that, so 10k items in 6MB.
+
+ // tell the kernel to kill us if we're using too much memory
+ struct rlimit old_limits, limits;
+ getrlimit(RLIMIT_DATA, &old_limits);
+ getrlimit(RLIMIT_DATA, &limits);
+ limits.rlim_cur = 6 * 1024 * 1024;
+ setrlimit(RLIMIT_DATA, &limits);
+
+ const std::size_t num_items = 10000;
+ malloc_info(0, stderr);
+
+ Data list = Data::nils(num_items);
+ malloc_info(0, stderr);
+ for (std::size_t i = 0; i < num_items; ++i) {
+ auto pos = Data::list(1.2, 3.4, 5.6);
+ list[i] = Data::list(pos, 7.8, 2);
+ }
+
+ malloc_info(0, stderr);
+
+ // reset memory limit
+ setrlimit(RLIMIT_DATA, &old_limits);
+}
+
diff --git a/libmuscle/python/libmuscle/communicator.py b/libmuscle/python/libmuscle/communicator.py
index 49f3ec06..0391501f 100644
--- a/libmuscle/python/libmuscle/communicator.py
+++ b/libmuscle/python/libmuscle/communicator.py
@@ -396,16 +396,16 @@ def __ports_from_conduits(self, conduits: List[Conduit]
"""
ports = dict()
for conduit in conduits:
- if conduit.sending_compute_element() == self._kernel:
+ if conduit.sending_component() == self._kernel:
port_id = conduit.sending_port()
operator = Operator.O_F
port_peer_dims = self._peer_manager.get_peer_dims(
- conduit.receiving_compute_element())
- elif conduit.receiving_compute_element() == self._kernel:
+ conduit.receiving_component())
+ elif conduit.receiving_component() == self._kernel:
port_id = conduit.receiving_port()
operator = Operator.F_INIT
port_peer_dims = self._peer_manager.get_peer_dims(
- conduit.sending_compute_element())
+ conduit.sending_component())
else:
continue
@@ -425,14 +425,14 @@ def __settings_in_port(self, conduits: List[Conduit]) -> Port:
conduits: The list of conduits.
"""
for conduit in conduits:
- if conduit.receiving_compute_element() == self._kernel:
+ if conduit.receiving_component() == self._kernel:
port_id = conduit.receiving_port()
if str(port_id) == 'muscle_settings_in':
return Port(str(port_id), Operator.F_INIT, False,
self._peer_manager.is_connected(port_id),
len(self._index),
self._peer_manager.get_peer_dims(
- conduit.sending_compute_element()))
+ conduit.sending_component()))
return Port('muscle_settings_in', Operator.F_INIT, False, False,
len(self._index), [])
diff --git a/libmuscle/python/libmuscle/instance.py b/libmuscle/python/libmuscle/instance.py
index 13338b2d..23923969 100644
--- a/libmuscle/python/libmuscle/instance.py
+++ b/libmuscle/python/libmuscle/instance.py
@@ -1,5 +1,7 @@
from copy import copy
import logging
+import os
+from pathlib import Path
import sys
from typing import cast, Dict, List, Optional, Tuple
@@ -20,6 +22,9 @@
_logger = logging.getLogger(__name__)
+_FInitCacheType = Dict[Tuple[str, Optional[int]], Message]
+
+
class Instance:
"""Represents a compute element instance in a MUSCLE3 simulation.
@@ -62,8 +67,7 @@ def __init__(self, ports: Optional[Dict[Operator, List[str]]] = None
self._first_run = True
"""Keeps track of whether this is the first reuse run."""
- FInitCacheType = Dict[Tuple[str, Optional[int]], Message]
- self._f_init_cache = dict() # type: FInitCacheType
+ self._f_init_cache = dict() # type: _FInitCacheType
self._register()
self._connect()
@@ -406,7 +410,8 @@ def __set_up_logging(self) -> None:
"""
id_str = str(self._instance_name())
- logfile = extract_log_file_location('muscle3.{}.log'.format(id_str))
+ logfile = extract_log_file_location(
+ Path.cwd(), 'muscle3.{}.log'.format(id_str))
local_handler = logging.FileHandler(str(logfile), mode='w')
formatter = logging.Formatter('%(asctime)-15s: %(name)s'
' %(levelname)s: %(message)s')
@@ -507,9 +512,14 @@ def split_reference(ref: Reference) -> Tuple[Reference, List[int]]:
name, index = split_reference(prefix_ref)
break
else:
- raise RuntimeError(('A --muscle-instance command line argument is'
- ' required to identify this instance. Please'
- ' add one.'))
+ if 'MUSCLE_INSTANCE' in os.environ:
+ prefix_ref = Reference(os.environ['MUSCLE_INSTANCE'])
+ name, index = split_reference(prefix_ref)
+ else:
+ raise RuntimeError((
+ 'A --muscle-instance command line argument or'
+ ' MUSCLE_INSTANCE environment variable is required to'
+ ' identify this instance. Please add one.'))
return name, index
def __list_declared_ports(self) -> List[Port]:
diff --git a/libmuscle/python/libmuscle/manager/logger.py b/libmuscle/python/libmuscle/manager/logger.py
index b91f321f..f948864f 100644
--- a/libmuscle/python/libmuscle/manager/logger.py
+++ b/libmuscle/python/libmuscle/manager/logger.py
@@ -1,21 +1,53 @@
import logging
+from pathlib import Path
+from typing import Optional
from libmuscle.logging import LogLevel, Timestamp
from libmuscle.util import extract_log_file_location
+class Formatter(logging.Formatter):
+ """A custom formatter that can format remote messages."""
+ def usesTime(self) -> bool:
+ """Tells the formatter to make asctime available."""
+ return True
+
+ def formatMessage(self, record: logging.LogRecord) -> str:
+ """Formats a message for a record.
+
+ If the record contains a time_stamp attribute, assumes that it
+ is a remote record and formats accordingly, otherwise formats
+ as a local record.
+
+ Args:
+ record: The LogRecord to format.
+
+ Returns:
+ The formatted message.
+ """
+ if 'time_stamp' in record.__dict__:
+ return (
+ '%(asctime)s %(name)s [%(time_stamp)-15s]'
+ ' %(levelname)s: %(message)s' % record.__dict__)
+ return ('%(asctime)s muscle_manager %(levelname)s: %(message)s' %
+ record.__dict__)
+
+
class Logger:
"""The MUSCLE 3 Manager Logger component.
The Logger component takes log messages and writes them to
standard out.
+
+ Args:
+ log_dir: Directory to write the log file into.
"""
- def __init__(self) -> None:
- logfile = extract_log_file_location('muscle3_manager.log')
+ def __init__(self, log_dir: Optional[Path] = None) -> None:
+ if log_dir is None:
+ log_dir = Path.cwd()
+ logfile = extract_log_file_location(log_dir, 'muscle3_manager.log')
self._local_handler = logging.FileHandler(str(logfile), mode='w')
- formatter = logging.Formatter('%(time_stamp)-15s: %(name)s'
- ' %(levelname)s: %(message)s')
- self._local_handler.setFormatter(formatter)
+ self._local_handler.setFormatter(Formatter())
# Find and remove default handler to disable automatic console output
# Testing for 'stderr' in the stringified version is not nice, but
diff --git a/libmuscle/python/libmuscle/manager/manager.py b/libmuscle/python/libmuscle/manager/manager.py
index 96116a02..2b3637e6 100644
--- a/libmuscle/python/libmuscle/manager/manager.py
+++ b/libmuscle/python/libmuscle/manager/manager.py
@@ -45,7 +45,7 @@ def generate_indices(multiplicity: List[int]) -> List[str]:
return indices
result = list() # type: List[str]
- for element in model.compute_elements:
+ for element in model.components:
if len(element.multiplicity) == 0:
result.append(str(element.name))
else:
diff --git a/libmuscle/python/libmuscle/manager/mmp_server.py b/libmuscle/python/libmuscle/manager/mmp_server.py
index 14e9775f..173f1ff8 100644
--- a/libmuscle/python/libmuscle/manager/mmp_server.py
+++ b/libmuscle/python/libmuscle/manager/mmp_server.py
@@ -252,14 +252,14 @@ def __init__(
instance_registry: InstanceRegistry,
topology_store: TopologyStore
) -> None:
- self.__instance_registry = instance_registry
- self.__servicer = MMPServicer(logger, settings, instance_registry,
- topology_store)
- self.__server = grpc.server(futures.ThreadPoolExecutor())
+ self._instance_registry = instance_registry
+ self._servicer = MMPServicer(logger, settings, instance_registry,
+ topology_store)
+ self._server = grpc.server(futures.ThreadPoolExecutor())
mmp_grpc.add_MuscleManagerServicer_to_server( # type: ignore
- self.__servicer, self.__server)
- self.__server.add_insecure_port('[::]:9000')
- self.__server.start()
+ self._servicer, self._server)
+ self._server.add_insecure_port('[::]:9000')
+ self._server.start()
def get_location(self) -> str:
"""Returns this server's network location.
@@ -275,9 +275,9 @@ def wait(self) -> None:
The server will shut down after every instance has been
registered and deregistered again.
"""
- self.__instance_registry.wait()
+ self._instance_registry.wait()
time.sleep(1)
- self.__server.stop(5)
+ self._server.stop(5)
def stop(self) -> None:
"""Stops the server.
@@ -285,4 +285,4 @@ def stop(self) -> None:
This makes the server stop serving requests, and shuts down its
background threads.
"""
- self.__server.stop(0)
+ self._server.stop(0)
diff --git a/libmuscle/python/libmuscle/manager/test/conftest.py b/libmuscle/python/libmuscle/manager/test/conftest.py
index 8d05a800..8d81f560 100644
--- a/libmuscle/python/libmuscle/manager/test/conftest.py
+++ b/libmuscle/python/libmuscle/manager/test/conftest.py
@@ -1,5 +1,7 @@
+from pathlib import Path
+
import pytest
-from ymmsl import (ComputeElement, Conduit, Configuration, Model, Reference,
+from ymmsl import (Component, Conduit, Configuration, Model, Reference,
Settings)
from libmuscle.manager.instance_registry import InstanceRegistry
@@ -9,8 +11,8 @@
@pytest.fixture
-def logger():
- test_logger = Logger()
+def logger(tmpdir):
+ test_logger = Logger(Path(str(tmpdir)))
yield test_logger
test_logger.close()
@@ -34,8 +36,8 @@ def topology_store() -> TopologyStore:
Model(
'test_model',
[
- ComputeElement('macro', 'macro_implementation'),
- ComputeElement(
+ Component('macro', 'macro_implementation'),
+ Component(
'micro', 'micro_implementation', [10, 10])],
[
Conduit('macro.out', 'micro.in'),
@@ -75,9 +77,9 @@ def topology_store2() -> TopologyStore:
Model(
'test_model',
[
- ComputeElement('macro', 'macro_implementation'),
- ComputeElement('meso', 'meso_implementation', [5]),
- ComputeElement('micro', 'micro_implementation', [5, 10])
+ Component('macro', 'macro_implementation'),
+ Component('meso', 'meso_implementation', [5]),
+ Component('micro', 'micro_implementation', [5, 10])
],
[
Conduit('macro.out', 'meso.in'),
diff --git a/libmuscle/python/libmuscle/manager/test/test_logger.py b/libmuscle/python/libmuscle/manager/test/test_logger.py
index 01f1166b..aed3f46a 100644
--- a/libmuscle/python/libmuscle/manager/test/test_logger.py
+++ b/libmuscle/python/libmuscle/manager/test/test_logger.py
@@ -1,4 +1,5 @@
import logging
+from pathlib import Path
from libmuscle.logging import LogLevel, Timestamp
from libmuscle.manager.logger import Logger
@@ -9,8 +10,8 @@ def test_log_level():
assert LogLevel.CRITICAL.value > LogLevel.DEBUG.value
-def test_create_logger():
- logger = Logger()
+def test_create_logger(tmpdir):
+ logger = Logger(Path(str(tmpdir)))
logger.close()
diff --git a/libmuscle/python/libmuscle/manager/topology_store.py b/libmuscle/python/libmuscle/manager/topology_store.py
index d3750f01..c8e3f864 100644
--- a/libmuscle/python/libmuscle/manager/topology_store.py
+++ b/libmuscle/python/libmuscle/manager/topology_store.py
@@ -1,6 +1,6 @@
from typing import Dict, List
-from ymmsl import Conduit, Configuration, Model, Reference
+from ymmsl import Conduit, PartialConfiguration, Model, Reference
class TopologyStore:
@@ -12,7 +12,7 @@ class TopologyStore:
Attributes:
conduits (List[Conduit]): A list of conduits.
"""
- def __init__(self, config: Configuration) -> None:
+ def __init__(self, config: PartialConfiguration) -> None:
"""Creates a TopologyStore.
Creates a TopologyStore containing conduits read from the given
@@ -28,7 +28,7 @@ def __init__(self, config: Configuration) -> None:
self.conduits = config.model.conduits
self.kernel_dimensions = {
k.name: k.multiplicity
- for k in config.model.compute_elements}
+ for k in config.model.components}
def has_kernel(self, kernel: Reference) -> bool:
"""Returns True iff the given kernel is in the model.
@@ -49,9 +49,9 @@ def get_conduits(self, kernel_name: Reference) -> List[Conduit]:
"""
ret = list()
for conduit in self.conduits:
- if conduit.sending_compute_element() == kernel_name:
+ if conduit.sending_component() == kernel_name:
ret.append(conduit)
- if conduit.receiving_compute_element() == kernel_name:
+ if conduit.receiving_component() == kernel_name:
ret.append(conduit)
return ret
@@ -70,10 +70,10 @@ def get_peer_dimensions(self, kernel_name: Reference
"""
ret = dict()
for conduit in self.conduits:
- if conduit.sending_compute_element() == kernel_name:
- recv = conduit.receiving_compute_element()
+ if conduit.sending_component() == kernel_name:
+ recv = conduit.receiving_component()
ret[recv] = self.kernel_dimensions[recv]
- if conduit.receiving_compute_element() == kernel_name:
- snd = conduit.sending_compute_element()
+ if conduit.receiving_component() == kernel_name:
+ snd = conduit.sending_component()
ret[snd] = self.kernel_dimensions[snd]
return ret
diff --git a/libmuscle/python/libmuscle/mcp/message.py b/libmuscle/python/libmuscle/mcp/message.py
index bdd25723..f5a6b8a8 100644
--- a/libmuscle/python/libmuscle/mcp/message.py
+++ b/libmuscle/python/libmuscle/mcp/message.py
@@ -1,5 +1,4 @@
from enum import IntEnum
-import struct
from typing import Any, cast, Optional
import msgpack
@@ -49,20 +48,6 @@ class ClosePort:
def _encode_grid(grid: Grid) -> msgpack.ExtType:
"""Encodes a Grid object into the wire format.
"""
- item_size_map = {
- 'int32': 4,
- 'int64': 8,
- 'float32': 4,
- 'float64': 8,
- 'bool': 1}
-
- format_map = {
- 'int32': ' msgpack.ExtType:
'bool': ExtTypeId.GRID_BOOL}
array = grid.array
- if array.flags.c_contiguous:
- # indexes that differ in the last place are adjacent
- order = 'la'
- elif array.flags.f_contiguous:
+ if array.flags.f_contiguous:
# indexes that differ in the first place are adjacent
order = 'fa'
+ else:
+ # indexes that differ in the last place are adjacent
+ order = 'la'
# dtype is a bit weird, but this seems to be consistent
if isinstance(array.dtype, np.dtype):
@@ -84,14 +69,10 @@ def _encode_grid(grid: Grid) -> msgpack.ExtType:
else:
array_type = str(np.dtype(array_type))
- if array_type not in item_size_map:
+ if array_type not in ext_type_map:
raise RuntimeError('Unsupported array data type')
- item_size = item_size_map[array_type]
- fmt = format_map[array_type]
- buf = bytearray(array.size * item_size)
- for i in range(array.size):
- struct.pack_into(fmt, buf, i * item_size, array.item(i))
+ buf = array.tobytes(order='A')
# array_type is redundant, but useful metadata.
grid_dict = {
diff --git a/libmuscle/python/libmuscle/mcp/pipe_multiplexer.py b/libmuscle/python/libmuscle/mcp/pipe_multiplexer.py
index 79e6b94a..2d8e7e85 100644
--- a/libmuscle/python/libmuscle/mcp/pipe_multiplexer.py
+++ b/libmuscle/python/libmuscle/mcp/pipe_multiplexer.py
@@ -19,7 +19,7 @@ class _InstancePipe:
"""Pipes for communicating between an instance and the mux.
Objects of this class contain the endpoints for a pipe
- that is used to communicate between an instance processe and the
+ that is used to communicate between an instance process and the
multiplexer. The multiplexer (in this module) facilitates the
creation of peer-to-peer pipe connections between the processes.
diff --git a/libmuscle/python/libmuscle/mcp/pipe_server.py b/libmuscle/python/libmuscle/mcp/pipe_server.py
index 3c22eb42..d0a57da2 100644
--- a/libmuscle/python/libmuscle/mcp/pipe_server.py
+++ b/libmuscle/python/libmuscle/mcp/pipe_server.py
@@ -31,7 +31,7 @@ def __init__(self, instance_id: Reference, post_office: PostOffice
self._shutdown_conn, self._handler_shutdown_conn = mp.Pipe()
self._server_thread = threading.Thread(
target=self.__conn_request_handler,
- name='PipeServer-{}'.format(instance_id))
+ name='PipeServer-{}'.format(instance_id), daemon=True)
self._server_thread.start()
else:
@@ -72,7 +72,7 @@ def __conn_request_handler(self) -> None:
target=self.__mcp_pipe_handler,
args=(client_id, connection),
name='PipeHandler-{}-{}'.format(
- self._instance_id, client_id))
+ self._instance_id, client_id), daemon=True)
conn_thread.start()
conn_threads.append(conn_thread)
except EOFError:
diff --git a/libmuscle/python/libmuscle/mcp/tcp_server.py b/libmuscle/python/libmuscle/mcp/tcp_server.py
index 5385ce48..ea312101 100644
--- a/libmuscle/python/libmuscle/mcp/tcp_server.py
+++ b/libmuscle/python/libmuscle/mcp/tcp_server.py
@@ -13,6 +13,8 @@
class TcpServerImpl(ss.ThreadingMixIn, ss.TCPServer):
+ daemon_threads = True
+
def __init__(self, host_port_tuple: Tuple[str, int],
streamhandler: Type, tcp_server: 'TcpServer'
) -> None:
@@ -65,7 +67,7 @@ def __init__(self, instance_id: Reference, post_office: PostOffice
self._server = TcpServerImpl(('', 0), TcpHandler, self)
self._server_thread = threading.Thread(
- target=self._server.serve_forever)
+ target=self._server.serve_forever, daemon=True)
self._server_thread.start()
def get_location(self) -> str:
diff --git a/libmuscle/python/libmuscle/mcp/test/test_message.py b/libmuscle/python/libmuscle/mcp/test/test_message.py
index 1a968068..30dad34a 100644
--- a/libmuscle/python/libmuscle/mcp/test/test_message.py
+++ b/libmuscle/python/libmuscle/mcp/test/test_message.py
@@ -121,15 +121,49 @@ def test_grid_roundtrip() -> None:
timestamp = 10.0
next_timestamp = 11.0
+ for order in ('C', 'F'):
+ array = np.array(
+ [[[1.0, 2.0, 3.0],
+ [4.0, 5.0, 6.0]],
+ [[7.0, 8.0, 9.0],
+ [10.0, 11.0, 12.0]]], np.float64, order=order)
+
+ assert array[0, 0, 0] == 1.0
+
+ grid = Grid(array, ['x', 'y', 'z'])
+ msg = Message(sender, receiver, None, timestamp, next_timestamp,
+ Settings(), grid)
+
+ wire_data = msg.encoded()
+ msg_out = Message.from_bytes(wire_data)
+
+ assert isinstance(msg_out.data, Grid)
+ grid_out = msg_out.data
+ assert grid_out.indexes == ['x', 'y', 'z']
+ assert isinstance(grid_out.array, np.ndarray)
+ assert grid_out.array.dtype == np.float64
+ assert grid_out.array.shape == (2, 2, 3)
+ assert grid_out.array.size == 12
+ assert grid_out.array[1, 0, 1] == 8.0
+ assert grid_out.array[0, 0, 2] == 3.0
+
+
+def test_non_contiguous_grid_roundtrip() -> None:
+ sender = Reference('sender.port')
+ receiver = Reference('receiver.port')
+ timestamp = 10.0
+ next_timestamp = 11.0
+
array = np.array(
- [[[1.0, 2.0, 3.0],
- [4.0, 5.0, 6.0]],
- [[7.0, 8.0, 9.0],
- [10.0, 11.0, 12.0]]], np.float64)
+ [[[1.0 + 0.125j, 2.0 + 0.25j, 3.0 + 0.375j],
+ [4.0 + 0.4375j, 5.0 + 0.5j, 6.0 + 0.625j]],
+ [[7.0 + 0.75j, 8.0 + 0.875j, 9.0 + 0.9375j],
+ [10.0 + 0.10j, 11.0 + 0.11j, 12.0 + 0.12j]]], np.complex64)
- assert(array[0, 0, 0] == 1.0)
+ assert array.real[0, 0, 0] == 1.0
+ assert array.imag[0, 0, 0] == 0.125
- grid = Grid(array, ['x', 'y', 'z'])
+ grid = Grid(array.real, ['a', 'b', 'c'])
msg = Message(sender, receiver, None, timestamp, next_timestamp,
Settings(), grid)
@@ -138,9 +172,8 @@ def test_grid_roundtrip() -> None:
assert isinstance(msg_out.data, Grid)
grid_out = msg_out.data
- assert grid_out.indexes == ['x', 'y', 'z']
assert isinstance(grid_out.array, np.ndarray)
- assert grid_out.array.dtype == np.float64
+ assert grid_out.array.dtype == np.float32
assert grid_out.array.shape == (2, 2, 3)
assert grid_out.array.size == 12
assert grid_out.array[1, 0, 1] == 8.0
diff --git a/libmuscle/python/libmuscle/peer_manager.py b/libmuscle/python/libmuscle/peer_manager.py
index 7066f244..57c50121 100644
--- a/libmuscle/python/libmuscle/peer_manager.py
+++ b/libmuscle/python/libmuscle/peer_manager.py
@@ -37,10 +37,10 @@ def __init__(self, kernel: Reference, index: List[int],
self.__peers = dict() # type: Dict[Reference, Reference]
for conduit in conduits:
- if str(conduit.sending_compute_element()) == str(kernel):
+ if str(conduit.sending_component()) == str(kernel):
# we send on the port this conduit attaches to
self.__peers[conduit.sender] = conduit.receiver
- if str(conduit.receiving_compute_element()) == str(kernel):
+ if str(conduit.receiving_component()) == str(kernel):
# we receive on the port this conduit attaches to
self.__peers[conduit.receiver] = conduit.sender
diff --git a/libmuscle/python/libmuscle/runner.py b/libmuscle/python/libmuscle/runner.py
index 3e422262..2e725c3d 100644
--- a/libmuscle/python/libmuscle/runner.py
+++ b/libmuscle/python/libmuscle/runner.py
@@ -246,7 +246,7 @@ def run_simulation(
' definition, so the simulation can not be run.')
instances = dict()
- for ce in configuration.model.compute_elements:
+ for ce in configuration.model.components:
impl_name = str(ce.implementation)
if impl_name not in implementations:
raise ValueError(('The model specifies an implementation named'
diff --git a/libmuscle/python/libmuscle/util.py b/libmuscle/python/libmuscle/util.py
index bcf27239..5afe8c35 100644
--- a/libmuscle/python/libmuscle/util.py
+++ b/libmuscle/python/libmuscle/util.py
@@ -1,6 +1,6 @@
from pathlib import Path
import sys
-from typing import Generator, List, Optional, cast
+from typing import Generator, List, cast
from ymmsl import Conduit, Reference
@@ -90,7 +90,7 @@ def increment_index(index: List[int], dims: List[int]) -> bool:
return False
-def extract_log_file_location(filename: str) -> Optional[Path]:
+def extract_log_file_location(run_dir: Path, filename: str) -> Path:
"""Gets the log file location from the command line.
Extracts the --muscle-log-file= argument to tell the
@@ -100,10 +100,11 @@ def extract_log_file_location(filename: str) -> Optional[Path]:
directory, will be written inside of that directory,
if the path is not an existing directory, then it will be used
as the name of the log file to write to. If no command line
- argument is given, will be written in the current
+ argument is given, will be written in the specified
directory.
Args:
+ run_dir: Default directory to use.
filename: Default file name to use.
Returns:
@@ -119,7 +120,7 @@ def extract_log_file_location(filename: str) -> Optional[Path]:
given_path_str = arg[len(prefix):]
if given_path_str == '':
- return Path('.') / filename
+ return run_dir / filename
given_path = Path(given_path_str)
diff --git a/muscle_manager/muscle_manager.py b/muscle_manager/muscle_manager.py
index 6385687b..73d3cbbc 100644
--- a/muscle_manager/muscle_manager.py
+++ b/muscle_manager/muscle_manager.py
@@ -1,16 +1,29 @@
+from typing import Sequence
+
import click
import ymmsl
+from ymmsl import Implementation, PartialConfiguration, Reference, Resources
from libmuscle.manager.manager import start_server
@click.command()
-@click.argument('ymmsl_file')
-def manage_simulation(ymmsl_file: str) -> None:
- with open(ymmsl_file) as f:
- configuration = ymmsl.load(f)
+@click.argument(
+ 'ymmsl_files', nargs=-1, required=True, type=click.Path(
+ exists=True, file_okay=True, dir_okay=False, readable=True,
+ allow_dash=True, resolve_path=True))
+def manage_simulation(ymmsl_files: Sequence[str]) -> None:
+ configuration = PartialConfiguration()
+ for path in ymmsl_files:
+ with open(path, 'r') as f:
+ configuration.update(ymmsl.load(f))
+ # temporary dummy data to satisfy type
+ configuration.update(PartialConfiguration(
+ None, None,
+ {Reference('dummy'): Implementation(Reference('dummy'), '')},
+ {Reference('dummy'): Resources(Reference('dummy'), 0)}))
- server = start_server(configuration)
+ server = start_server(configuration.as_configuration())
print(server.get_location())
server.wait()
diff --git a/setup.cfg b/setup.cfg
index 4742cd5b..5c62d746 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -4,14 +4,13 @@ test = pytest
[tool:pytest]
testpaths = muscle_manager libmuscle/python integration_test
-addopts = --mypy --flake8 --cov --cov-report xml --cov-report term-missing -x
+addopts = --mypy --flake8 --cov --cov-report xml --cov-report term-missing
flake8-ignore =
setup.py E501
muscle_manager/protocol/*.py ALL
libmuscle/manager_protocol/*.py ALL
[mypy]
-paths = muscle_manager libmuscle/python
mypy_path = libmuscle/python:muscle_manager_protocol
warn_unused_configs = True
disallow_subclassing_any = True
diff --git a/setup.py b/setup.py
index 4c771b49..824e96ad 100644
--- a/setup.py
+++ b/setup.py
@@ -45,14 +45,14 @@
},
python_requires='>=3.5, <4',
install_requires=[
- 'click',
+ 'click>=6',
'grpcio>=1.24.3, <2',
'msgpack',
'netifaces',
- 'numpy>=1.12',
+ 'numpy>=1.12,<1.20',
'protobuf>=3.10.0, <4',
'typing_extensions',
- 'ymmsl==0.10.1' # Also in CI, update there as well
+ 'ymmsl>=0.11.0' # Also in CI, update there as well
],
setup_requires=[
'pytest-runner',
@@ -67,10 +67,11 @@
tests_require=[
'coverage<5',
'mypy',
- 'pytest>=3.5',
+ 'pytest>=3.5,<6.2',
'pytest-cov',
'pytest-flake8',
- 'pytest-mypy'
+ 'pytest-mypy',
+ 'importlib-metadata==2.1.0'
],
extras_require={
'dev': [