diff --git a/.bettercodehub.yml b/.bettercodehub.yml new file mode 100644 index 00000000..cfb0713d --- /dev/null +++ b/.bettercodehub.yml @@ -0,0 +1,25 @@ +component_depth: 1 +languages: + - name: python + production: + exclude: + - /muscle_manager_protocol/.*\.py + - /scripts/.*\.py + - /setup.py + - /docs/.* + - .*/test/.*\.py + test: + include: + - .*/test/.*\.py + - /integration_test/.*\.py + - name: cpp + production: + exclude: + - /libmuscle/cpp/src/muscle_manager_protocol/.* + - /libmuscle/cpp/src/libmuscle/bindings/.* + - /libmuscle/cpp/src/ymmsl/bindings/.* + - .*/tests/.*\.?pp + - /docs/.* + test: + include: + - .*/tests/.*\.?pp diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 1f2527ba..45d2b554 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -18,7 +18,7 @@ jobs: sudo apt-get update sudo apt-get install -y build-essential cmake gfortran libopenmpi-dev pkg-config wget valgrind sudo apt-get install -y libssl-dev zlib1g-dev - pip install ymmsl==0.10.1 + pip install ymmsl==0.11.0 - name: Build and run the test suite env: diff --git a/.github/workflows/ci_python3.5.1.yaml b/.github/workflows/ci_python3.5.1.yaml index 414110fb..32c4b1d7 100644 --- a/.github/workflows/ci_python3.5.1.yaml +++ b/.github/workflows/ci_python3.5.1.yaml @@ -13,7 +13,7 @@ jobs: uses: actions/cache@v1 with: path: ${{ github.workspace }}/.eggs - key: python-compatibility-3.5.1-eggs + key: python-compatibility-3.5.1-eggs2 - name: Run Python tests on 3.5.1 - run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.5.1 /bin/bash -c 'cd /home/muscle3 && pip install -U pip setuptools wheel "numpy<1.19" ymmsl==0.10.1 && make test_python_only' + run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.5.1 /bin/bash -c 'cd /home/muscle3 && pip install -U pip setuptools wheel "numpy<1.19" ymmsl==0.11.0 && make test_python_only' diff --git a/.github/workflows/ci_python3.5.yaml b/.github/workflows/ci_python3.5.yaml index 3d79cf5f..5f310a8e 100644 --- a/.github/workflows/ci_python3.5.yaml +++ b/.github/workflows/ci_python3.5.yaml @@ -16,4 +16,4 @@ jobs: key: python-compatibility-3.5-eggs - name: Run Python tests on 3.5 latest - run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.5 /bin/bash -c 'cd /home/muscle3 && pip install numpy ymmsl==0.10.1 && make test_python_only' + run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.5 /bin/bash -c 'cd /home/muscle3 && pip install numpy ymmsl==0.11.0 && make test_python_only' diff --git a/.github/workflows/ci_python3.6.1.yaml b/.github/workflows/ci_python3.6.1.yaml index 75fe0a25..da7822a4 100644 --- a/.github/workflows/ci_python3.6.1.yaml +++ b/.github/workflows/ci_python3.6.1.yaml @@ -16,4 +16,4 @@ jobs: key: python-compatibility-3.6.1-eggs - name: Run Python tests on 3.6.1 latest - run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.6.1 /bin/bash -c 'cd /home/muscle3 && pip install -U pip setuptools wheel ymmsl==0.10.1 && make test_python_only' + run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.6.1 /bin/bash -c 'cd /home/muscle3 && pip install -U pip setuptools wheel ymmsl==0.11.0 && make test_python_only' diff --git a/.github/workflows/ci_python3.6.yaml b/.github/workflows/ci_python3.6.yaml index eab001e9..09478248 100644 --- a/.github/workflows/ci_python3.6.yaml +++ b/.github/workflows/ci_python3.6.yaml @@ -16,4 +16,4 @@ jobs: key: python-compatibility-3.6-eggs - name: Run Python tests on 3.6 latest - run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.6 /bin/bash -c 'cd /home/muscle3 && pip install ymmsl==0.10.1 && make test_python_only' + run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.6 /bin/bash -c 'cd /home/muscle3 && pip install ymmsl==0.11.0 && make test_python_only' diff --git a/.github/workflows/ci_python3.7.yaml b/.github/workflows/ci_python3.7.yaml index 88fbd066..b110f88f 100644 --- a/.github/workflows/ci_python3.7.yaml +++ b/.github/workflows/ci_python3.7.yaml @@ -16,4 +16,4 @@ jobs: key: python-compatibility-3.7-eggs - name: Run Python tests on 3.7 latest - run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.7 /bin/bash -c 'cd /home/muscle3 && pip install ymmsl==0.10.1 && make test_python_only' + run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.7 /bin/bash -c 'cd /home/muscle3 && pip install ymmsl==0.11.0 && make test_python_only' diff --git a/.github/workflows/ci_python3.8.yaml b/.github/workflows/ci_python3.8.yaml index 549b2d38..93756697 100644 --- a/.github/workflows/ci_python3.8.yaml +++ b/.github/workflows/ci_python3.8.yaml @@ -16,4 +16,4 @@ jobs: key: python-compatibility-3.8-eggs - name: Run Python tests on 3.8 latest - run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.8 /bin/bash -c 'cd /home/muscle3 && pip install ymmsl==0.10.1 && make test_python_only' + run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.8 /bin/bash -c 'cd /home/muscle3 && pip install ymmsl==0.11.0 && make test_python_only' diff --git a/.github/workflows/ci_ubuntu16.04.yaml b/.github/workflows/ci_ubuntu16.04.yaml index 7d783789..716bb931 100644 --- a/.github/workflows/ci_ubuntu16.04.yaml +++ b/.github/workflows/ci_ubuntu16.04.yaml @@ -15,4 +15,4 @@ jobs: - uses: actions/checkout@v2 - name: Run tests on Ubuntu 16.04 - run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:16.04 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran valgrind libopenmpi-dev pkg-config python3 python3-pip python3-venv curl && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && pip3 install ymmsl==0.10.1 && make test_examples' + run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:16.04 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran valgrind libopenmpi-dev pkg-config python3 python3-pip python3-venv curl && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && pip3 install ymmsl==0.11.0 && make test_examples' diff --git a/.github/workflows/ci_ubuntu18.04.yaml b/.github/workflows/ci_ubuntu18.04.yaml index 3d9b313f..0a226a83 100644 --- a/.github/workflows/ci_ubuntu18.04.yaml +++ b/.github/workflows/ci_ubuntu18.04.yaml @@ -15,4 +15,4 @@ jobs: - uses: actions/checkout@v2 - name: Run tests on Ubuntu 18.04 - run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:18.04 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran valgrind libopenmpi-dev pkg-config python3 python3-pip python3-venv curl && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && pip3 install ymmsl==0.10.1 && make test_examples' + run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:18.04 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran valgrind libopenmpi-dev pkg-config python3 python3-pip python3-venv curl && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && pip3 install ymmsl==0.11.0 && make test_examples' diff --git a/.github/workflows/ci_ubuntu19.10.yaml b/.github/workflows/ci_ubuntu19.10.yaml deleted file mode 100644 index 560d5284..00000000 --- a/.github/workflows/ci_ubuntu19.10.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# Run Continuous Integration for the latest Ubuntu release -# This mainly checks for issues/regressions in the native build -name: native_compatibility_ubuntu19.10 -on: - schedule: - - cron: '0 3 * * 0' - push: - branches: - - 'release-*' -jobs: - build: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 - - - name: Run tests on Ubuntu 19.10 - run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:19.10 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran valgrind libopenmpi-dev pkg-config python3 python3-pip python3-venv curl && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && pip3 install ymmsl==0.10.1 && make test_examples' diff --git a/.github/workflows/ci_ubuntu20.04.yaml b/.github/workflows/ci_ubuntu20.04.yaml index ea711557..0b4cb0da 100644 --- a/.github/workflows/ci_ubuntu20.04.yaml +++ b/.github/workflows/ci_ubuntu20.04.yaml @@ -15,4 +15,4 @@ jobs: - uses: actions/checkout@v2 - name: Run tests on Ubuntu 20.04 - run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:20.04 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran valgrind libopenmpi-dev pkg-config python3 python3-pip python3-venv curl && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && pip3 install ymmsl==0.10.1 && make test_examples' + run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:20.04 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran valgrind libopenmpi-dev pkg-config python3 python3-pip python3-venv curl && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && pip3 install ymmsl==0.11.0 && make test_examples' diff --git a/.github/workflows/ci_ubuntu20.10.yaml b/.github/workflows/ci_ubuntu20.10.yaml index 285eb940..5c2db62c 100644 --- a/.github/workflows/ci_ubuntu20.10.yaml +++ b/.github/workflows/ci_ubuntu20.10.yaml @@ -15,4 +15,4 @@ jobs: - uses: actions/checkout@v2 - name: Run tests on Ubuntu 20.10 - run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:20.10 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran valgrind libopenmpi-dev pkg-config python3 python3-pip python3-venv curl && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && pip3 install ymmsl==0.10.1 && make test_examples' + run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:20.10 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran valgrind libopenmpi-dev pkg-config python3 python3-pip python3-venv curl && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && pip3 install ymmsl==0.11.0 && make test_examples' diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 2994d239..ed1b6927 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -5,6 +5,28 @@ Change Log All notable changes to this project will be documented in this file. This project adheres to `Semantic Versioning `_. +0.4.0 +***** + +Incompatible changes +-------------------- + +* `compute_elements` are now called `components` in .ymmsl files + +Improved +-------- + +* Use latest OpenSSL library when installing it automatically + +Fixed +----- + +* Handling of non-contiguous and F-order numpy arrays +* C++ memory usage for large dicts/lists now more reasonable +* Improved shutdown when Python submodel crashes +* Logging warning message + + 0.3.2 ***** diff --git a/CITATION.cff b/CITATION.cff index d576d1a0..9b2c5304 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -1,18 +1,57 @@ # YAML 1.2 --- +cff-version: "1.1.0" + +title: "MUSCLE 3: The Multiscale Coupling Library and Environment" +doi: "10.5281/zenodo.3258864" +repository-code: "https://github.com/multiscale/muscle3" + +message: "If you use MUSCLE 3, please cite it using these metadata." authors: - family-names: Veen given-names: Lourens orcid: "https://orcid.org/0000-0002-6311-1168" -cff-version: "1.1.0" -doi: "10.5281/zenodo.3258864" + keywords: - multiscale - coupling - MUSCLE + license: "Apache-2.0" -message: "If you use MUSCLE 3, please cite it using these metadata." -repository-code: "https://github.com/multiscale/muscle3" -title: "MUSCLE 3: The Multiscale Coupling Library and Environment" + +references: + - type: conference-paper + doi: "10.1007/978-3-030-50433-5_33" + authors: + - family-names: Veen + given-names: Lourens E. + - family-names: Hoekstra + given-names: Alfons G. + title: Easing Multiscale Model Design and Coupling with MUSCLE 3 + year: 2020 + + collection-title: Lecture Notes in Computer Science + volume: 12142 + volume-title: Computational Science -- ICCS 2020 + editors: + - family-names: Krzhizhanovskaya + given-names: Valeria V. + - family-names: Závodszky + given-names: Gábor + - family-names: Lees + given-names: Michael H. + - family-names: Dongarra + given-names: Jack J. + - family-names: Sloot + given-names: Peter M. A. + - family-names: Brissos + given-names: Sèrgio + - family-names: Teixeira + given-names: João + start: 425 + end: 438 + publisher: + name: Springer International Publishing + city: Cham ... diff --git a/VERSION b/VERSION index d15723fb..1d0ba9ea 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.3.2 +0.4.0 diff --git a/docs/source/cplusplus.rst b/docs/source/cplusplus.rst index 6b49f09f..aa83b4a4 100644 --- a/docs/source/cplusplus.rst +++ b/docs/source/cplusplus.rst @@ -334,9 +334,18 @@ objects containing different kinds of data: "existing_object", d4 ); + // grid + // +-----------------+ + // | 1.0 | 2.0 | 3.0 | + // +-----------------+ + // | 4.0 | 5.0 | 6.0 | + // +-----------------+ + std::vector array = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0}; + auto d11 = Data::grid(array.data(), {2, 3}, {"row", "col"}); + // byte array std::vector bytes; - auto d11 = Data::byte_array(bytes.data(), bytes.size()); + auto d12 = Data::byte_array(bytes.data(), bytes.size()); // simple types are created automatically when making a Message instance.send("output_port", Message(t_cur, 1.0)); @@ -344,15 +353,18 @@ objects containing different kinds of data: std::string text("A text message"); instance.send("output_port", Message(t_cur, text)); - // but you have to name dicts, lists, and byte arrays + // but you have to name dicts, lists, grids and byte arrays instance.send("output_port", Message(t_cur, Data::list("a", 10))); As you can see, sending complex data types with MUSCLE is almost as easy in C++ -as it is in Python. One thing that's still missing is sending and receiving -multidimensional arrays of numbers. We hope to add that in a future version; for -now you'll need to put them into a list of numbers manually. - +as it is in Python. There is however a bit of a cost to this ease-of-use in the +form of extra memory usage. If you're sending large amounts of data, then it is +best to use as many grids as you can, as those are much more efficient than +dictionaries and lists. In particular, a set of objects (agents for example) is +much more efficiently sent as a dictionary-of-grids (with one 1D-grid for each +attribute) than as a list of dictionaries. (The latter will work up to a million +objects or so, but it will be slower.) .. code-block:: cpp diff --git a/docs/source/examples/rd_settings.ymmsl b/docs/source/examples/rd_settings.ymmsl new file mode 100644 index 00000000..3aefddde --- /dev/null +++ b/docs/source/examples/rd_settings.ymmsl @@ -0,0 +1,14 @@ +ymmsl_version: v0.1 + +model: + name: reaction_diffusion + +settings: + micro.t_max: 2.469136e-06 + micro.dt: 2.469136e-08 + macro.t_max: 0.0001234568 + macro.dt: 2.469136e-06 + x_max: 1.0 + dx: 0.01 + k: -40500.0 + d: 0.0405 diff --git a/docs/source/examples/reaction_diffusion.ymmsl b/docs/source/examples/reaction_diffusion.ymmsl index 5cf3468b..c75ea225 100644 --- a/docs/source/examples/reaction_diffusion.ymmsl +++ b/docs/source/examples/reaction_diffusion.ymmsl @@ -2,19 +2,9 @@ ymmsl_version: v0.1 model: name: reaction_diffusion - compute_elements: + components: macro: diffusion micro: reaction conduits: macro.state_out: micro.initial_state micro.final_state: macro.state_in - -settings: - micro.t_max: 2.469136e-06 - micro.dt: 2.469136e-08 - macro.t_max: 0.0001234568 - macro.dt: 2.469136e-06 - x_max: 1.0 - dx: 0.01 - k: -40500.0 - d: 0.0405 diff --git a/docs/source/examples/reaction_diffusion_cpp.sh b/docs/source/examples/reaction_diffusion_cpp.sh index 4e421527..f984a421 100755 --- a/docs/source/examples/reaction_diffusion_cpp.sh +++ b/docs/source/examples/reaction_diffusion_cpp.sh @@ -9,7 +9,7 @@ fi echo 'Running reaction-diffusion in C++' . python/build/venv/bin/activate -muscle_manager reaction_diffusion.ymmsl & +muscle_manager reaction_diffusion.ymmsl rd_settings.ymmsl & manager_pid=$! diff --git a/docs/source/examples/reaction_diffusion_fortran.sh b/docs/source/examples/reaction_diffusion_fortran.sh index bdce14eb..6792da11 100755 --- a/docs/source/examples/reaction_diffusion_fortran.sh +++ b/docs/source/examples/reaction_diffusion_fortran.sh @@ -9,7 +9,7 @@ fi echo 'Running reaction-diffusion in Fortran' . python/build/venv/bin/activate -muscle_manager reaction_diffusion.ymmsl & +muscle_manager reaction_diffusion.ymmsl rd_settings.ymmsl & manager_pid=$! diff --git a/docs/source/examples/reaction_diffusion_mc.ymmsl b/docs/source/examples/reaction_diffusion_mc.ymmsl index 4e50db94..ccfa2f6b 100644 --- a/docs/source/examples/reaction_diffusion_mc.ymmsl +++ b/docs/source/examples/reaction_diffusion_mc.ymmsl @@ -2,7 +2,7 @@ ymmsl_version: v0.1 model: name: reaction_diffusion_mc - compute_elements: + components: mc: mc_driver rr: load_balancer macro: diff --git a/docs/source/examples/reaction_diffusion_mpi_cpp.sh b/docs/source/examples/reaction_diffusion_mpi_cpp.sh index dcc18da1..338ae0b8 100755 --- a/docs/source/examples/reaction_diffusion_mpi_cpp.sh +++ b/docs/source/examples/reaction_diffusion_mpi_cpp.sh @@ -9,7 +9,7 @@ fi echo 'Running reaction-diffusion with MPI in C++' . python/build/venv/bin/activate -muscle_manager reaction_diffusion.ymmsl & +muscle_manager reaction_diffusion.ymmsl rd_settings.ymmsl & manager_pid=$! diff --git a/docs/source/examples/reaction_diffusion_mpi_fortran.sh b/docs/source/examples/reaction_diffusion_mpi_fortran.sh index acdd6566..394b74d4 100755 --- a/docs/source/examples/reaction_diffusion_mpi_fortran.sh +++ b/docs/source/examples/reaction_diffusion_mpi_fortran.sh @@ -9,7 +9,7 @@ fi echo 'Running reaction-diffusion MPI in Fortran' . python/build/venv/bin/activate -muscle_manager reaction_diffusion.ymmsl & +muscle_manager reaction_diffusion.ymmsl rd_settings.ymmsl & manager_pid=$! diff --git a/docs/source/examples/reaction_diffusion_python.sh b/docs/source/examples/reaction_diffusion_python.sh index 843dbf11..7f9ff37d 100755 --- a/docs/source/examples/reaction_diffusion_python.sh +++ b/docs/source/examples/reaction_diffusion_python.sh @@ -1,7 +1,7 @@ #!/bin/bash . python/build/venv/bin/activate -muscle_manager reaction_diffusion.ymmsl & +muscle_manager reaction_diffusion.ymmsl rd_settings.ymmsl & manager_pid=$! diff --git a/docs/source/examples/reaction_diffusion_python_cpp.sh b/docs/source/examples/reaction_diffusion_python_cpp.sh index f04c0276..bbff9fab 100755 --- a/docs/source/examples/reaction_diffusion_python_cpp.sh +++ b/docs/source/examples/reaction_diffusion_python_cpp.sh @@ -9,7 +9,7 @@ fi echo 'Running reaction-diffusion in Python and C++' . python/build/venv/bin/activate -muscle_manager reaction_diffusion.ymmsl & +muscle_manager reaction_diffusion.ymmsl rd_settings.ymmsl & manager_pid=$! diff --git a/docs/source/examples/reaction_diffusion_python_fortran.sh b/docs/source/examples/reaction_diffusion_python_fortran.sh index 5aef427a..60b41469 100755 --- a/docs/source/examples/reaction_diffusion_python_fortran.sh +++ b/docs/source/examples/reaction_diffusion_python_fortran.sh @@ -9,7 +9,7 @@ fi echo 'Running reaction-diffusion in Python and Fortran' . python/build/venv/bin/activate -muscle_manager reaction_diffusion.ymmsl & +muscle_manager reaction_diffusion.ymmsl rd_settings.ymmsl & manager_pid=$! diff --git a/docs/source/fortran.rst b/docs/source/fortran.rst index cd817464..9ffd4575 100644 --- a/docs/source/fortran.rst +++ b/docs/source/fortran.rst @@ -346,7 +346,8 @@ your model!): .. code-block:: fortran - type(LIBMUSCLE_Data) :: d1, d2, d3, d4, d5, d6, d7, d8, d9, d10, d11 + type(LIBMUSCLE_Data) :: d1, d2, d3, d4, d5, d6, d7, d8, d9, d10, d11, d12 + integer (LIBMUSCLE_int8), dimension(2, 3) :: ar character(len=1), dimension(1024) :: bytes ! create a Data containing a nil value (None in Python) @@ -377,13 +378,24 @@ your model!): call LIBMUSCLE_Data_set_item(d10, 'note', 'Keys must be strings') call LIBMUSCLE_Data_set_item(d10, 'nest all you want', d9) + ! grid + ar = reshape(spread((/1_LIBMUSCLE_int8/), 1, 6), (/2, 3/)) + d12 = LIBMUSCLE_Data_create_grid(ar) + ! byte array - d11 = LIBMUSCLE_Data_create_byte_array(bytes) + d12 = LIBMUSCLE_Data_create_byte_array(bytes) As you can see, sending complex data types with MUSCLE is a bit more difficult in Fortran than in Python, but it is not too burdensome. +If you want to send large amounts of data, then use grids (arrays) as much as +possible. Lists and dicts are more flexible, but they are also slower and take +up much more memory. In particular, a set of objects (agents for example) is +better sent as a dict of arrays (with one 1D array for each attribute) than as a +list of dicts. (The latter will probably work up to 1 million objects or so, but +it will still be slower.) + Handling errors ``````````````` diff --git a/docs/source/index.rst b/docs/source/index.rst index d584d604..45291e2d 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -9,6 +9,21 @@ Welcome to MUSCLE 3's documentation! MUSCLE 3 is the third incarnation of the Multiscale Coupling Library and Environment. +With MUSCLE 3, you can connect multiple simulation models together into a +multiscale simulation. Simulation models can be as simple as a single Python +file, or as complex as a combination of multiple separate simulation codes +written in C++ or Fortran, and running on an HPC machine. + +If you use MUSCLE 3 for scientific work, please `cite +the version of the MUSCLE 3 software you used +`_ and the following paper: + +Veen L.E., Hoekstra A.G. (2020) Easing Multiscale Model Design and Coupling +with MUSCLE 3. In: Krzhizhanovskaya V. et al. (eds) Computational Science – +ICCS 2020. ICCS 2020. Lecture Notes in Computer Science, vol 12142. Springer, +Cham. ``_ + + .. toctree:: :maxdepth: 4 :caption: Contents: diff --git a/docs/source/installing.rst.in b/docs/source/installing.rst.in index d57974c1..0560b3c9 100644 --- a/docs/source/installing.rst.in +++ b/docs/source/installing.rst.in @@ -109,7 +109,7 @@ The dependencies are: If your model uses any of these dependencies directly, then it's best to install that dependency on your system, either via the package manager or from source, and then link both your library and MUSCLE 3 to the dependency. (See below for -how t opoint the build to your installation.) This avoids having two different +how to point the build to your installation.) This avoids having two different versions around and active at the same time. Otherwise, it's easier to rely on the automatic installation. Note that the gRPC and Protobuf dependencies are exact; getting them to install and work correctly on all systems is diff --git a/docs/source/tutorial.rst b/docs/source/tutorial.rst index 681189de..1590ba5b 100644 --- a/docs/source/tutorial.rst +++ b/docs/source/tutorial.rst @@ -248,8 +248,9 @@ and is set to ``None`` here. MUSCLE 3 uses `MessagePack `_ to encode messages between models. MessagePack is a binary encoding format which can be thought of as a binary version of JSON. That means that the message can be an integer, float, -bool, string, or a list or dictionary containing such. Like with JSON, these can -be nested, so you can send a dictionary containing lists of floats for example. +bool, string, or a list or dictionary containing such, and MUSCLE 3 also +supports NumPy arrays and byte arrays. Like with JSON, these can be nested, so +you can send a dictionary containing lists of floats for example. MessagePack is self-describing, so you can inspect the received message to find out what you were sent. In most cases, all data you send on a particular @@ -268,6 +269,13 @@ name of the dimensions. In this case there's only one, and ``x`` is not very descriptive, so we could have also passed ``U`` directly, in which case MUSCLE would have sent a :class:`libmuscle.Grid` without index names automatically. +Note that grids (and NumPy arrays) are much more efficient than lists and +dictionaries. If you have a lot of data to send, then you should use those as +much as possible. For example, a set of agents is best sent as a dictionary of +1D NumPy arrays, with one array/grid for each attribute. A list of dictionaries +will be quite a bit slower and use a lot more memory, but it should work up to +a million or so objects. + Finally, if you want to use your own encoding, you can just send a ``bytes`` object, which will be transmitted as-is, with minimal overhead. diff --git a/integration_test/conftest.py b/integration_test/conftest.py index a64b0717..b2ae93f3 100644 --- a/integration_test/conftest.py +++ b/integration_test/conftest.py @@ -67,7 +67,7 @@ def mmp_server_process(yatiml_log_warning): 'ymmsl_version: v0.1\n' 'model:\n' ' name: test_model\n' - ' compute_elements:\n' + ' components:\n' ' macro: macro_implementation\n' ' micro:\n' ' implementation: micro_implementation\n' @@ -84,6 +84,9 @@ def mmp_server_process(yatiml_log_warning): ' test6:\n' ' - [1.0, 2.0]\n' ' - [3.0, 1.0]\n' + 'implementations:\n' + ' macro_implementation: macro.py\n' + ' micro_implementation: micro.py\n' ) ymmsl_doc = ymmsl.load(ymmsl_text) @@ -96,7 +99,7 @@ def mmp_server_process_simple(yatiml_log_warning): 'ymmsl_version: v0.1\n' 'model:\n' ' name: test_model\n' - ' compute_elements:\n' + ' components:\n' ' macro: macro_implementation\n' ' micro: micro_implementation\n' ' conduits:\n' @@ -123,7 +126,7 @@ def mmp_server(yatiml_log_warning): 'ymmsl_version: v0.1\n' 'model:\n' ' name: test_model\n' - ' compute_elements:\n' + ' components:\n' ' macro: macro_implementation\n' ' micro:\n' ' implementation: micro_implementation\n' diff --git a/integration_test/test_all.py b/integration_test/test_all.py index f314396a..8764f3a4 100644 --- a/integration_test/test_all.py +++ b/integration_test/test_all.py @@ -1,7 +1,7 @@ from collections import OrderedDict import numpy as np -from ymmsl import (ComputeElement, Conduit, Configuration, Model, Operator, +from ymmsl import (Component, Conduit, Configuration, Model, Operator, Settings) from libmuscle import Grid, Instance, Message @@ -63,8 +63,8 @@ def test_all(log_file_in_tmpdir): """A positive all-up test of everything. """ elements = [ - ComputeElement('macro', 'macro_impl'), - ComputeElement('micro', 'micro_impl', [10])] + Component('macro', 'macro_impl'), + Component('micro', 'micro_impl', [10])] conduits = [ Conduit('macro.out', 'micro.in'), diff --git a/integration_test/test_cpp_mmp_client.py b/integration_test/test_cpp_mmp_client.py index 1c51de0a..54faaafe 100644 --- a/integration_test/test_cpp_mmp_client.py +++ b/integration_test/test_cpp_mmp_client.py @@ -20,7 +20,7 @@ def do_mmp_client_test(caplog): 'ymmsl_version: v0.1\n' 'model:\n' ' name: test_model\n' - ' compute_elements:\n' + ' components:\n' ' macro: macro_implementation\n' ' micro:\n' ' implementation: micro_implementation\n' @@ -83,10 +83,12 @@ def mock_remove(name: Reference): assert result.returncode == 0 # check submit_log_message - assert caplog.records[0].name == 'instances.test_logging' - assert caplog.records[0].time_stamp == '1970-01-01T00:00:02Z' - assert caplog.records[0].levelname == 'CRITICAL' - assert caplog.records[0].message == 'Integration testing' + for rec in caplog.records: + if rec.name == 'instances.test_logging': + assert rec.time_stamp == '1970-01-01T00:00:02Z' + assert rec.levelname == 'CRITICAL' + assert rec.message == 'Integration testing' + break # check register_instance assert (instance_registry.get_locations('micro[3]') == diff --git a/integration_test/test_duplication_mapper.py b/integration_test/test_duplication_mapper.py index 36dbf3ba..9717ad36 100644 --- a/integration_test/test_duplication_mapper.py +++ b/integration_test/test_duplication_mapper.py @@ -1,4 +1,4 @@ -from ymmsl import (ComputeElement, Conduit, Configuration, Operator, Model, +from ymmsl import (Component, Conduit, Configuration, Operator, Model, Settings) from libmuscle import Instance, Message @@ -36,9 +36,9 @@ def test_duplication_mapper(log_file_in_tmpdir): This is an acyclic workflow. """ elements = [ - ComputeElement('dm', 'muscle.duplication_mapper'), - ComputeElement('first', 'receiver'), - ComputeElement('second', 'receiver')] + Component('dm', 'muscle.duplication_mapper'), + Component('first', 'receiver'), + Component('second', 'receiver')] conduits = [ Conduit('dm.out', 'first.in'), diff --git a/integration_test/test_logging.py b/integration_test/test_logging.py index 1c92a694..c5ced013 100644 --- a/integration_test/test_logging.py +++ b/integration_test/test_logging.py @@ -16,7 +16,7 @@ def do_logging_test(caplog): 'ymmsl_version: v0.1\n' 'model:\n' ' name: test_model\n' - ' compute_elements:\n' + ' components:\n' ' macro: macro_implementation\n' ' micro:\n' ' implementation: micro_implementation\n' @@ -54,10 +54,12 @@ def do_logging_test(caplog): # log and check client.submit_log_message(message) - assert caplog.records[0].name == 'instances.test_logging' - assert caplog.records[0].time_stamp == '1970-01-01T00:00:02Z' - assert caplog.records[0].levelname == 'DEBUG' - assert caplog.records[0].message == 'Integration testing' + for rec in caplog.records: + if rec.name == 'instances.test_logging': + assert rec.time_stamp == '1970-01-01T00:00:02Z' + assert rec.levelname == 'DEBUG' + assert rec.message == 'Integration testing' + break server.stop() diff --git a/integration_test/test_parameter_overlays.py b/integration_test/test_parameter_overlays.py index a4211cef..cf091193 100644 --- a/integration_test/test_parameter_overlays.py +++ b/integration_test/test_parameter_overlays.py @@ -1,6 +1,6 @@ from collections import OrderedDict -from ymmsl import (ComputeElement, Conduit, Configuration, Model, Operator, +from ymmsl import (Component, Conduit, Configuration, Model, Operator, Settings) from libmuscle import Instance, Message @@ -94,11 +94,11 @@ def test_settings_overlays(log_file_in_tmpdir): """A positive all-up test of settings overlays. """ elements = [ - ComputeElement('qmc', 'qmc'), - ComputeElement('macro', 'macro', [10]), - ComputeElement('relay', 'explicit_relay'), - ComputeElement('relay2', 'explicit_relay'), - ComputeElement('micro', 'micro', [10])] + Component('qmc', 'qmc'), + Component('macro', 'macro', [10]), + Component('relay', 'explicit_relay'), + Component('relay2', 'explicit_relay'), + Component('micro', 'micro', [10])] conduits = [ Conduit('qmc.settings_out', 'macro.muscle_settings_in'), diff --git a/integration_test/test_registration.py b/integration_test/test_registration.py index 77cc81e9..3ed68d96 100644 --- a/integration_test/test_registration.py +++ b/integration_test/test_registration.py @@ -14,7 +14,7 @@ def test_registration(log_file_in_tmpdir, mmp_server): client.register_instance(instance_name, ['tcp://localhost:10000'], [port]) - servicer = mmp_server._MMPServer__servicer + servicer = mmp_server._servicer registry = servicer._MMPServicer__instance_registry assert registry.get_locations(instance_name) == ['tcp://localhost:10000'] diff --git a/libmuscle/cpp/build/grpc/Makefile b/libmuscle/cpp/build/grpc/Makefile index 5c688488..ad61c0d1 100644 --- a/libmuscle/cpp/build/grpc/Makefile +++ b/libmuscle/cpp/build/grpc/Makefile @@ -40,7 +40,7 @@ include $(TOOLDIR)/make_available.make dep_name := openssl dep_version_constraint := >= 1.0.2 -dep_version := 1.1.1f +dep_version := 1.1.1i dep_pkgconfig_name := openssl dep_install := 1 include $(TOOLDIR)/make_available.make diff --git a/libmuscle/cpp/build/libmuscle/tests/Makefile b/libmuscle/cpp/build/libmuscle/tests/Makefile index e285659a..45304a4e 100644 --- a/libmuscle/cpp/build/libmuscle/tests/Makefile +++ b/libmuscle/cpp/build/libmuscle/tests/Makefile @@ -95,3 +95,7 @@ test_dep_lib_paths := $(subst $(space),:,$(foreach DIR,$(DEP_DIRS),$(DIR)/lib)) run_test%: test% export LD_LIBRARY_PATH=$(test_dep_lib_paths) ; $(VALGRIND) ./$< +# This uses setrlimit, which is not compatible with Valgrind, so run without +.PHONY: run_test_data_memory_use +run_test_data_memory_use: test_data_memory_use + export LD_LIBRARY_PATH=$(test_dep_lib_paths) ; ./$< diff --git a/libmuscle/cpp/src/libmuscle/data.cpp b/libmuscle/cpp/src/libmuscle/data.cpp index c7b0245c..dee33ef6 100644 --- a/libmuscle/cpp/src/libmuscle/data.cpp +++ b/libmuscle/cpp/src/libmuscle/data.cpp @@ -97,7 +97,6 @@ DataConstRef DataConstRef::grid_data_( DataConstRef::DataConstRef() : mp_zones_(new std::vector>()) { - mp_zones_->push_back(std::make_shared()); mp_obj_ = zone_alloc_(); mp_obj_->type = msgpack::type::NIL; } @@ -846,17 +845,23 @@ Data Data::byte_array(char const * buf, uint32_t size) { Data & Data::operator=(Data const & rhs) { if (mp_obj_ != rhs.mp_obj_) { *mp_obj_ = *rhs.mp_obj_; - - // We can't overwrite mp_zones_ here, because mp_obj_ is allocated on - // one of them, and we don't know which. So we just append, which is - // suboptimal because it may keep objects alive that are no longer - // reachable. Consider a separate shared_ptr to the zone that mp_obj_ - // is on (in a separate member), so that we can safely overwrite - // mp_zones_. - if (mp_zones_ != rhs.mp_zones_) - mp_zones_->insert(mp_zones_->end(), - rhs.mp_zones_->cbegin(), rhs.mp_zones_->cend()); obj_cache_ = rhs.obj_cache_; + + if ( + rhs.mp_obj_->type == msgpack::type::STR || + rhs.mp_obj_->type == msgpack::type::BIN || + rhs.mp_obj_->type == msgpack::type::ARRAY || + rhs.mp_obj_->type == msgpack::type::MAP || + rhs.mp_obj_->type == msgpack::type::EXT) + { + // The above assignment will only copy the pointer for these + // types. So we need to add the source zones to our own to + // ensure that the data structure pointed to by that pointer + // continues to exist for as long as we do. + if (mp_zones_ != rhs.mp_zones_) + mp_zones_->insert(mp_zones_->end(), + rhs.mp_zones_->cbegin(), rhs.mp_zones_->cend()); + } } return *this; } diff --git a/libmuscle/cpp/src/libmuscle/data.tpp b/libmuscle/cpp/src/libmuscle/data.tpp index 3276fb1b..c2a4d5ec 100644 --- a/libmuscle/cpp/src/libmuscle/data.tpp +++ b/libmuscle/cpp/src/libmuscle/data.tpp @@ -30,6 +30,8 @@ T DataConstRef::as() const { template T * DataConstRef::zone_alloc_(uint32_t size) { + if (mp_zones_->empty()) + mp_zones_->push_back(std::make_shared(24)); auto num_bytes = sizeof(T) * size; return static_cast((*mp_zones_)[0]->allocate_align( num_bytes, MSGPACK_ZONE_ALIGNOF(T))); diff --git a/libmuscle/cpp/src/libmuscle/tests/test_data_memory_use.cpp b/libmuscle/cpp/src/libmuscle/tests/test_data_memory_use.cpp new file mode 100644 index 00000000..e4913fcd --- /dev/null +++ b/libmuscle/cpp/src/libmuscle/tests/test_data_memory_use.cpp @@ -0,0 +1,98 @@ +#include "libmuscle/data.hpp" +#include "libmuscle/mcp/data_pack.hpp" + +#include +#include +#include +#include + +#include +#include +#include + +#include + + +using libmuscle::impl::Data; + + +int main(int argc, char *argv[]) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +TEST(libmuscle_mcp_data, large_structure) { + // produce a large with items in it + // regression test for excess memory usage + + // at default malloc settings, we can measure to about 128kB accuracy + const std::size_t granularity = 128u * 1024u; + + // we want to measure to 1% accuracy + const std::size_t data_size = granularity * 100u; + + // we set max overhead at 110%, because msgpack's allocator allocates + // in powers of 2 so we need 100% for that, and a bit of overhead for + // the list itself and overhead. + const std::size_t overhead = data_size * 110 / 100; + + // data segment before allocating is about 200kB + const std::size_t base_size = 200 * 1024; + + const std::size_t max_size = base_size + data_size + overhead; + + // tell the kernel to kill us if we're using too much memory + struct rlimit old_limits, limits; + getrlimit(RLIMIT_DATA, &old_limits); + getrlimit(RLIMIT_DATA, &limits); + limits.rlim_cur = max_size; + setrlimit(RLIMIT_DATA, &limits); + + // This many items are needed to reach the given data size + // A MsgPack object is 24 bytes + const std::size_t num_items = data_size / 24; + + malloc_info(0, stderr); + + Data list = Data::nils(num_items); + malloc_info(0, stderr); + for (std::size_t i = 0; i < num_items; ++i) + list[i] = Data(42424242); + + malloc_info(0, stderr); + + // reset memory limit + setrlimit(RLIMIT_DATA, &old_limits); +} + +TEST(libmuscle_mcp_data, nested_structure) { + // produce a large list with nested structures in it + // regression test for excess memory usage + + // Originally reported with a 10M item list, which would try to use + // 600GB. Scaled down here to speed things up, and we want to do at least + // 100x better than that, so 10k items in 6MB. + + // tell the kernel to kill us if we're using too much memory + struct rlimit old_limits, limits; + getrlimit(RLIMIT_DATA, &old_limits); + getrlimit(RLIMIT_DATA, &limits); + limits.rlim_cur = 6 * 1024 * 1024; + setrlimit(RLIMIT_DATA, &limits); + + const std::size_t num_items = 10000; + malloc_info(0, stderr); + + Data list = Data::nils(num_items); + malloc_info(0, stderr); + for (std::size_t i = 0; i < num_items; ++i) { + auto pos = Data::list(1.2, 3.4, 5.6); + list[i] = Data::list(pos, 7.8, 2); + } + + malloc_info(0, stderr); + + // reset memory limit + setrlimit(RLIMIT_DATA, &old_limits); +} + diff --git a/libmuscle/python/libmuscle/communicator.py b/libmuscle/python/libmuscle/communicator.py index 49f3ec06..0391501f 100644 --- a/libmuscle/python/libmuscle/communicator.py +++ b/libmuscle/python/libmuscle/communicator.py @@ -396,16 +396,16 @@ def __ports_from_conduits(self, conduits: List[Conduit] """ ports = dict() for conduit in conduits: - if conduit.sending_compute_element() == self._kernel: + if conduit.sending_component() == self._kernel: port_id = conduit.sending_port() operator = Operator.O_F port_peer_dims = self._peer_manager.get_peer_dims( - conduit.receiving_compute_element()) - elif conduit.receiving_compute_element() == self._kernel: + conduit.receiving_component()) + elif conduit.receiving_component() == self._kernel: port_id = conduit.receiving_port() operator = Operator.F_INIT port_peer_dims = self._peer_manager.get_peer_dims( - conduit.sending_compute_element()) + conduit.sending_component()) else: continue @@ -425,14 +425,14 @@ def __settings_in_port(self, conduits: List[Conduit]) -> Port: conduits: The list of conduits. """ for conduit in conduits: - if conduit.receiving_compute_element() == self._kernel: + if conduit.receiving_component() == self._kernel: port_id = conduit.receiving_port() if str(port_id) == 'muscle_settings_in': return Port(str(port_id), Operator.F_INIT, False, self._peer_manager.is_connected(port_id), len(self._index), self._peer_manager.get_peer_dims( - conduit.sending_compute_element())) + conduit.sending_component())) return Port('muscle_settings_in', Operator.F_INIT, False, False, len(self._index), []) diff --git a/libmuscle/python/libmuscle/instance.py b/libmuscle/python/libmuscle/instance.py index 13338b2d..23923969 100644 --- a/libmuscle/python/libmuscle/instance.py +++ b/libmuscle/python/libmuscle/instance.py @@ -1,5 +1,7 @@ from copy import copy import logging +import os +from pathlib import Path import sys from typing import cast, Dict, List, Optional, Tuple @@ -20,6 +22,9 @@ _logger = logging.getLogger(__name__) +_FInitCacheType = Dict[Tuple[str, Optional[int]], Message] + + class Instance: """Represents a compute element instance in a MUSCLE3 simulation. @@ -62,8 +67,7 @@ def __init__(self, ports: Optional[Dict[Operator, List[str]]] = None self._first_run = True """Keeps track of whether this is the first reuse run.""" - FInitCacheType = Dict[Tuple[str, Optional[int]], Message] - self._f_init_cache = dict() # type: FInitCacheType + self._f_init_cache = dict() # type: _FInitCacheType self._register() self._connect() @@ -406,7 +410,8 @@ def __set_up_logging(self) -> None: """ id_str = str(self._instance_name()) - logfile = extract_log_file_location('muscle3.{}.log'.format(id_str)) + logfile = extract_log_file_location( + Path.cwd(), 'muscle3.{}.log'.format(id_str)) local_handler = logging.FileHandler(str(logfile), mode='w') formatter = logging.Formatter('%(asctime)-15s: %(name)s' ' %(levelname)s: %(message)s') @@ -507,9 +512,14 @@ def split_reference(ref: Reference) -> Tuple[Reference, List[int]]: name, index = split_reference(prefix_ref) break else: - raise RuntimeError(('A --muscle-instance command line argument is' - ' required to identify this instance. Please' - ' add one.')) + if 'MUSCLE_INSTANCE' in os.environ: + prefix_ref = Reference(os.environ['MUSCLE_INSTANCE']) + name, index = split_reference(prefix_ref) + else: + raise RuntimeError(( + 'A --muscle-instance command line argument or' + ' MUSCLE_INSTANCE environment variable is required to' + ' identify this instance. Please add one.')) return name, index def __list_declared_ports(self) -> List[Port]: diff --git a/libmuscle/python/libmuscle/manager/logger.py b/libmuscle/python/libmuscle/manager/logger.py index b91f321f..f948864f 100644 --- a/libmuscle/python/libmuscle/manager/logger.py +++ b/libmuscle/python/libmuscle/manager/logger.py @@ -1,21 +1,53 @@ import logging +from pathlib import Path +from typing import Optional from libmuscle.logging import LogLevel, Timestamp from libmuscle.util import extract_log_file_location +class Formatter(logging.Formatter): + """A custom formatter that can format remote messages.""" + def usesTime(self) -> bool: + """Tells the formatter to make asctime available.""" + return True + + def formatMessage(self, record: logging.LogRecord) -> str: + """Formats a message for a record. + + If the record contains a time_stamp attribute, assumes that it + is a remote record and formats accordingly, otherwise formats + as a local record. + + Args: + record: The LogRecord to format. + + Returns: + The formatted message. + """ + if 'time_stamp' in record.__dict__: + return ( + '%(asctime)s %(name)s [%(time_stamp)-15s]' + ' %(levelname)s: %(message)s' % record.__dict__) + return ('%(asctime)s muscle_manager %(levelname)s: %(message)s' % + record.__dict__) + + class Logger: """The MUSCLE 3 Manager Logger component. The Logger component takes log messages and writes them to standard out. + + Args: + log_dir: Directory to write the log file into. """ - def __init__(self) -> None: - logfile = extract_log_file_location('muscle3_manager.log') + def __init__(self, log_dir: Optional[Path] = None) -> None: + if log_dir is None: + log_dir = Path.cwd() + logfile = extract_log_file_location(log_dir, 'muscle3_manager.log') self._local_handler = logging.FileHandler(str(logfile), mode='w') - formatter = logging.Formatter('%(time_stamp)-15s: %(name)s' - ' %(levelname)s: %(message)s') - self._local_handler.setFormatter(formatter) + self._local_handler.setFormatter(Formatter()) # Find and remove default handler to disable automatic console output # Testing for 'stderr' in the stringified version is not nice, but diff --git a/libmuscle/python/libmuscle/manager/manager.py b/libmuscle/python/libmuscle/manager/manager.py index 96116a02..2b3637e6 100644 --- a/libmuscle/python/libmuscle/manager/manager.py +++ b/libmuscle/python/libmuscle/manager/manager.py @@ -45,7 +45,7 @@ def generate_indices(multiplicity: List[int]) -> List[str]: return indices result = list() # type: List[str] - for element in model.compute_elements: + for element in model.components: if len(element.multiplicity) == 0: result.append(str(element.name)) else: diff --git a/libmuscle/python/libmuscle/manager/mmp_server.py b/libmuscle/python/libmuscle/manager/mmp_server.py index 14e9775f..173f1ff8 100644 --- a/libmuscle/python/libmuscle/manager/mmp_server.py +++ b/libmuscle/python/libmuscle/manager/mmp_server.py @@ -252,14 +252,14 @@ def __init__( instance_registry: InstanceRegistry, topology_store: TopologyStore ) -> None: - self.__instance_registry = instance_registry - self.__servicer = MMPServicer(logger, settings, instance_registry, - topology_store) - self.__server = grpc.server(futures.ThreadPoolExecutor()) + self._instance_registry = instance_registry + self._servicer = MMPServicer(logger, settings, instance_registry, + topology_store) + self._server = grpc.server(futures.ThreadPoolExecutor()) mmp_grpc.add_MuscleManagerServicer_to_server( # type: ignore - self.__servicer, self.__server) - self.__server.add_insecure_port('[::]:9000') - self.__server.start() + self._servicer, self._server) + self._server.add_insecure_port('[::]:9000') + self._server.start() def get_location(self) -> str: """Returns this server's network location. @@ -275,9 +275,9 @@ def wait(self) -> None: The server will shut down after every instance has been registered and deregistered again. """ - self.__instance_registry.wait() + self._instance_registry.wait() time.sleep(1) - self.__server.stop(5) + self._server.stop(5) def stop(self) -> None: """Stops the server. @@ -285,4 +285,4 @@ def stop(self) -> None: This makes the server stop serving requests, and shuts down its background threads. """ - self.__server.stop(0) + self._server.stop(0) diff --git a/libmuscle/python/libmuscle/manager/test/conftest.py b/libmuscle/python/libmuscle/manager/test/conftest.py index 8d05a800..8d81f560 100644 --- a/libmuscle/python/libmuscle/manager/test/conftest.py +++ b/libmuscle/python/libmuscle/manager/test/conftest.py @@ -1,5 +1,7 @@ +from pathlib import Path + import pytest -from ymmsl import (ComputeElement, Conduit, Configuration, Model, Reference, +from ymmsl import (Component, Conduit, Configuration, Model, Reference, Settings) from libmuscle.manager.instance_registry import InstanceRegistry @@ -9,8 +11,8 @@ @pytest.fixture -def logger(): - test_logger = Logger() +def logger(tmpdir): + test_logger = Logger(Path(str(tmpdir))) yield test_logger test_logger.close() @@ -34,8 +36,8 @@ def topology_store() -> TopologyStore: Model( 'test_model', [ - ComputeElement('macro', 'macro_implementation'), - ComputeElement( + Component('macro', 'macro_implementation'), + Component( 'micro', 'micro_implementation', [10, 10])], [ Conduit('macro.out', 'micro.in'), @@ -75,9 +77,9 @@ def topology_store2() -> TopologyStore: Model( 'test_model', [ - ComputeElement('macro', 'macro_implementation'), - ComputeElement('meso', 'meso_implementation', [5]), - ComputeElement('micro', 'micro_implementation', [5, 10]) + Component('macro', 'macro_implementation'), + Component('meso', 'meso_implementation', [5]), + Component('micro', 'micro_implementation', [5, 10]) ], [ Conduit('macro.out', 'meso.in'), diff --git a/libmuscle/python/libmuscle/manager/test/test_logger.py b/libmuscle/python/libmuscle/manager/test/test_logger.py index 01f1166b..aed3f46a 100644 --- a/libmuscle/python/libmuscle/manager/test/test_logger.py +++ b/libmuscle/python/libmuscle/manager/test/test_logger.py @@ -1,4 +1,5 @@ import logging +from pathlib import Path from libmuscle.logging import LogLevel, Timestamp from libmuscle.manager.logger import Logger @@ -9,8 +10,8 @@ def test_log_level(): assert LogLevel.CRITICAL.value > LogLevel.DEBUG.value -def test_create_logger(): - logger = Logger() +def test_create_logger(tmpdir): + logger = Logger(Path(str(tmpdir))) logger.close() diff --git a/libmuscle/python/libmuscle/manager/topology_store.py b/libmuscle/python/libmuscle/manager/topology_store.py index d3750f01..c8e3f864 100644 --- a/libmuscle/python/libmuscle/manager/topology_store.py +++ b/libmuscle/python/libmuscle/manager/topology_store.py @@ -1,6 +1,6 @@ from typing import Dict, List -from ymmsl import Conduit, Configuration, Model, Reference +from ymmsl import Conduit, PartialConfiguration, Model, Reference class TopologyStore: @@ -12,7 +12,7 @@ class TopologyStore: Attributes: conduits (List[Conduit]): A list of conduits. """ - def __init__(self, config: Configuration) -> None: + def __init__(self, config: PartialConfiguration) -> None: """Creates a TopologyStore. Creates a TopologyStore containing conduits read from the given @@ -28,7 +28,7 @@ def __init__(self, config: Configuration) -> None: self.conduits = config.model.conduits self.kernel_dimensions = { k.name: k.multiplicity - for k in config.model.compute_elements} + for k in config.model.components} def has_kernel(self, kernel: Reference) -> bool: """Returns True iff the given kernel is in the model. @@ -49,9 +49,9 @@ def get_conduits(self, kernel_name: Reference) -> List[Conduit]: """ ret = list() for conduit in self.conduits: - if conduit.sending_compute_element() == kernel_name: + if conduit.sending_component() == kernel_name: ret.append(conduit) - if conduit.receiving_compute_element() == kernel_name: + if conduit.receiving_component() == kernel_name: ret.append(conduit) return ret @@ -70,10 +70,10 @@ def get_peer_dimensions(self, kernel_name: Reference """ ret = dict() for conduit in self.conduits: - if conduit.sending_compute_element() == kernel_name: - recv = conduit.receiving_compute_element() + if conduit.sending_component() == kernel_name: + recv = conduit.receiving_component() ret[recv] = self.kernel_dimensions[recv] - if conduit.receiving_compute_element() == kernel_name: - snd = conduit.sending_compute_element() + if conduit.receiving_component() == kernel_name: + snd = conduit.sending_component() ret[snd] = self.kernel_dimensions[snd] return ret diff --git a/libmuscle/python/libmuscle/mcp/message.py b/libmuscle/python/libmuscle/mcp/message.py index bdd25723..f5a6b8a8 100644 --- a/libmuscle/python/libmuscle/mcp/message.py +++ b/libmuscle/python/libmuscle/mcp/message.py @@ -1,5 +1,4 @@ from enum import IntEnum -import struct from typing import Any, cast, Optional import msgpack @@ -49,20 +48,6 @@ class ClosePort: def _encode_grid(grid: Grid) -> msgpack.ExtType: """Encodes a Grid object into the wire format. """ - item_size_map = { - 'int32': 4, - 'int64': 8, - 'float32': 4, - 'float64': 8, - 'bool': 1} - - format_map = { - 'int32': ' msgpack.ExtType: 'bool': ExtTypeId.GRID_BOOL} array = grid.array - if array.flags.c_contiguous: - # indexes that differ in the last place are adjacent - order = 'la' - elif array.flags.f_contiguous: + if array.flags.f_contiguous: # indexes that differ in the first place are adjacent order = 'fa' + else: + # indexes that differ in the last place are adjacent + order = 'la' # dtype is a bit weird, but this seems to be consistent if isinstance(array.dtype, np.dtype): @@ -84,14 +69,10 @@ def _encode_grid(grid: Grid) -> msgpack.ExtType: else: array_type = str(np.dtype(array_type)) - if array_type not in item_size_map: + if array_type not in ext_type_map: raise RuntimeError('Unsupported array data type') - item_size = item_size_map[array_type] - fmt = format_map[array_type] - buf = bytearray(array.size * item_size) - for i in range(array.size): - struct.pack_into(fmt, buf, i * item_size, array.item(i)) + buf = array.tobytes(order='A') # array_type is redundant, but useful metadata. grid_dict = { diff --git a/libmuscle/python/libmuscle/mcp/pipe_multiplexer.py b/libmuscle/python/libmuscle/mcp/pipe_multiplexer.py index 79e6b94a..2d8e7e85 100644 --- a/libmuscle/python/libmuscle/mcp/pipe_multiplexer.py +++ b/libmuscle/python/libmuscle/mcp/pipe_multiplexer.py @@ -19,7 +19,7 @@ class _InstancePipe: """Pipes for communicating between an instance and the mux. Objects of this class contain the endpoints for a pipe - that is used to communicate between an instance processe and the + that is used to communicate between an instance process and the multiplexer. The multiplexer (in this module) facilitates the creation of peer-to-peer pipe connections between the processes. diff --git a/libmuscle/python/libmuscle/mcp/pipe_server.py b/libmuscle/python/libmuscle/mcp/pipe_server.py index 3c22eb42..d0a57da2 100644 --- a/libmuscle/python/libmuscle/mcp/pipe_server.py +++ b/libmuscle/python/libmuscle/mcp/pipe_server.py @@ -31,7 +31,7 @@ def __init__(self, instance_id: Reference, post_office: PostOffice self._shutdown_conn, self._handler_shutdown_conn = mp.Pipe() self._server_thread = threading.Thread( target=self.__conn_request_handler, - name='PipeServer-{}'.format(instance_id)) + name='PipeServer-{}'.format(instance_id), daemon=True) self._server_thread.start() else: @@ -72,7 +72,7 @@ def __conn_request_handler(self) -> None: target=self.__mcp_pipe_handler, args=(client_id, connection), name='PipeHandler-{}-{}'.format( - self._instance_id, client_id)) + self._instance_id, client_id), daemon=True) conn_thread.start() conn_threads.append(conn_thread) except EOFError: diff --git a/libmuscle/python/libmuscle/mcp/tcp_server.py b/libmuscle/python/libmuscle/mcp/tcp_server.py index 5385ce48..ea312101 100644 --- a/libmuscle/python/libmuscle/mcp/tcp_server.py +++ b/libmuscle/python/libmuscle/mcp/tcp_server.py @@ -13,6 +13,8 @@ class TcpServerImpl(ss.ThreadingMixIn, ss.TCPServer): + daemon_threads = True + def __init__(self, host_port_tuple: Tuple[str, int], streamhandler: Type, tcp_server: 'TcpServer' ) -> None: @@ -65,7 +67,7 @@ def __init__(self, instance_id: Reference, post_office: PostOffice self._server = TcpServerImpl(('', 0), TcpHandler, self) self._server_thread = threading.Thread( - target=self._server.serve_forever) + target=self._server.serve_forever, daemon=True) self._server_thread.start() def get_location(self) -> str: diff --git a/libmuscle/python/libmuscle/mcp/test/test_message.py b/libmuscle/python/libmuscle/mcp/test/test_message.py index 1a968068..30dad34a 100644 --- a/libmuscle/python/libmuscle/mcp/test/test_message.py +++ b/libmuscle/python/libmuscle/mcp/test/test_message.py @@ -121,15 +121,49 @@ def test_grid_roundtrip() -> None: timestamp = 10.0 next_timestamp = 11.0 + for order in ('C', 'F'): + array = np.array( + [[[1.0, 2.0, 3.0], + [4.0, 5.0, 6.0]], + [[7.0, 8.0, 9.0], + [10.0, 11.0, 12.0]]], np.float64, order=order) + + assert array[0, 0, 0] == 1.0 + + grid = Grid(array, ['x', 'y', 'z']) + msg = Message(sender, receiver, None, timestamp, next_timestamp, + Settings(), grid) + + wire_data = msg.encoded() + msg_out = Message.from_bytes(wire_data) + + assert isinstance(msg_out.data, Grid) + grid_out = msg_out.data + assert grid_out.indexes == ['x', 'y', 'z'] + assert isinstance(grid_out.array, np.ndarray) + assert grid_out.array.dtype == np.float64 + assert grid_out.array.shape == (2, 2, 3) + assert grid_out.array.size == 12 + assert grid_out.array[1, 0, 1] == 8.0 + assert grid_out.array[0, 0, 2] == 3.0 + + +def test_non_contiguous_grid_roundtrip() -> None: + sender = Reference('sender.port') + receiver = Reference('receiver.port') + timestamp = 10.0 + next_timestamp = 11.0 + array = np.array( - [[[1.0, 2.0, 3.0], - [4.0, 5.0, 6.0]], - [[7.0, 8.0, 9.0], - [10.0, 11.0, 12.0]]], np.float64) + [[[1.0 + 0.125j, 2.0 + 0.25j, 3.0 + 0.375j], + [4.0 + 0.4375j, 5.0 + 0.5j, 6.0 + 0.625j]], + [[7.0 + 0.75j, 8.0 + 0.875j, 9.0 + 0.9375j], + [10.0 + 0.10j, 11.0 + 0.11j, 12.0 + 0.12j]]], np.complex64) - assert(array[0, 0, 0] == 1.0) + assert array.real[0, 0, 0] == 1.0 + assert array.imag[0, 0, 0] == 0.125 - grid = Grid(array, ['x', 'y', 'z']) + grid = Grid(array.real, ['a', 'b', 'c']) msg = Message(sender, receiver, None, timestamp, next_timestamp, Settings(), grid) @@ -138,9 +172,8 @@ def test_grid_roundtrip() -> None: assert isinstance(msg_out.data, Grid) grid_out = msg_out.data - assert grid_out.indexes == ['x', 'y', 'z'] assert isinstance(grid_out.array, np.ndarray) - assert grid_out.array.dtype == np.float64 + assert grid_out.array.dtype == np.float32 assert grid_out.array.shape == (2, 2, 3) assert grid_out.array.size == 12 assert grid_out.array[1, 0, 1] == 8.0 diff --git a/libmuscle/python/libmuscle/peer_manager.py b/libmuscle/python/libmuscle/peer_manager.py index 7066f244..57c50121 100644 --- a/libmuscle/python/libmuscle/peer_manager.py +++ b/libmuscle/python/libmuscle/peer_manager.py @@ -37,10 +37,10 @@ def __init__(self, kernel: Reference, index: List[int], self.__peers = dict() # type: Dict[Reference, Reference] for conduit in conduits: - if str(conduit.sending_compute_element()) == str(kernel): + if str(conduit.sending_component()) == str(kernel): # we send on the port this conduit attaches to self.__peers[conduit.sender] = conduit.receiver - if str(conduit.receiving_compute_element()) == str(kernel): + if str(conduit.receiving_component()) == str(kernel): # we receive on the port this conduit attaches to self.__peers[conduit.receiver] = conduit.sender diff --git a/libmuscle/python/libmuscle/runner.py b/libmuscle/python/libmuscle/runner.py index 3e422262..2e725c3d 100644 --- a/libmuscle/python/libmuscle/runner.py +++ b/libmuscle/python/libmuscle/runner.py @@ -246,7 +246,7 @@ def run_simulation( ' definition, so the simulation can not be run.') instances = dict() - for ce in configuration.model.compute_elements: + for ce in configuration.model.components: impl_name = str(ce.implementation) if impl_name not in implementations: raise ValueError(('The model specifies an implementation named' diff --git a/libmuscle/python/libmuscle/util.py b/libmuscle/python/libmuscle/util.py index bcf27239..5afe8c35 100644 --- a/libmuscle/python/libmuscle/util.py +++ b/libmuscle/python/libmuscle/util.py @@ -1,6 +1,6 @@ from pathlib import Path import sys -from typing import Generator, List, Optional, cast +from typing import Generator, List, cast from ymmsl import Conduit, Reference @@ -90,7 +90,7 @@ def increment_index(index: List[int], dims: List[int]) -> bool: return False -def extract_log_file_location(filename: str) -> Optional[Path]: +def extract_log_file_location(run_dir: Path, filename: str) -> Path: """Gets the log file location from the command line. Extracts the --muscle-log-file= argument to tell the @@ -100,10 +100,11 @@ def extract_log_file_location(filename: str) -> Optional[Path]: directory, will be written inside of that directory, if the path is not an existing directory, then it will be used as the name of the log file to write to. If no command line - argument is given, will be written in the current + argument is given, will be written in the specified directory. Args: + run_dir: Default directory to use. filename: Default file name to use. Returns: @@ -119,7 +120,7 @@ def extract_log_file_location(filename: str) -> Optional[Path]: given_path_str = arg[len(prefix):] if given_path_str == '': - return Path('.') / filename + return run_dir / filename given_path = Path(given_path_str) diff --git a/muscle_manager/muscle_manager.py b/muscle_manager/muscle_manager.py index 6385687b..73d3cbbc 100644 --- a/muscle_manager/muscle_manager.py +++ b/muscle_manager/muscle_manager.py @@ -1,16 +1,29 @@ +from typing import Sequence + import click import ymmsl +from ymmsl import Implementation, PartialConfiguration, Reference, Resources from libmuscle.manager.manager import start_server @click.command() -@click.argument('ymmsl_file') -def manage_simulation(ymmsl_file: str) -> None: - with open(ymmsl_file) as f: - configuration = ymmsl.load(f) +@click.argument( + 'ymmsl_files', nargs=-1, required=True, type=click.Path( + exists=True, file_okay=True, dir_okay=False, readable=True, + allow_dash=True, resolve_path=True)) +def manage_simulation(ymmsl_files: Sequence[str]) -> None: + configuration = PartialConfiguration() + for path in ymmsl_files: + with open(path, 'r') as f: + configuration.update(ymmsl.load(f)) + # temporary dummy data to satisfy type + configuration.update(PartialConfiguration( + None, None, + {Reference('dummy'): Implementation(Reference('dummy'), '')}, + {Reference('dummy'): Resources(Reference('dummy'), 0)})) - server = start_server(configuration) + server = start_server(configuration.as_configuration()) print(server.get_location()) server.wait() diff --git a/setup.cfg b/setup.cfg index 4742cd5b..5c62d746 100644 --- a/setup.cfg +++ b/setup.cfg @@ -4,14 +4,13 @@ test = pytest [tool:pytest] testpaths = muscle_manager libmuscle/python integration_test -addopts = --mypy --flake8 --cov --cov-report xml --cov-report term-missing -x +addopts = --mypy --flake8 --cov --cov-report xml --cov-report term-missing flake8-ignore = setup.py E501 muscle_manager/protocol/*.py ALL libmuscle/manager_protocol/*.py ALL [mypy] -paths = muscle_manager libmuscle/python mypy_path = libmuscle/python:muscle_manager_protocol warn_unused_configs = True disallow_subclassing_any = True diff --git a/setup.py b/setup.py index 4c771b49..824e96ad 100644 --- a/setup.py +++ b/setup.py @@ -45,14 +45,14 @@ }, python_requires='>=3.5, <4', install_requires=[ - 'click', + 'click>=6', 'grpcio>=1.24.3, <2', 'msgpack', 'netifaces', - 'numpy>=1.12', + 'numpy>=1.12,<1.20', 'protobuf>=3.10.0, <4', 'typing_extensions', - 'ymmsl==0.10.1' # Also in CI, update there as well + 'ymmsl>=0.11.0' # Also in CI, update there as well ], setup_requires=[ 'pytest-runner', @@ -67,10 +67,11 @@ tests_require=[ 'coverage<5', 'mypy', - 'pytest>=3.5', + 'pytest>=3.5,<6.2', 'pytest-cov', 'pytest-flake8', - 'pytest-mypy' + 'pytest-mypy', + 'importlib-metadata==2.1.0' ], extras_require={ 'dev': [