Merge branch 'release-0.3.2'

.github/workflows/ci.yaml

@@ -15,7 +15,8 @@ jobs:
 
     - name: Install dependencies
       run: |
-        sudo apt-get install -y build-essential cmake gfortran libopenmpi-dev pkg-config wget
+        sudo apt-get update
+        sudo apt-get install -y build-essential cmake gfortran libopenmpi-dev pkg-config wget valgrind
         sudo apt-get install -y libssl-dev zlib1g-dev
         pip install ymmsl==0.10.1
 

.github/workflows/ci_python3.5.1.yaml

@@ -15,5 +15,5 @@ jobs:
         path: ${{ github.workspace }}/.eggs
         key: python-compatibility-3.5.1-eggs
 
-    - name: Run Python tests on 3.5.1 latest
-      run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.5.1 /bin/bash -c 'cd /home/muscle3 && pip install -U pip setuptools wheel ymmsl==0.10.1 && make test_python_only'
+    - name: Run Python tests on 3.5.1
+      run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.5.1 /bin/bash -c 'cd /home/muscle3 && pip install -U pip setuptools wheel "numpy<1.19" ymmsl==0.10.1 && make test_python_only'

.github/workflows/ci_python3.5.yaml

@@ -16,4 +16,4 @@ jobs:
         key: python-compatibility-3.5-eggs
 
     - name: Run Python tests on 3.5 latest
-      run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.5 /bin/bash -c 'cd /home/muscle3 && pip install ymmsl==0.10.1 && make test_python_only'
+      run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.5 /bin/bash -c 'cd /home/muscle3 && pip install numpy ymmsl==0.10.1 && make test_python_only'

.github/workflows/ci_ubuntu16.04.yaml

@@ -4,6 +4,9 @@ name: native_compatibility_ubuntu16.04
 on:
   schedule:
     - cron: '0 1 * * 0'
+  push:
+    branches:
+      - 'release-*'
 jobs:
   build:
     runs-on: ubuntu-latest
@@ -12,4 +15,4 @@ jobs:
     - uses: actions/checkout@v2
 
     - name: Run tests on Ubuntu 16.04
-      run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:16.04 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran libopenmpi-dev pkg-config python3 python3-pip python3-venv curl && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && pip3 install ymmsl==0.10.1 && make test_examples'
+      run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:16.04 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran valgrind libopenmpi-dev pkg-config python3 python3-pip python3-venv curl && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && pip3 install ymmsl==0.10.1 && make test_examples'

.github/workflows/ci_ubuntu18.04.yaml

@@ -4,6 +4,9 @@ name: native_compatibility_ubuntu18.04
 on:
   schedule:
     - cron: '0 2 * * 0'
+  push:
+    branches:
+      - 'release-*'
 jobs:
   build:
     runs-on: ubuntu-latest
@@ -12,4 +15,4 @@ jobs:
     - uses: actions/checkout@v2
 
     - name: Run tests on Ubuntu 18.04
-      run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:18.04 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran libopenmpi-dev pkg-config python3 python3-pip python3-venv curl && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && pip3 install ymmsl==0.10.1 && make test_examples'
+      run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:18.04 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran valgrind libopenmpi-dev pkg-config python3 python3-pip python3-venv curl && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && pip3 install ymmsl==0.10.1 && make test_examples'

.github/workflows/ci_ubuntu19.10.yaml

@@ -4,6 +4,9 @@ name: native_compatibility_ubuntu19.10
 on:
   schedule:
     - cron: '0 3 * * 0'
+  push:
+    branches:
+      - 'release-*'
 jobs:
   build:
     runs-on: ubuntu-latest
@@ -12,4 +15,4 @@ jobs:
     - uses: actions/checkout@v2
 
     - name: Run tests on Ubuntu 19.10
-      run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:19.10 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran libopenmpi-dev pkg-config python3 python3-pip python3-venv curl && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && pip3 install ymmsl==0.10.1 && make test_examples'
+      run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:19.10 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran valgrind libopenmpi-dev pkg-config python3 python3-pip python3-venv curl && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && pip3 install ymmsl==0.10.1 && make test_examples'

.github/workflows/ci_ubuntu20.04.yaml

@@ -4,6 +4,9 @@ name: native_compatibility_ubuntu20.04
 on:
   schedule:
     - cron: '0 4 * * 0'
+  push:
+    branches:
+      - 'release-*'
 jobs:
   build:
     runs-on: ubuntu-latest
@@ -12,4 +15,4 @@ jobs:
     - uses: actions/checkout@v2
 
     - name: Run tests on Ubuntu 20.04
-      run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:20.04 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran libopenmpi-dev pkg-config python3 python3-pip python3-venv curl && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && pip3 install ymmsl==0.10.1 && make test_examples'
+      run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:20.04 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran valgrind libopenmpi-dev pkg-config python3 python3-pip python3-venv curl && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && pip3 install ymmsl==0.10.1 && make test_examples'

.github/workflows/ci_ubuntu20.10.yaml

@@ -0,0 +1,18 @@
+# Run Continuous Integration for the latest Ubuntu release
+# This mainly checks for issues/regressions in the native build
+name: native_compatibility_ubuntu20.10
+on:
+  schedule:
+    - cron: '0 4 * * 0'
+  push:
+    branches:
+      - 'release-*'
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Run tests on Ubuntu 20.10
+      run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:20.10 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran valgrind libopenmpi-dev pkg-config python3 python3-pip python3-venv curl && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && pip3 install ymmsl==0.10.1 && make test_examples'

CHANGELOG.rst

@@ -5,6 +5,32 @@ Change Log
 All notable changes to this project will be documented in this file.
 This project adheres to `Semantic Versioning <http://semver.org/>`_.
 
+0.3.2
+*****
+
+Improved
+--------
+
+* Accessing settings from C++ now more flexible
+* Python produces more detailed logs to aid in debugging
+* Improved pkg-config set-up
+* Improved build system output to help find problems
+* Documentation on logging in Python
+* Protobuf dependency build now more compatible
+
+Fixed
+-----
+
+* C++ list/dict building functions
+* C++ use-after-free when receiving grids
+
+Thanks
+------
+
+* Pavel for testing and reporting issues
+* Dongwei for testing and reporting issues
+
+
 0.3.1
 *****
 

Makefile

@@ -85,7 +85,7 @@ install: all
 	@echo "    You can also use pkg-config. Add $(PREFIX)/lib/pkgconfig"
 	@echo '*   to your PKG_CONFIG_PATH environment variable and use module    *'
 	@echo '*   names libmuscle, libmuscle_mpi, libmuscle_fortran or           *'
-	@echo '*   libmuscle_mpi_fortran.                                         *'
+	@echo '*   libmuscle_mpi_fortran and ymmsl or ymmsl_fortran.              *'
 	@echo '*                                                                  *'
 	@echo '*   If the directory you installed MUSCLE 3 in is not in your      *'
 	@echo "*   system's library search path, then you have to set             *"
@@ -116,6 +116,7 @@ clean:
 	cd scripts && $(MAKE) clean
 	cd docs/source/examples && $(MAKE) clean
 	rm -rf ./build
+	rm -rf $(CURDIR)/libmuscle/build/test_install/*
 
 .PHONY: distclean
 distclean:
@@ -124,6 +125,7 @@ distclean:
 	cd scripts && $(MAKE) distclean
 	cd docs/source/examples && $(MAKE) clean
 	rm -rf ./build
+	rm -rf $(CURDIR)/libmuscle/build/test_install/*
 
 .PHONY: fortran
 fortran: cpp

README.rst

@@ -1,4 +1,4 @@
-.. image:: https://github.com/multiscale/muscle3/raw/master/docs/source/muscle3_logo_readme.png
+.. image:: https://github.com/multiscale/muscle3/raw/develop/docs/source/muscle3_logo_readme.png
     :alt: MUSCLE 3
 
 .. image:: https://readthedocs.org/projects/muscle3/badge/?version=master

VERSION

@@ -1 +1 @@
-0.3.1
+0.3.2

docs/source/examples/cpp/build/Makefile

@@ -1,10 +1,12 @@
 CXX ?= g++
-MPICXX := mpic++
-CXXFLAGS += -std=c++14 -g $(shell pkg-config --cflags libmuscle)
-MPI_CXXFLAGS := -std=c++14 -g $(shell pkg-config --cflags libmuscle_mpi)
+CXXFLAGS += -std=c++14 -g $(shell pkg-config --cflags libmuscle ymmsl)
+LDFLAGS += $(shell pkg-config --libs libmuscle ymmsl)
 
-MPI_LDFLAGS := $(LDFLAGS) $(shell pkg-config --libs libmuscle_mpi)
-LDFLAGS += $(shell pkg-config --libs libmuscle)
+ifdef MUSCLE_ENABLE_MPI
+    MPICXX := mpic++
+    MPI_CXXFLAGS := -std=c++14 -g $(shell pkg-config --cflags libmuscle_mpi ymmsl)
+    MPI_LDFLAGS := $(shell pkg-config --libs libmuscle_mpi ymmsl)
+endif
 
 
 binaries := reaction diffusion mc_driver load_balancer

docs/source/examples/fortran/build/Makefile

@@ -1,9 +1,12 @@
 FC ?= gfortran
-MPIFC ?= mpifort
-FFLAGS += -std=f2003 -g $(shell pkg-config --cflags libmuscle_fortran)
+FFLAGS += -std=f2003 -g $(shell pkg-config --cflags libmuscle_fortran ymmsl_fortran)
+LDFLAGS := $(shell pkg-config --libs libmuscle_fortran ymmsl_fortran)
 
-MPI_LDFLAGS := $(shell pkg-config --libs libmuscle_mpi_fortran)
-LDFLAGS := $(shell pkg-config --libs libmuscle_fortran)
+ifdef MUSCLE_ENABLE_MPI
+    MPIFC ?= mpifort
+    MPI_FFLAGS := -std=f2003 -g $(shell pkg-config --cflags libmuscle_mpi_fortran ymmsl_fortran)
+    MPI_LDFLAGS := $(shell pkg-config --libs libmuscle_mpi_fortran ymmsl_fortran)
+endif
 
 
 binaries := reaction diffusion mc_driver load_balancer
@@ -26,5 +29,5 @@ clean:
 	LD_LIBRARY_PATH=$(MUSCLE3_HOME)/lib:$(LD_LIBRARY_PATH) $(FC) $(FFLAGS) -o $@ $^ $(LDFLAGS)
 
 %_mpi: ../%_mpi.f03
-	LD_LIBRARY_PATH=$(MUSCLE3_HOME)/lib:$(LD_LIBRARY_PATH) $(MPIFC) $(FFLAGS) -o $@ $^ $(MPI_LDFLAGS)
+	LD_LIBRARY_PATH=$(MUSCLE3_HOME)/lib:$(LD_LIBRARY_PATH) $(MPIFC) $(MPI_FFLAGS) -o $@ $^ $(MPI_LDFLAGS)
 

docs/source/fortran_api.rst

@@ -2302,6 +2302,22 @@ YMMSL_Settings
     :r is: ``.true.`` if the value is of type logical.
     :rtype is: logical
 
+.. f:function:: YMMSL_Settings_is_a_int4(self, key, err_code, err_msg)
+
+    Return whether a value is of type ``YMMSL_int4``.
+
+    This returns ``.true.`` if the value is an integer and fits in an int4.
+
+    If the given key does not exist, then ``err_code`` will be set to
+    ``YMMSL_out_of_bounds`` and the result will be invalid.
+
+    :p YMMSL_Settings self: The Settings object to inspect.
+    :p character key: The name of the setting to check.
+    :p integer err_code: An error code output (optional).
+    :p character err_msg: An error message output (allocatable, optional).
+    :r is: ``.true.`` if the value is of type ``YMMSL_int4``.
+    :rtype is: logical
+
 .. f:function:: YMMSL_Settings_is_a_int8(self, key, err_code, err_msg)
 
     Return whether a value is of type ``YMMSL_int8``.
@@ -2320,6 +2336,9 @@ YMMSL_Settings
 
     Return whether a value is of type ``YMMSL_real8``.
 
+    This will also return ``.true.`` if the value is an integer, even if
+    converting it would lose precision.
+
     If the given key does not exist, then ``err_code`` will be set to
     ``YMMSL_out_of_bounds`` and the result will be invalid.
 
@@ -2365,9 +2384,10 @@ YMMSL_Settings
     If no setting with the given key exists, one is added, if one does,
     it is overwritten.
 
-    ``value`` may be a character (string), a logical, an 8-byte integer (e.g.
-    ``YMMSL_int8``), an 8-byte real number (``YMMSL_real8``), or a one- or
-    two-dimensional arrays of 8-byte real numbers.
+    ``value`` may be a character (string), a logical, a 4-byte integer (e.g.
+    ``YMMSL_int4``), an 8-byte integer (e.g.  ``YMMSL_int8``), an 8-byte real
+    number (``YMMSL_real8``), or a one- or two-dimensional arrays of 8-byte real
+    numbers.
 
     :p YMMSL_Settings self: The Settings object to modify.
     :p character key: The name of the setting.
@@ -2401,6 +2421,20 @@ YMMSL_Settings
     :r value: The value at the given index
     :rtype value: logical
 
+.. f:function:: YMMSL_Settings_get_as_int4(self, key, err_code, err_msg)
+
+    Return the value of an integer-typed setting.
+
+    If this setting is not currently set to a integer-typed value, or the
+    value is out of range for an int4, then ``err_code`` will be set to
+    ``YMMSL_bad_cast`` and the result will be invalid.
+
+    :p character key: The name of the setting to get.
+    :p integer err_code: An error code output (optional).
+    :p character err_msg: An error message output (allocatable, optional).
+    :r value: The value at the given index (YMMSL_int4)
+    :rtype value: integer
+
 .. f:function:: YMMSL_Settings_get_as_int8(self, key, err_code, err_msg)
 
     Return the value of an integer-typed setting.
@@ -2419,7 +2453,10 @@ YMMSL_Settings
 
     Return the value of a real-typed setting.
 
-    If this setting is not currently set to a real-typed value,
+    This will also work if the setting is integer-typed in which case it
+    will be converted, with possible loss of precision.
+
+    If this setting is not currently set to a real- or integer-typed value,
     then ``err_code`` will be set to ``YMMSL_bad_cast`` and the result
     will be invalid.
 

docs/source/installing.rst.in

@@ -305,7 +305,7 @@ libraries in case you need them. If you link statically, then you must add the
 
 MUSCLE 3 also supports ``pkg-config``. To use ``pkg-config``, add
 ``<PREFIX>/lib/pkgconfig`` to your ``PKG_CONFIG_PATH`` and use the module names
-``libmuscle`` or ``libmuscle_mpi``.
+``ymmsl`` and either ``libmuscle`` or ``libmuscle_mpi``.
 
 There's one more thing: the directory that you've installed MUSCLE into is
 probably not in your system's library search path, and as a result the dynamic
@@ -397,7 +397,7 @@ the libmuscle library instead:
 
 MUSCLE 3 also supports ``pkg-config``. To use ``pkg-config``, add
 ``<PREFIX>/lib/pkgconfig`` to your ``PKG_CONFIG_PATH`` and use the module names
-``libmuscle_fortran`` or ``libmuscle_mpi_fortran``.
+``ymmsl_fortran`` and either ``libmuscle_fortran`` or ``libmuscle_mpi_fortran``.
 
 There's one more thing: the directory that you've installed MUSCLE into is
 probably not in your system's library search path, and as a result the dynamic

docs/source/releasing.rst

@@ -55,7 +55,7 @@ shows up:
 
 .. code-block:: bash
 
-  python setup.py build_sphinx
+  make docs
 
 It may give some warnings about missing references, that's a known issue and
 normally harmless. Next, point your web browser to
@@ -67,8 +67,8 @@ Run tests
 ---------
 
 Before we make a commit, the tests should be run, and this is a good idea anyway
-if we're making a release. So run ``python setup.py test`` and check that
-everything is in order.
+if we're making a release. So run ``make test`` and check that everything is in
+order.
 
 Commit the version update
 -------------------------
@@ -136,8 +136,13 @@ can start using it. To build, use:
 
 .. code-block:: bash
 
+  rm -r ./build
   python3 setup.py sdist bdist_wheel
 
+Note that we remove ``./build``, which is the build directory setuptools uses,
+to ensure that we're doing a clean build, I've seen some weird mixes of versions
+on occasion so it's better to be safe than sorry.
+
 We can then check to see if everything is okay using
 
 .. code-block:: bash
@@ -150,6 +155,13 @@ and if all seems well, we can upload to PyPI:
 
   twine upload dist/muscle3-x.y.z*
 
+Announce release
+----------------
+
+Announce the release in the usual places, so that people know it exists. There
+should be a short release message listing new features and fixed bugs, and don't
+forget to thank everyone who contributed!
+
 Merge the release branch back into develop
 ------------------------------------------