diff --git a/README.md b/README.md index 967338fc..cc09139c 100644 --- a/README.md +++ b/README.md @@ -24,15 +24,10 @@ GeoNode template project. Generates a django project with GeoNode support. pip install Django==3.2.* mkdir ~/project_name - ``` - ```bash GN_VERSION=master - django-admin startproject --template=https://github.com/GeoNode/geonode-project/archive/refs/tags/$GN_VERSION.zip -e py,sh,md,rst,json,yml,ini,env,sample,properties -n monitoring-cron -n Dockerfile project_name ~/project_name - ``` - - ```bash + cd ~/project_name python create-envfile.py ``` @@ -57,6 +52,8 @@ GeoNode template project. Generates a django project with GeoNode support. docker compose up -d ``` +In case you want to customize the Docker images for Geoserver, Nginx, PostGIS, Letsencrypt or the data-dir setup, you can use the `docker-compose-local-builds.yml` override, which builds the images from the `./docker/*` subfolders + ## Developer Workshop Available at diff --git a/docker-compose-local-builds.yml b/docker-compose-local-builds.yml new file mode 100644 index 00000000..127b5379 --- /dev/null +++ b/docker-compose-local-builds.yml @@ -0,0 +1,20 @@ +version: '3.9' + + # Nginx is serving django static and media files and proxies to django and geonode + geonode: + build: ./docker/nginx/ + + # Gets and installs letsencrypt certificates + letsencrypt: + build: ./docker/letsencrypt/ + + # Geoserver backend + geoserver: + build: ./docker/geoserver/ + + data-dir-conf: + build: ./docker/data_dir_conf/ + + # PostGIS database. + db: + build: ./docker/postgis/ diff --git a/docker-compose.yml b/docker-compose.yml index 4933a28e..023235c1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -54,7 +54,6 @@ services: # Nginx is serving django static and media files and proxies to django and geonode geonode: image: geonode/nginx:1.25.1 - build: ./docker/nginx/ container_name: nginx4${COMPOSE_PROJECT_NAME} env_file: - .env @@ -72,7 +71,6 @@ services: # Gets and installs letsencrypt certificates letsencrypt: image: geonode/letsencrypt:latest - build: ./docker/letsencrypt/ container_name: letsencrypt4${COMPOSE_PROJECT_NAME} env_file: - .env @@ -83,7 +81,6 @@ services: # Geoserver backend geoserver: image: geonode/geoserver:2.23.0 - build: ./docker/geoserver/ container_name: geoserver4${COMPOSE_PROJECT_NAME} healthcheck: test: "curl -m 10 --fail --silent --write-out 'HTTP CODE : %{http_code}\n' --output /dev/null http://geoserver:8080/geoserver/ows" diff --git a/docker/data_dir_conf/Dockerfile b/docker/data_dir_conf/Dockerfile new file mode 100644 index 00000000..7d95c825 --- /dev/null +++ b/docker/data_dir_conf/Dockerfile @@ -0,0 +1,25 @@ +FROM alpine:latest +LABEL GeoNode development team + +# Install curl in alpine 3.3+ +RUN apk --no-cache add curl + +# Download required files +RUN mkdir -p /tmp/geonode/downloaded +ENV TEMP_DOWNLOADED /tmp/geonode/downloaded +WORKDIR ${TEMP_DOWNLOADED} + +ENV GEOSERVER_VERSION=2.23.0 + +ADD download.sh ${TEMP_DOWNLOADED} +RUN chmod +x ${TEMP_DOWNLOADED}/download.sh +RUN ${TEMP_DOWNLOADED}/download.sh $GEOSERVER_VERSION $TEMP_DOWNLOADED + +# for debugging +RUN ls -lart + +# preparing the volume +ENV BASE_GEOSERVER_DATA_DIR /geoserver_data +RUN mkdir -p ${BASE_GEOSERVER_DATA_DIR} +RUN cp -r ${TEMP_DOWNLOADED}/data ${BASE_GEOSERVER_DATA_DIR} +VOLUME ${BASE_GEOSERVER_DATA_DIR}/data diff --git a/docker/data_dir_conf/README.rst b/docker/data_dir_conf/README.rst new file mode 100644 index 00000000..3cc26686 --- /dev/null +++ b/docker/data_dir_conf/README.rst @@ -0,0 +1,192 @@ +***************************** +How to start with data-docker +***************************** + +What you can do with this docker container +========================================== + +The **data-docker** project can be your base data container volume to add data from scratch to GeoServer and PostGIS and then make them persisted when you want to stop your current containers. + +Quick introduction on creating persisted storage in Docker +========================================================== + +Data persistence strategies +--------------------------- + +The most commonly used approches in `Docker`_ are `Data volumes`_ and `Data volume containers`_ which essentially create a file system data volume rather than a dedicated data volume container. + +.. _Docker: https://www.docker.com/technologies/overview + +.. warning:: If you want to follow this readme and run the docker command you have to make sure that your docker host environment has been already set and your docker default machine has been started. + +.. note:: This is required by all developers who are using `Docker-Machine`_ for running containers. Those use native linux or vm like `Docker for Mac`_ and `Docker for Windows`_ don't need these checks. + +.. _Docker-Machine: https://docs.docker.com/machine/ +.. _Docker for Mac: https://www.docker.com/docker-mac +.. _Docker for Windows: https://www.docker.com/docker-windows + +Docker-Machine +-------------- + +In order to verify base prerequirements please run this commands below: + +.. code-block:: console + + $ docker-machine env + +which should output something similar: + +.. code-block:: console + + export DOCKER_TLS_VERIFY="1" + export DOCKER_HOST="tcp://192.168.99.100:2376" + export DOCKER_CERT_PATH="/.docker/machine/machines/default" + export DOCKER_MACHINE_NAME="default" + +and then configure your shell environment + +.. code-block:: console + + $ eval $(docker-machine env) + +Data volumes +------------ + +You can easily add a data volume to a container using the option `-v` with the command `run` as the example below: + +.. code-block:: console + + $ docker run -d -P --name geonode-data-container -v /data geonode/geonode-data touch /data/test.txt + +Which creates a data volume within the new container in the `/data` directory. Data volumes are very useful because can be shared and included into other containers. It can be also remarked that the volume created will persist the `test.txt` file. + +Host data volumes +^^^^^^^^^^^^^^^^^ + +You can also mount a directory from your Docker daemon’s host into a container and this could be very useful for testing but even for production in case your host machine can share a storage mount point like a network file system (*NFS*). This approach despite its ease to implement nature has actually an heavy constraint that you have to pre-configure the mount point in your docker host. +This breaks two of the biggest Docker advantages: **container portability** and **applications run anywhere**. + +.. code-block:: console + + $ docker run -d -P --name geonode-data-container -v /geonode_data/data:/data geonode/geonode-data touch /data/test.txt + +In case of the GeoNode *data* for example you cannot start from scratch in development like you actually do with + +.. code-block:: console + + (venv)$ paver reset_hard + +Data volume containers +---------------------- +A data volume container is essentially a docker image that defines storage space. The container itself just defines a place inside docker's virtual file system where data is stored. The container doesn’t run a process and in fact *stops* immediately after `docker run` is called as the container exists in a stopped state, so along with its data. + +So let's create a dedicated container that holds all of GeoNode persistent shareable data resources, mounting the data inside of it and then eventually into other containers once created and setup: + +.. code-block:: console + + $ docker create -v /geonode-store --name geonode-data-container geonode/geonode-data /bin/true + +.. note:: `/bin/true` returns a `0` and does nothing if the command was successful. + +And the with the option `--volumes-from` you can mount the created volume `/geonode-store` within other containers by running: + +.. code-block:: console + + $ docker run -d --volumes-from geonode-data-container --name geoserver geonode/geoserver + +.. hint:: Notice that if you remove containers that mount volumes, the volume store and its data will not be deleted since docker preserves that. + +To completely delete a volume from the file system you have to run: + +.. code-block:: console + + $ docker rm -v geoserver + +How to start with docker-compose for geonode data volume container +================================================================== + +Start the creation of a volume with the **GEOSERVER_DATA_DIR** in the directory `/geoserver_data/data`: + +.. code-block:: console + + $ docker-compose up + +Run a geoserver container with such created volume: + +.. code-block:: console + + $ docker run -d --volumes-from geoserver_data_dir --name geoserver geonode/geoserver + +Verify that the preloaded `GeoServer Data Directory for GeoServer 2.18.3`_ build from Jenkins is actually there: + +.. _GeoServer Data Directory for GeoServer 2.18.3: https://www.dropbox.com/s/cd20is9ddjz7ti5/data-2.18.3.zip + +.. code-block:: console + + $ docker exec -it geoserver ls -lart /geoserver_data/data/ + +The output should be something similar: + +.. code-block:: console + + total 76 + drwxr-xr-x 3 root root 4096 Aug 27 16:51 workspaces + -rw-r--r-- 1 root root 1547 Aug 27 16:51 wms.xml + -rw-r--r-- 1 root root 2013 Aug 27 16:51 wfs.xml + -rw-r--r-- 1 root root 1285 Aug 27 16:51 wcs.xml + drwxr-xr-x 2 root root 4096 Aug 27 16:51 styles + drwxr-xr-x 8 root root 4096 Aug 27 16:51 security + drwxr-xr-x 2 root root 4096 Aug 27 16:51 printing + drwxr-xr-x 2 root root 4096 Aug 27 16:51 plugIns + drwxr-xr-x 2 root root 4096 Aug 27 16:51 palettes + drwxr-xr-x 2 root root 4096 Aug 27 16:51 logs + -rw-r--r-- 1 root root 144 Aug 27 16:51 logging.xml + drwxr-xr-x 2 root root 4096 Aug 27 16:51 images + -rw-r--r-- 1 root root 1111 Aug 27 16:51 gwc-gs.xml + -rw-r--r-- 1 root root 1374 Aug 27 16:51 global.xml + drwxr-xr-x 2 root root 4096 Aug 27 16:51 geonode + drwxr-xr-x 2 root root 4096 Aug 27 16:51 demo + -rw-r--r-- 1 root root 184 Aug 27 16:51 README.rst + drwxr-xr-x 12 root root 4096 Aug 27 16:51 . + drwxr-xr-x 7 root root 4096 Aug 27 17:08 .. + +How to define a docker-compose that uses data-docker +---------------------------------------------------- + +A docker-compose.yml can be defined in such a way with a service that mounts this data directory from a `tag of Docker Hub builds`_, in this case the version for `GeoServer-GeoNode 2.18.3`_: + +.. _tag of Docker Hub builds: https://hub.docker.com/r/geonode/geoserver_data/builds/ + +.. _GeoServer-GeoNode 2.18.3: https://github.com/GeoNode/geoserver-geonode-ext/tree/2.18.3 + +.. code-block:: yaml + + version: '2' + + services: + geoserver: + build: . + ports: + - "8888:8080" + volumes_from: + # reference to the service which has the volume with the preloaded geoserver_data_dir + - data_dir_conf + data_dir_conf: + image: geonode/geoserver_data:2.18.3 + container_name: geoserver_data_dir # named data container + command: /bin/true + volumes: + - /geoserver_data/data + + volumes: + # reference to the named data container that holds the preloaded geoserver data directory + geoserver_data_dir: + + +Available tags +-------------- + +There are several different tags from the `Docker Hub builds`_: + +.. _Docker Hub builds: https://cloud.docker.com/u/geonode/repository/registry-1.docker.io/geonode/geoserver_data/builds/ + diff --git a/docker/data_dir_conf/docker-compose.yml b/docker/data_dir_conf/docker-compose.yml new file mode 100644 index 00000000..11dea36d --- /dev/null +++ b/docker/data_dir_conf/docker-compose.yml @@ -0,0 +1,14 @@ +version: '2' + +services: + data_dir_conf: + build: . + image: geonode/geoserver_data:2.23.0 + container_name: geoserver_data_dir + command: /bin/true + volumes: + - /geoserver_data/data + +volumes: + # reference to the named data container that holds the preloaded geoserver data directory + geoserver_data_dir: \ No newline at end of file diff --git a/docker/data_dir_conf/download.sh b/docker/data_dir_conf/download.sh new file mode 100644 index 00000000..04dcea61 --- /dev/null +++ b/docker/data_dir_conf/download.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env sh + +# Wait for version to come up before downloading it +# args $1 - version +# args $2 - temp directory + +echo "GeoServer Data Dir version is $1" +echo "-----------------------------------------------------------------------------------------------" +echo "Archive temporary directory is $2" + +GEOSERVER_VERSION=$1 +TEMP_DOWNLOADED=$2 + +echo "GeoServer Data Directory is going to be downloaded" +artifact_url="https://artifacts.geonode.org/geoserver/$GEOSERVER_VERSION/geonode-geoserver-ext-web-app-data.zip" +echo "Downloading: $artifact_url" +curl -k -L "$artifact_url" --output data.zip && unzip -x -d ${TEMP_DOWNLOADED} data.zip +echo "GeoServer Data Directory download has been completed" diff --git a/docker/postgis/Dockerfile b/docker/postgis/Dockerfile new file mode 100644 index 00000000..7a473874 --- /dev/null +++ b/docker/postgis/Dockerfile @@ -0,0 +1,5 @@ +FROM postgis/postgis:15-3.3-alpine +LABEL GeoNode development team + +COPY ./initdb-geonode.sh /docker-entrypoint-initdb.d/geonode.sh +RUN chmod +x /docker-entrypoint-initdb.d/geonode.sh \ No newline at end of file diff --git a/docker/postgis/initdb-geonode.sh b/docker/postgis/initdb-geonode.sh new file mode 100644 index 00000000..11f358bb --- /dev/null +++ b/docker/postgis/initdb-geonode.sh @@ -0,0 +1,67 @@ +#!/bin/bash +set -e + +function create_geonode_user_and_database() { + local db=$1 + local geonode_user="${GEONODE_DATABASE_USER:-$db}" + local geonode_password="$GEONODE_DATABASE_PASSWORD" + echo " Creating user and database '$db' with Geonode user: $geonode_user" + psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" <<-EOSQL + CREATE USER $db; + ALTER USER $geonode_user with encrypted password '$geonode_password'; + ALTER USER $geonode_user CREATEDB; + CREATE DATABASE $db; + GRANT CREATE ON SCHEMA public TO $geonode_user; + GRANT USAGE ON SCHEMA public TO $geonode_user; + GRANT ALL PRIVILEGES ON DATABASE $db TO $geonode_user; +EOSQL + psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" -d "$db" -c "GRANT ALL ON SCHEMA public TO $geonode_user;" + psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" -d "$db" -c "GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO $geonode_user;" + psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" -d "$db" -c "GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO $geonode_user;" + psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" -d "$db" -c "GRANT INSERT, SELECT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO $geonode_user;" +} + +function create_geonode_user_and_geodatabase() { + local geodb=$1 + local geonode_geodb_user="${GEONODE_GEODATABASE_USER:-$geodb}" + local geonode_geodb_password="$GEONODE_GEODATABASE_PASSWORD" + + echo " Creating user and database '$geodb' with Geonode GeoDB user: $geonode_geodb_user" + psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" <<-EOSQL + CREATE USER $geodb; + ALTER USER $geonode_geodb_user with encrypted password '$geonode_geodb_password'; + ALTER USER $geonode_geodb_user CREATEDB; + CREATE DATABASE $geodb; + GRANT CREATE ON SCHEMA public TO $geonode_geodb_user; + GRANT USAGE ON SCHEMA public TO $geonode_geodb_user; + GRANT ALL PRIVILEGES ON DATABASE $geodb TO $geonode_geodb_user; +EOSQL + psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" -d "$geodb" -c "GRANT ALL ON SCHEMA public TO $geonode_geodb_user;" + psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" -d "$geodb" -c "GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO $geonode_geodb_user;" + psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" -d "$geodb" -c "GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO $geonode_geodb_user;" + psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" -d "$geodb" -c "GRANT INSERT, SELECT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO $geonode_geodb_user;" +} + +function update_database_with_postgis() { + local db=$1 + echo " Updating databse '$db' with extension" + psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$db" <<-EOSQL + CREATE EXTENSION IF NOT EXISTS postgis; + GRANT ALL ON geometry_columns TO PUBLIC; + GRANT ALL ON spatial_ref_sys TO PUBLIC; +EOSQL +} + +if [ -n "$GEONODE_DATABASE" ]; then + echo "Geonode database creation requested: $GEONODE_DATABASE" + create_geonode_user_and_database $GEONODE_DATABASE + update_database_with_postgis $GEONODE_DATABASE + echo "Geonode database created" +fi + +if [ -n "$GEONODE_GEODATABASE" ]; then + echo "Geonode geodatabase creation requested: $GEONODE_GEODATABASE" + create_geonode_user_and_geodatabase $GEONODE_GEODATABASE + update_database_with_postgis $GEONODE_GEODATABASE + echo "Geonode geodatabase created" +fi \ No newline at end of file