From e9fd95dc1e61608dcf26d031f66350a2d746f5a8 Mon Sep 17 00:00:00 2001 From: Brett Date: Thu, 9 Dec 2021 14:41:16 +0100 Subject: [PATCH 01/14] nginx changes --- nginx/Dockerfile | 2 +- nginx/{ => setup}/index.html | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename nginx/{ => setup}/index.html (100%) diff --git a/nginx/Dockerfile b/nginx/Dockerfile index a62d0d3..763ec0c 100644 --- a/nginx/Dockerfile +++ b/nginx/Dockerfile @@ -12,4 +12,4 @@ COPY setup/sites-available/* ${NGINX_DIR}/sites-available RUN ln -s ${NGINX_DIR}/sites-available/ckan.conf ${NGINX_DIR}/sites-enabled/ckan.conf -EXPOSE 80 \ No newline at end of file +EXPOSE 80 diff --git a/nginx/index.html b/nginx/setup/index.html similarity index 100% rename from nginx/index.html rename to nginx/setup/index.html From c5731f0cc240ca8964206fc4a20c318fc2689c96 Mon Sep 17 00:00:00 2001 From: Brett Date: Thu, 9 Dec 2021 14:44:35 +0100 Subject: [PATCH 02/14] update nginx Dockerfile --- nginx/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nginx/Dockerfile b/nginx/Dockerfile index 763ec0c..8bdcef2 100644 --- a/nginx/Dockerfile +++ b/nginx/Dockerfile @@ -2,7 +2,7 @@ FROM nginx:alpine ENV NGINX_DIR=/etc/nginx -COPY index.html /usr/share/nginx/html/index.html +COPY setup/index.html /usr/share/nginx/html/index.html RUN mkdir -p ${NGINX_DIR}/sites-available RUN mkdir -p ${NGINX_DIR}/sites-enabled From c729b06cdb2a8632460789f38fa14760c986b1d0 Mon Sep 17 00:00:00 2001 From: Brett Date: Fri, 10 Dec 2021 11:43:04 +0100 Subject: [PATCH 03/14] Update solr --- solr/Dockerfile | 18 +++++++++--------- solr/{solrconfig.xml => solrconfig-2.9.4.xml} | 0 2 files changed, 9 insertions(+), 9 deletions(-) rename solr/{solrconfig.xml => solrconfig-2.9.4.xml} (100%) diff --git a/solr/Dockerfile b/solr/Dockerfile index 8922342..feeb423 100755 --- a/solr/Dockerfile +++ b/solr/Dockerfile @@ -3,7 +3,7 @@ FROM solr:6.6.6 # Enviroment variables ENV SOLR_CORE ckan ENV SOLR_VERSION 6.6.6 -ENV CKAN_VERSION 2.9.1 +ENV CKAN_VERSION 2.9.4 ###TODO!!! CKAN_VERSION to be passed in as an ARG # root user for initial config @@ -15,14 +15,14 @@ RUN mkdir -p /opt/solr/server/solr/${SOLR_CORE}/conf && \ mkdir -p /opt/solr/server/solr/${SOLR_CORE}/data/index # Add files -ADD solrconfig.xml \ -https://raw.githubusercontent.com/ckan/ckan/ckan-${CKAN_VERSION}/ckan/config/solr/schema.xml \ -https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/$SOLR_VERSION/solr/server/solr/configsets/basic_configs/conf/currency.xml \ -https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/$SOLR_VERSION/solr/server/solr/configsets/basic_configs/conf/synonyms.txt \ -https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/$SOLR_VERSION/solr/server/solr/configsets/basic_configs/conf/stopwords.txt \ -https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/$SOLR_VERSION/solr/server/solr/configsets/basic_configs/conf/protwords.txt \ -https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/$SOLR_VERSION/solr/server/solr/configsets/data_driven_schema_configs/conf/elevate.xml \ -/opt/solr/server/solr/$SOLR_CORE/conf/ +COPY solrconfig-${CKAN_VERSION}.xml /opt/solr/server/solr/${SOLR_CORE}/conf/solrconfig.xml +ADD https://raw.githubusercontent.com/ckan/ckan/ckan-${CKAN_VERSION}/ckan/config/solr/schema.xml \ +https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/${SOLR_VERSION}/solr/server/solr/configsets/basic_configs/conf/currency.xml \ +https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/${SOLR_VERSION}/solr/server/solr/configsets/basic_configs/conf/synonyms.txt \ +https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/${SOLR_VERSION}/solr/server/solr/configsets/basic_configs/conf/stopwords.txt \ +https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/${SOLR_VERSION}/solr/server/solr/configsets/basic_configs/conf/protwords.txt \ +https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/${SOLR_VERSION}/solr/server/solr/configsets/data_driven_schema_configs/conf/elevate.xml \ +/opt/solr/server/solr/${SOLR_CORE}/conf/ # Create core.properties RUN echo name=${SOLR_CORE} > /opt/solr/server/solr/${SOLR_CORE}/core.properties diff --git a/solr/solrconfig.xml b/solr/solrconfig-2.9.4.xml similarity index 100% rename from solr/solrconfig.xml rename to solr/solrconfig-2.9.4.xml From 60a9d301563d66f9a393734d461a9011939c3e56 Mon Sep 17 00:00:00 2001 From: Brett Date: Fri, 10 Dec 2021 14:35:25 +0100 Subject: [PATCH 04/14] env stuff --- .env | 19 +++++++++++++------ docker-compose.yml | 2 +- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/.env b/.env index a3d9300..864d5d4 100644 --- a/.env +++ b/.env @@ -28,16 +28,23 @@ CKAN_SMTP_PASSWORD=pass CKAN_SMTP_MAIL_FROM=ckan@localhost TZ=UTC -# Other services +# Solr +SOLR_VERSION=6.6.6 CKAN_SOLR_URL=http://solr:8983/solr/ckan -CKAN_REDIS_URL=redis://redis:6379/1 -CKAN_DATAPUSHER_URL=http://datapusher:8801 -CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://ckan:5000 - -# test connections TEST_CKAN_SOLR_URL=http://solr:8983/solr/ckan + +# Redis +REDIS_VERSION=6.0.7 +CKAN_REDIS_URL=redis://redis:6379/1 TEST_CKAN_REDIS_URL=redis://redis:6379/1 +# Datapusher +DATAPUSHER_VERSION=0.0.17 +CKAN_DATAPUSHER_URL=http://datapusher:8800 +CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://ckan:5000 +DATAPUSHER_REWRITE_RESOURCES=True +DATAPUSHER_REWRITE_URL=http://ckan:5000 + # Extensions CKAN__PLUGINS=envvars image_view text_view recline_view datastore datapusher CKAN__HARVEST__MQ__TYPE=redis diff --git a/docker-compose.yml b/docker-compose.yml index b2f5aa1..d140bda 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -60,7 +60,7 @@ services: redis: container_name: redis - image: redis:alpine + image: redis:${REDIS_VERSION} volumes: ckan_storage: From 9d375f727a82e3fd6955c32bdb2ec0f93b22b4ac Mon Sep 17 00:00:00 2001 From: Brett Date: Fri, 10 Dec 2021 14:42:47 +0100 Subject: [PATCH 05/14] Create pruneall.sh --- pruneall.sh | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100755 pruneall.sh diff --git a/pruneall.sh b/pruneall.sh new file mode 100755 index 0000000..d1a8d1f --- /dev/null +++ b/pruneall.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +# Stop all containers +docker kill $(docker ps -q) + +# Remove all stopped containers +docker rm -f $(docker ps -aq) + +# Remove all docker images +docker rmi $(docker images -q) + +# Remove all volumes +docker volume rm $(docker volume ls -q) + +# Remove ckan-docker_default network +docker network rm ckan-docker_default From 212d32ee33c9697538b83c5e8dfb9488ab8cbd0f Mon Sep 17 00:00:00 2001 From: Brett Date: Fri, 10 Dec 2021 14:48:02 +0100 Subject: [PATCH 06/14] update README --- README.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.txt b/README.txt index 90e45fd..37e3dc7 100644 --- a/README.txt +++ b/README.txt @@ -22,3 +22,6 @@ docker-compose -f docker-compose.dev.yml up -d --build docker-compose -f docker-compose.dev.yml build docker-compose -f docker-compose.dev.yml up +ARG should be used for sensitive variables so as to keep the values of these variables +out of the build and hence will not show with an "inspect image" command + From da98af0ddbd1008c2c8fe0bcaf2314d20e518d83 Mon Sep 17 00:00:00 2001 From: Brett Date: Fri, 10 Dec 2021 15:01:16 +0100 Subject: [PATCH 07/14] update spaces in env file --- .env | 2 +- .env.template | 19 +++++++++++++------ docker-compose.yml | 2 +- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/.env b/.env index 864d5d4..29e9bcf 100644 --- a/.env +++ b/.env @@ -46,7 +46,7 @@ DATAPUSHER_REWRITE_RESOURCES=True DATAPUSHER_REWRITE_URL=http://ckan:5000 # Extensions -CKAN__PLUGINS=envvars image_view text_view recline_view datastore datapusher +CKAN__PLUGINS="envvars image_view text_view recline_view datastore datapusher" CKAN__HARVEST__MQ__TYPE=redis CKAN__HARVEST__MQ__HOSTNAME=redis CKAN__HARVEST__MQ__PORT=6379 diff --git a/.env.template b/.env.template index a3d9300..864d5d4 100644 --- a/.env.template +++ b/.env.template @@ -28,16 +28,23 @@ CKAN_SMTP_PASSWORD=pass CKAN_SMTP_MAIL_FROM=ckan@localhost TZ=UTC -# Other services +# Solr +SOLR_VERSION=6.6.6 CKAN_SOLR_URL=http://solr:8983/solr/ckan -CKAN_REDIS_URL=redis://redis:6379/1 -CKAN_DATAPUSHER_URL=http://datapusher:8801 -CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://ckan:5000 - -# test connections TEST_CKAN_SOLR_URL=http://solr:8983/solr/ckan + +# Redis +REDIS_VERSION=6.0.7 +CKAN_REDIS_URL=redis://redis:6379/1 TEST_CKAN_REDIS_URL=redis://redis:6379/1 +# Datapusher +DATAPUSHER_VERSION=0.0.17 +CKAN_DATAPUSHER_URL=http://datapusher:8800 +CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://ckan:5000 +DATAPUSHER_REWRITE_RESOURCES=True +DATAPUSHER_REWRITE_URL=http://ckan:5000 + # Extensions CKAN__PLUGINS=envvars image_view text_view recline_view datastore datapusher CKAN__HARVEST__MQ__TYPE=redis diff --git a/docker-compose.yml b/docker-compose.yml index d140bda..0d8dffa 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -33,7 +33,7 @@ services: datapusher: container_name: datapusher - image: kowhai/datapusher:0.0.17 + image: kowhai/datapusher:${DATAPUSHER_VERSION} ports: - "8800:8800" From 06c14e985ffcb21a384b2feabed844a5a6c1ecc1 Mon Sep 17 00:00:00 2001 From: Brett Date: Thu, 16 Dec 2021 13:59:49 +0100 Subject: [PATCH 08/14] more changes --- README.txt | 34 +++++++++++------------- not-compose.sh | 71 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+), 19 deletions(-) create mode 100755 not-compose.sh diff --git a/README.txt b/README.txt index 37e3dc7..33f2cc2 100644 --- a/README.txt +++ b/README.txt @@ -1,27 +1,23 @@ ### Please note that this re-purposed CKAN Docker repo is a WORK IN PROGRESS ### ### It should not be used to install CKAN via Docker until this page is updated ### -- Maybe use build/up time variables loaded via the .env file, runtime variables loaded via a .ckan-env file - both -located at the root directory -- Look at using ghcr.io (GitHub Packages) to store Docker Images rather than DockerHub -- all username/passwords as environment variables rather than hardcoded -- make sure the "development mode" path is taken cared of with any change -- Create an admin user during the container deployment -- should there be a datapusher image pre-built like ckan? maybe an xloader image? +(From Adria) +Be limited in scope, and act as a base that users can extend to their own needs +Be opinionated, and provide one way to do things +Be automatically tested - -### Difference between ckan-base and ckan-dev ### - -ckan-base -docker-compose up -d --build -docker-compose build -docker-compose up - -ckan-dev -docker-compose -f docker-compose.dev.yml up -d --build -docker-compose -f docker-compose.dev.yml build -docker-compose -f docker-compose.dev.yml up +have an nginx container included? ARG should be used for sensitive variables so as to keep the values of these variables out of the build and hence will not show with an "inspect image" command +Do we pre-build a CKAN image and use that (and extend) as the base image OR + just build it from scratch and while saving to storage what we need to + +Use a bind mount for the config file (ckan.ini) + docker cp ckan:/srv/app/ckan.ini ./ckan.ini + docker run -p : --mount type=bind,source="$(pwd)"/ckan.ini,target=/srv/app/ckan.ini -d + +Maybe include a script to replace docker-compose if required + + diff --git a/not-compose.sh b/not-compose.sh new file mode 100755 index 0000000..791bb28 --- /dev/null +++ b/not-compose.sh @@ -0,0 +1,71 @@ +#!/bin/bash + +# +# Create the network first +# +docker network create ckan-net +# + + +# Redis - Run only (image already built) +# +REDIS_CONTAINERNAME=redis +REDIS_HOSTNAME=redis + +docker run -d -it --net ckan-net --name ${REDIS_CONTAINERNAME} --hostname ${REDIS_HOSTNAME} redis:latest + +# +# Solr - Run/Build +# +SOLR_CONTAINERNAME=solr +SOLR_HOSTNAME=solr +SOLR_IMAGENAME=solr + +cd solr +docker image build -t ${SOLR_IMAGENAME} . +docker run -d -it --net ckan-net --name ${SOLR_CONTAINERNAME} --hostname ${SOLR_HOSTNAME} \ + -v solr_core:/opt/solr/server/solr ${SOLR_IMAGENAME} +cd - + +# +# PostgreSQL (DB) - Run/Build +# +DB_CONTAINERNAME=db +DB_HOSTNAME=db +DB_IMAGENAME=db + +cd postgresql +docker image build -t ${DB_IMAGENAME} . +docker run -d -it --net ckan-net --name ${DB_CONTAINERNAME} --hostname ${DB_HOSTNAME} --env-file=../environ \ + -v pg_data:/var/lib/postgresql/data ${DB_IMAGENAME} +cd - + +# +# CKAN - Run/Build +# +CKAN_CONTAINERNAME=ckan +CKAN_HOSTNAME=ckan +CKAN_IMAGENAME=ckan + +cd ckan +docker image build -t ${CKAN_IMAGENAME} . +docker run -d -it --net ckan-net --name ${CKAN_CONTAINERNAME} --hostname ${CKAN_HOSTNAME} --env-file=../environ \ + -v ckan_storage:/var/lib/ckan ${CKAN_IMAGENAME} +cd - + +# +# Nginx - Run/Build +# +NGINX_CONTAINERNAME=nginx +NGINX_HOSTNAME=nginx +NGINX_IMAGENAME=nginx + +cd nginx +docker image build -t ${NGINX_IMAGENAME} . +docker run -d -it --net ckan-net --name ${NGINX_CONTAINERNAME} --hostname ${NGINX_HOSTNAME} -p 0.0.0.0:80:80 ${NGINX_IMAGENAME} +cd - + +# +# PostgreSQL client +# +docker run -d -it --net ckan-net --name postgres-client --hostname postgres-client -e POSTGRES_PASSWORD=ckan -p 5432:5432 postgres From 6d2c7ca328b2ea308444860501d8d7ff1033ebb4 Mon Sep 17 00:00:00 2001 From: Brett Date: Wed, 5 Jan 2022 11:04:38 +0100 Subject: [PATCH 09/14] More image build changes --- README.txt | 1 - images/README | 13 +++++++++++-- images/ckan-base/2.7/Dockerfile | 8 ++++---- images/ckan-base/2.8/Dockerfile | 8 ++++---- images/ckan-base/2.9/Dockerfile | 18 ++++++++++-------- 5 files changed, 29 insertions(+), 19 deletions(-) diff --git a/README.txt b/README.txt index 33f2cc2..0a61e51 100644 --- a/README.txt +++ b/README.txt @@ -20,4 +20,3 @@ Use a bind mount for the config file (ckan.ini) Maybe include a script to replace docker-compose if required - diff --git a/images/README b/images/README index 44e18b4..ac994c2 100755 --- a/images/README +++ b/images/README @@ -1,4 +1,13 @@ Build the images from this directory using: + cd ckan-base/ + docker build -t ckan/ckan-base:testing-only.2.9 -f 2.9/Dockerfile . + docker build -t ckan/ckan-base:testing-only.2.8 -f 2.8/Dockerfile . + docker build -t ckan/ckan-base:testing-only.2.7 -f 2.7/Dockerfile . - docker build -t openknowledge/ckan-dev:2.7 -f 2.7/Dockerfile . - docker build -t openknowledge/ckan-dev:2.8 -f 2.8/Dockerfile . +Login to Dockerhub +docker login --username=kowhai +(will then ask for a password) + +Push image to (ckan) Dockerhub +docker push ckan/ckan-base:testing-only.2.9 + diff --git a/images/ckan-base/2.7/Dockerfile b/images/ckan-base/2.7/Dockerfile index f3b7099..13e079a 100755 --- a/images/ckan-base/2.7/Dockerfile +++ b/images/ckan-base/2.7/Dockerfile @@ -8,7 +8,7 @@ ENV PIP_SRC=${SRC_DIR} ENV CKAN_STORAGE_PATH=/var/lib/ckan ENV GIT_URL=https://github.com/ckan/ckan.git # CKAN version to build -ENV GIT_BRANCH=ckan-2.7.8 +ENV GIT_BRANCH=ckan-2.7.12 # Customize these on the .env file if needed ENV CKAN_SITE_URL=http://localhost:5000 ENV CKAN__PLUGINS image_view text_view recline_view datastore datapusher envvars @@ -46,11 +46,11 @@ RUN apk add --no-cache tzdata \ python-dev \ libxml2-dev \ libxslt-dev \ - linux-headers && \ + linux-headers # Create SRC_DIR - mkdir -p ${SRC_DIR} && \ + RUN mkdir -p ${SRC_DIR} && \ # Install pip, supervisord and uwsgi - curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/2.7/get-pip.py && \ + curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/pip/2.7/get-pip.py && \ python ${SRC_DIR}/get-pip.py 'pip==20.3.3' && \ pip install supervisor && \ mkdir /etc/supervisord.d && \ diff --git a/images/ckan-base/2.8/Dockerfile b/images/ckan-base/2.8/Dockerfile index 48847e7..e045408 100755 --- a/images/ckan-base/2.8/Dockerfile +++ b/images/ckan-base/2.8/Dockerfile @@ -8,7 +8,7 @@ ENV PIP_SRC=${SRC_DIR} ENV CKAN_STORAGE_PATH=/var/lib/ckan ENV GIT_URL=https://github.com/ckan/ckan.git # CKAN version to build -ENV GIT_BRANCH=ckan-2.8.5 +ENV GIT_BRANCH=ckan-2.8.9 # Customize these on the .env file if needed ENV CKAN_SITE_URL=http://localhost:5000 ENV CKAN__PLUGINS image_view text_view recline_view datastore datapusher envvars @@ -46,11 +46,11 @@ RUN apk add --no-cache tzdata \ python-dev \ libxml2-dev \ libxslt-dev \ - linux-headers && \ + linux-headers # Create SRC_DIR - mkdir -p ${SRC_DIR} && \ + RUN mkdir -p ${SRC_DIR} && \ # Install pip, supervisord and uwsgi - curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/2.7/get-pip.py && \ + curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/pip/2.7/get-pip.py && \ python ${SRC_DIR}/get-pip.py 'pip==20.3.3' && \ pip install supervisor && \ mkdir /etc/supervisord.d && \ diff --git a/images/ckan-base/2.9/Dockerfile b/images/ckan-base/2.9/Dockerfile index d2665cd..4cda64c 100755 --- a/images/ckan-base/2.9/Dockerfile +++ b/images/ckan-base/2.9/Dockerfile @@ -1,4 +1,4 @@ -FROM alpine:3.7 +FROM alpine:3.13 # Internal environment variables ENV APP_DIR=/srv/app @@ -8,7 +8,7 @@ ENV PIP_SRC=${SRC_DIR} ENV CKAN_STORAGE_PATH=/var/lib/ckan ENV GIT_URL=https://github.com/ckan/ckan.git # CKAN version to build -ENV GIT_BRANCH=ckan-2.9.0 +ENV GIT_BRANCH=ckan-2.9.4 # Customize these on the .env file if needed ENV CKAN_SITE_URL=http://localhost:5000 ENV CKAN__PLUGINS image_view text_view recline_view datastore datapusher envvars @@ -46,12 +46,13 @@ RUN apk add --no-cache tzdata \ py3-virtualenv \ libxml2-dev \ libxslt-dev \ - linux-headers && \ - # Create SRC_DIR - mkdir -p ${SRC_DIR} && \ + linux-headers + +# Create SRC_DIR and install pip +RUN mkdir -p ${SRC_DIR} && \ # Install pip - curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/2.7/get-pip.py && \ - python3 ${SRC_DIR}/get-pip.py 'pip==20.3.3' && \ + curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/get-pip.py && \ + python3 ${SRC_DIR}/get-pip.py && \ rm -rf ${SRC_DIR}/get-pip.py # Set up Python3 virtual environment @@ -67,7 +68,8 @@ RUN pip3 install -e git+${GIT_URL}@${GIT_BRANCH}#egg=ckan && \ pip3 install uwsgi && \ cd ${SRC_DIR}/ckan && \ cp who.ini ${APP_DIR} && \ - pip install --no-binary :all: -r requirements.txt && \ + pip3 install -r requirement-setuptools.txt && \ + pip3 install --no-binary :all: -r requirements.txt && \ # Install CKAN envvars to support loading config from environment variables pip3 install -e git+https://github.com/okfn/ckanext-envvars.git#egg=ckanext-envvars && \ # Create and update CKAN config From 691aa4604a16088d12341e7a35373cc95fe4d097 Mon Sep 17 00:00:00 2001 From: Brett Date: Wed, 5 Jan 2022 14:30:19 +0100 Subject: [PATCH 10/14] More README updates, environ file, local ckan.ini file --- README.txt | 7 +- ckan.ini | 252 +++++++++++++++++++++++++++++++++++++++++++++++++++++ environ | 34 ++++++++ 3 files changed, 290 insertions(+), 3 deletions(-) create mode 100644 ckan.ini create mode 100644 environ diff --git a/README.txt b/README.txt index 0a61e51..7396a20 100644 --- a/README.txt +++ b/README.txt @@ -1,12 +1,14 @@ ### Please note that this re-purposed CKAN Docker repo is a WORK IN PROGRESS ### ### It should not be used to install CKAN via Docker until this page is updated ### +------------------------------------- (From Adria) Be limited in scope, and act as a base that users can extend to their own needs Be opinionated, and provide one way to do things Be automatically tested +------------------------------------- -have an nginx container included? +should we have an nginx container included? maybe just instructions on how to include one ARG should be used for sensitive variables so as to keep the values of these variables out of the build and hence will not show with an "inspect image" command @@ -16,7 +18,6 @@ Do we pre-build a CKAN image and use that (and extend) as the base image OR Use a bind mount for the config file (ckan.ini) docker cp ckan:/srv/app/ckan.ini ./ckan.ini - docker run -p : --mount type=bind,source="$(pwd)"/ckan.ini,target=/srv/app/ckan.ini -d + docker run -p 0.0.0.0:5000:5000 --net ckan-docker_default --hostname ckan --name ckan --env-file=./environ --mount type=bind,source=$(pwd)/ckan.ini,target=/srv/app/ckan.ini -d ckan-docker_ckan Maybe include a script to replace docker-compose if required - diff --git a/ckan.ini b/ckan.ini new file mode 100644 index 0000000..61e85d2 --- /dev/null +++ b/ckan.ini @@ -0,0 +1,252 @@ +# +# CKAN - Pylons configuration +# +# These are some of the configuration options available for your CKAN +# instance. Check the documentation in 'doc/configuration.rst' or at the +# following URL for a description of what they do and the full list of +# available options: +# +# http://docs.ckan.org/en/latest/maintaining/configuration.html +# +# The %(here)s variable will be replaced with the parent directory of this file +# + +[DEFAULT] + +# WARNING: *THIS SETTING MUST BE SET TO FALSE ON A PUBLIC ENVIRONMENT* +# With debug mode enabled, a visitor to your site could execute malicious commands. +debug = false + +[app:main] +use = egg:ckan + +## Development settings +ckan.devserver.host = localhost +ckan.devserver.port = 5000 + + +## Session settings +cache_dir = /tmp/%(ckan.site_id)s/ +beaker.session.key = ckan + +# This is the secret token that the beaker library uses to hash the cookie sent +# to the client. `ckan generate config` generates a unique value for this each +# time it generates a config file. +beaker.session.secret = EHU6ef6rWzspiSQutl2F4G28n + +# `ckan generate config` generates a unique value for this each time it generates +# a config file. +app_instance_uuid = 52ba6253-cf07-4f0b-ba3f-1c685b4a708c + +# repoze.who config +who.config_file = %(here)s/who.ini +who.log_level = warning +who.log_file = %(cache_dir)s/who_log.ini +# Session timeout (user logged out after period of inactivity, in seconds). +# Inactive by default, so the session doesn't expire. +# who.timeout = 86400 + +## Database Settings +sqlalchemy.url = postgresql://ckan_default:ManHat212@localhost/ckan_default + +#ckan.datastore.write_url = postgresql://ckan_default:pass@localhost/datastore_default +#ckan.datastore.read_url = postgresql://datastore_default:pass@localhost/datastore_default + +# PostgreSQL' full-text search parameters +ckan.datastore.default_fts_lang = english +ckan.datastore.default_fts_index_method = gist + + +## Site Settings + +ckan.site_url = +#ckan.use_pylons_response_cleanup_middleware = true + +## Authorization Settings + +ckan.auth.anon_create_dataset = false +ckan.auth.create_unowned_dataset = false +ckan.auth.create_dataset_if_not_in_organization = false +ckan.auth.user_create_groups = false +ckan.auth.user_create_organizations = false +ckan.auth.user_delete_groups = true +ckan.auth.user_delete_organizations = true +ckan.auth.create_user_via_api = false +ckan.auth.create_user_via_web = true +ckan.auth.roles_that_cascade_to_sub_groups = admin +ckan.auth.public_user_details = true +ckan.auth.public_activity_stream_detail = true +ckan.auth.allow_dataset_collaborators = false +ckan.auth.create_default_api_keys = false + +## API Token Settings +api_token.nbytes = 60 +api_token.jwt.encode.secret = string:EHU6ef6rWzspiSQutl2F4G28n +api_token.jwt.decode.secret = string:EHU6ef6rWzspiSQutl2F4G28n +api_token.jwt.algorithm = HS256 + +## API Token: expire_api_token plugin +expire_api_token.default_lifetime = 3600 + +## Search Settings + +ckan.site_id = default +#solr_url = http://127.0.0.1:8983/solr + + +## Redis Settings + +# URL to your Redis instance, including the database to be used. +#ckan.redis.url = redis://localhost:6379/0 + + +## CORS Settings + +# If cors.origin_allow_all is true, all origins are allowed. +# If false, the cors.origin_whitelist is used. +# ckan.cors.origin_allow_all = true +# cors.origin_whitelist is a space separated list of allowed domains. +# ckan.cors.origin_whitelist = http://example1.com http://example2.com + + +## Plugins Settings + +# Note: Add ``datastore`` to enable the CKAN DataStore +# Add ``datapusher`` to enable DataPusher +# Add ``resource_proxy`` to enable resorce proxying and get around the +# same origin policy +ckan.plugins = image_view text_view recline_view datastore datapusher envvars + +# Define which views should be created by default +# (plugins must be loaded in ckan.plugins) +ckan.views.default_views = image_view text_view recline_view + +# Customize which text formats the text_view plugin will show +#ckan.preview.json_formats = json +#ckan.preview.xml_formats = xml rdf rdf+xml owl+xml atom rss +#ckan.preview.text_formats = text plain text/plain + +# Customize which image formats the image_view plugin will show +#ckan.preview.image_formats = png jpeg jpg gif + +## Front-End Settings + +ckan.site_title = CKAN +ckan.site_logo = /base/images/ckan-logo.png +ckan.site_description = +ckan.favicon = /base/images/ckan.ico +ckan.gravatar_default = identicon +ckan.preview.direct = png jpg gif +ckan.preview.loadable = html htm rdf+xml owl+xml xml n3 n-triples turtle plain atom csv tsv rss txt json +ckan.display_timezone = server + +# package_hide_extras = for_search_index_only +#package_edit_return_url = http://another.frontend/dataset/ +#package_new_return_url = http://another.frontend/dataset/ +#ckan.recaptcha.publickey = +#ckan.recaptcha.privatekey = +#licenses_group_url = http://licenses.opendefinition.org/licenses/groups/ckan.json +# ckan.template_footer_end = + + +## Internationalisation Settings +ckan.locale_default = en +ckan.locale_order = en pt_BR ja it cs_CZ ca es fr el sv sr sr@latin no sk fi ru de pl nl bg ko_KR hu sa sl lv +ckan.locales_offered = +ckan.locales_filtered_out = en_GB + +## Feeds Settings + +ckan.feeds.authority_name = +ckan.feeds.date = +ckan.feeds.author_name = +ckan.feeds.author_link = + +## Storage Settings + +#ckan.storage_path = /var/lib/ckan +#ckan.max_resource_size = 10 +#ckan.max_image_size = 2 + +## Webassets Settings +#ckan.webassets.use_x_sendfile = false +#ckan.webassets.path = /var/lib/ckan/webassets + + +## Datapusher settings + +# Make sure you have set up the DataStore + +#ckan.datapusher.formats = csv xls xlsx tsv application/csv application/vnd.ms-excel application/vnd.openxmlformats-officedocument.spreadsheetml.sheet +#ckan.datapusher.url = http://127.0.0.1:8800/ +#ckan.datapusher.assume_task_stale_after = 3600 + +# Resource Proxy settings +# Preview size limit, default: 1MB +#ckan.resource_proxy.max_file_size = 1048576 +# Size of chunks to read/write. +#ckan.resource_proxy.chunk_size = 4096 + +## Activity Streams Settings + +#ckan.activity_streams_enabled = true +#ckan.activity_list_limit = 31 +#ckan.activity_streams_email_notifications = true +#ckan.email_notifications_since = 2 days +ckan.hide_activity_from_users = %(ckan.site_id)s + + +## Email settings + +#email_to = errors@example.com +#error_email_from = ckan-errors@example.com +#smtp.server = localhost +#smtp.starttls = False +#smtp.user = username@example.com +#smtp.password = your_password +#smtp.mail_from = +#smtp.reply_to = + +## Background Job Settings +ckan.jobs.timeout = 180 + +## Logging configuration +[loggers] +keys = root, ckan, ckanext, werkzeug + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARNING +handlers = console + +[logger_werkzeug] +level = WARNING +handlers = console +qualname = werkzeug +propagate = 0 + +[logger_ckan] +level = INFO +handlers = console +qualname = ckan +propagate = 0 + +[logger_ckanext] +level = DEBUG +handlers = console +qualname = ckanext +propagate = 0 + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(asctime)s %(levelname)-5.5s [%(name)s] %(message)s diff --git a/environ b/environ new file mode 100644 index 0000000..df4e7f3 --- /dev/null +++ b/environ @@ -0,0 +1,34 @@ +REDIS_CONTAINERNAME=redis +REDIS_HOSTNAME=redis + +SOLR_CONTAINERNAME=solr +SOLR_HOSTNAME=solr +SOLR_IMAGENAME=solr + +DB_CONTAINERNAME=db +DB_HOSTNAME=db +DB_IMAGENAME=db +POSTGRES_PASSWORD=ckan +DS_RO_PASS=datastore +PGDATA=/var/lib/postgresql/data/db + +CKAN_CONTAINERNAME=ckan +CKAN_HOSTNAME=ckan +CKAN_IMAGENAME=ckan + +CKAN_SQLALCHEMY_URL=postgresql://ckan:ckan@db/ckan +CKAN_DATASTORE_WRITE_URL=postgresql://ckan:ckan@db/datastore +CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:datastore@db/datastore +CKAN_SOLR_URL=http://solr:8983/solr/ckan +CKAN_REDIS_URL=redis://redis:6379/1 +CKAN_SITE_URL=http://localhost +CKAN_MAX_UPLOAD_SIZE_MB=32 +POSTGRES_PASSWORD=ckan +DS_RO_PASS=datastore +CKAN_PORT=5000 +CKAN_DATAPUSHER_URL=http://datapusher:8800 +CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://ckan:5000 + +NGINX_CONTAINERNAME=nginx +NGINX_HOSTNAME=nginx +NGINX_IMAGENAME=nginx From 86be0bcadfafbad4a2fdc1dd7992e3ca04523939 Mon Sep 17 00:00:00 2001 From: Brett Date: Fri, 7 Jan 2022 15:16:52 +0100 Subject: [PATCH 11/14] More bind mounts --- README.txt | 10 +- local/ckan.ini | 252 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 260 insertions(+), 2 deletions(-) create mode 100644 local/ckan.ini diff --git a/README.txt b/README.txt index 7396a20..c0b737d 100644 --- a/README.txt +++ b/README.txt @@ -17,7 +17,13 @@ Do we pre-build a CKAN image and use that (and extend) as the base image OR just build it from scratch and while saving to storage what we need to Use a bind mount for the config file (ckan.ini) - docker cp ckan:/srv/app/ckan.ini ./ckan.ini - docker run -p 0.0.0.0:5000:5000 --net ckan-docker_default --hostname ckan --name ckan --env-file=./environ --mount type=bind,source=$(pwd)/ckan.ini,target=/srv/app/ckan.ini -d ckan-docker_ckan + mkdir local + docker cp ckan:/srv/app/ckan.ini ./local/ckan.ini + docker cp ckan:/srv/app/src ./local/src + stop/remove ckan container + docker run -p 0.0.0.0:5000:5000 --net ckan-docker_default --hostname ckan --name ckan \ + --env-file=./environ --mount type=bind,source=$(pwd)/local/ckan.ini,target=/srv/app/ckan.ini \ + --mount type=bind,source=$(pwd)/local/src,target=/srv/app/src \ + -d ckan-docker_ckan Maybe include a script to replace docker-compose if required diff --git a/local/ckan.ini b/local/ckan.ini new file mode 100644 index 0000000..90721b8 --- /dev/null +++ b/local/ckan.ini @@ -0,0 +1,252 @@ +# +# CKAN - Pylons configuration +# +# These are some of the configuration options available for your CKAN +# instance. Check the documentation in 'doc/configuration.rst' or at the +# following URL for a description of what they do and the full list of +# available options: +# +# http://docs.ckan.org/en/latest/maintaining/configuration.html +# +# The %(here)s variable will be replaced with the parent directory of this file +# + +[DEFAULT] + +# WARNING: *THIS SETTING MUST BE SET TO FALSE ON A PUBLIC ENVIRONMENT* +# With debug mode enabled, a visitor to your site could execute malicious commands. +debug = false + +[app:main] +use = egg:ckan + +## Development settings +ckan.devserver.host = localhost +ckan.devserver.port = 5000 + + +## Session settings +cache_dir = /tmp/%(ckan.site_id)s/ +beaker.session.key = ckan + +# This is the secret token that the beaker library uses to hash the cookie sent +# to the client. `ckan generate config` generates a unique value for this each +# time it generates a config file. +beaker.session.secret = EHU6ef6rWzspiSQutl2F4G28n + +# `ckan generate config` generates a unique value for this each time it generates +# a config file. +app_instance_uuid = 52ba6253-cf07-4f0b-ba3f-1c685b4a708c + +# repoze.who config +who.config_file = %(here)s/who.ini +who.log_level = warning +who.log_file = %(cache_dir)s/who_log.ini +# Session timeout (user logged out after period of inactivity, in seconds). +# Inactive by default, so the session doesn't expire. +# who.timeout = 86400 + +## Database Settings +sqlalchemy.url = postgresql://ckan_default:pass@localhost/ckan_default + +#ckan.datastore.write_url = postgresql://ckan_default:pass@localhost/datastore_default +#ckan.datastore.read_url = postgresql://datastore_default:pass@localhost/datastore_default + +# PostgreSQL' full-text search parameters +ckan.datastore.default_fts_lang = english +ckan.datastore.default_fts_index_method = gist + + +## Site Settings + +ckan.site_url = +#ckan.use_pylons_response_cleanup_middleware = true + +## Authorization Settings + +ckan.auth.anon_create_dataset = false +ckan.auth.create_unowned_dataset = false +ckan.auth.create_dataset_if_not_in_organization = false +ckan.auth.user_create_groups = false +ckan.auth.user_create_organizations = false +ckan.auth.user_delete_groups = true +ckan.auth.user_delete_organizations = true +ckan.auth.create_user_via_api = false +ckan.auth.create_user_via_web = true +ckan.auth.roles_that_cascade_to_sub_groups = admin +ckan.auth.public_user_details = true +ckan.auth.public_activity_stream_detail = true +ckan.auth.allow_dataset_collaborators = false +ckan.auth.create_default_api_keys = false + +## API Token Settings +api_token.nbytes = 60 +api_token.jwt.encode.secret = string:EHU6ef6rWzspiSQutl2F4G28n +api_token.jwt.decode.secret = string:EHU6ef6rWzspiSQutl2F4G28n +api_token.jwt.algorithm = HS256 + +## API Token: expire_api_token plugin +expire_api_token.default_lifetime = 3600 + +## Search Settings + +ckan.site_id = default +#solr_url = http://127.0.0.1:8983/solr + + +## Redis Settings + +# URL to your Redis instance, including the database to be used. +#ckan.redis.url = redis://localhost:6379/0 + + +## CORS Settings + +# If cors.origin_allow_all is true, all origins are allowed. +# If false, the cors.origin_whitelist is used. +# ckan.cors.origin_allow_all = true +# cors.origin_whitelist is a space separated list of allowed domains. +# ckan.cors.origin_whitelist = http://example1.com http://example2.com + + +## Plugins Settings + +# Note: Add ``datastore`` to enable the CKAN DataStore +# Add ``datapusher`` to enable DataPusher +# Add ``resource_proxy`` to enable resorce proxying and get around the +# same origin policy +ckan.plugins = image_view text_view recline_view datastore datapusher envvars + +# Define which views should be created by default +# (plugins must be loaded in ckan.plugins) +ckan.views.default_views = image_view text_view recline_view + +# Customize which text formats the text_view plugin will show +#ckan.preview.json_formats = json +#ckan.preview.xml_formats = xml rdf rdf+xml owl+xml atom rss +#ckan.preview.text_formats = text plain text/plain + +# Customize which image formats the image_view plugin will show +#ckan.preview.image_formats = png jpeg jpg gif + +## Front-End Settings + +ckan.site_title = CKAN +ckan.site_logo = /base/images/ckan-logo.png +ckan.site_description = +ckan.favicon = /base/images/ckan.ico +ckan.gravatar_default = identicon +ckan.preview.direct = png jpg gif +ckan.preview.loadable = html htm rdf+xml owl+xml xml n3 n-triples turtle plain atom csv tsv rss txt json +ckan.display_timezone = server + +# package_hide_extras = for_search_index_only +#package_edit_return_url = http://another.frontend/dataset/ +#package_new_return_url = http://another.frontend/dataset/ +#ckan.recaptcha.publickey = +#ckan.recaptcha.privatekey = +#licenses_group_url = http://licenses.opendefinition.org/licenses/groups/ckan.json +# ckan.template_footer_end = + + +## Internationalisation Settings +ckan.locale_default = en +ckan.locale_order = en pt_BR ja it cs_CZ ca es fr el sv sr sr@latin no sk fi ru de pl nl bg ko_KR hu sa sl lv +ckan.locales_offered = +ckan.locales_filtered_out = en_GB + +## Feeds Settings + +ckan.feeds.authority_name = +ckan.feeds.date = +ckan.feeds.author_name = +ckan.feeds.author_link = + +## Storage Settings + +#ckan.storage_path = /var/lib/ckan +#ckan.max_resource_size = 10 +#ckan.max_image_size = 2 + +## Webassets Settings +#ckan.webassets.use_x_sendfile = false +#ckan.webassets.path = /var/lib/ckan/webassets + + +## Datapusher settings + +# Make sure you have set up the DataStore + +#ckan.datapusher.formats = csv xls xlsx tsv application/csv application/vnd.ms-excel application/vnd.openxmlformats-officedocument.spreadsheetml.sheet +#ckan.datapusher.url = http://127.0.0.1:8800/ +#ckan.datapusher.assume_task_stale_after = 3600 + +# Resource Proxy settings +# Preview size limit, default: 1MB +#ckan.resource_proxy.max_file_size = 1048576 +# Size of chunks to read/write. +#ckan.resource_proxy.chunk_size = 4096 + +## Activity Streams Settings + +#ckan.activity_streams_enabled = true +#ckan.activity_list_limit = 31 +#ckan.activity_streams_email_notifications = true +#ckan.email_notifications_since = 2 days +ckan.hide_activity_from_users = %(ckan.site_id)s + + +## Email settings + +#email_to = errors@example.com +#error_email_from = ckan-errors@example.com +#smtp.server = localhost +#smtp.starttls = False +#smtp.user = username@example.com +#smtp.password = your_password +#smtp.mail_from = +#smtp.reply_to = + +## Background Job Settings +ckan.jobs.timeout = 180 + +## Logging configuration +[loggers] +keys = root, ckan, ckanext, werkzeug + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARNING +handlers = console + +[logger_werkzeug] +level = WARNING +handlers = console +qualname = werkzeug +propagate = 0 + +[logger_ckan] +level = INFO +handlers = console +qualname = ckan +propagate = 0 + +[logger_ckanext] +level = DEBUG +handlers = console +qualname = ckanext +propagate = 0 + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(asctime)s %(levelname)-5.5s [%(name)s] %(message)s From 9607fb7bbc7a3329b801a73438fc34ad2fb0f432 Mon Sep 17 00:00:00 2001 From: Brett Date: Fri, 7 Jan 2022 15:20:06 +0100 Subject: [PATCH 12/14] Update .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 1d88d85..06c7c30 100755 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ _service-provider/* _solr/schema.xml _src/* +local/* From 879e114de6f8132169172c762a710ceb69b9f777 Mon Sep 17 00:00:00 2001 From: Brett Date: Tue, 11 Jan 2022 13:39:39 +0100 Subject: [PATCH 13/14] Major setup dir changes --- README.txt | 18 +- ckan.ini | 252 ------------------ images/ckan-base/2.7/Dockerfile | 8 +- images/ckan-base/2.7/setup/ckan-uwsgi.ini | 15 ++ .../2.7/setup/supervisor.worker.conf | 13 - images/ckan-base/2.7/setup/wsgi.py | 9 + images/ckan-base/2.8/Dockerfile | 8 +- images/ckan-base/2.8/setup/ckan-uwsgi.ini | 15 ++ .../2.8/setup/supervisor.worker.conf | 13 - images/ckan-base/2.8/setup/wsgi.py | 9 + images/ckan-base/2.9/Dockerfile | 8 +- .../2.9/setup/supervisor.worker.conf | 12 - 12 files changed, 73 insertions(+), 307 deletions(-) delete mode 100644 ckan.ini create mode 100644 images/ckan-base/2.7/setup/ckan-uwsgi.ini delete mode 100755 images/ckan-base/2.7/setup/supervisor.worker.conf create mode 100644 images/ckan-base/2.7/setup/wsgi.py create mode 100644 images/ckan-base/2.8/setup/ckan-uwsgi.ini delete mode 100755 images/ckan-base/2.8/setup/supervisor.worker.conf create mode 100644 images/ckan-base/2.8/setup/wsgi.py delete mode 100644 images/ckan-base/2.9/setup/supervisor.worker.conf diff --git a/README.txt b/README.txt index c0b737d..a769c0e 100644 --- a/README.txt +++ b/README.txt @@ -16,14 +16,16 @@ out of the build and hence will not show with an "inspect image" command Do we pre-build a CKAN image and use that (and extend) as the base image OR just build it from scratch and while saving to storage what we need to +# docker-compose up -d --build +# mkdir local +# docker cp ckan:/srv/app/ckan.ini ./local/ckan.ini +# docker cp ckan:/srv/app/src ./local/src +Stop/Remove ckan container +Start ckan container as follows Use a bind mount for the config file (ckan.ini) - mkdir local - docker cp ckan:/srv/app/ckan.ini ./local/ckan.ini - docker cp ckan:/srv/app/src ./local/src - stop/remove ckan container - docker run -p 0.0.0.0:5000:5000 --net ckan-docker_default --hostname ckan --name ckan \ - --env-file=./environ --mount type=bind,source=$(pwd)/local/ckan.ini,target=/srv/app/ckan.ini \ - --mount type=bind,source=$(pwd)/local/src,target=/srv/app/src \ - -d ckan-docker_ckan +# docker run -p 0.0.0.0:5000:5000 --net ckan-docker_default --hostname ckan --name ckan \ + --env-file=./environ --mount type=bind,source=$(pwd)/local/ckan.ini,target=/srv/app/ckan.ini \ + --mount type=bind,source=$(pwd)/local/src,target=/srv/app/src \ + -d ckan-docker_ckan Maybe include a script to replace docker-compose if required diff --git a/ckan.ini b/ckan.ini deleted file mode 100644 index 61e85d2..0000000 --- a/ckan.ini +++ /dev/null @@ -1,252 +0,0 @@ -# -# CKAN - Pylons configuration -# -# These are some of the configuration options available for your CKAN -# instance. Check the documentation in 'doc/configuration.rst' or at the -# following URL for a description of what they do and the full list of -# available options: -# -# http://docs.ckan.org/en/latest/maintaining/configuration.html -# -# The %(here)s variable will be replaced with the parent directory of this file -# - -[DEFAULT] - -# WARNING: *THIS SETTING MUST BE SET TO FALSE ON A PUBLIC ENVIRONMENT* -# With debug mode enabled, a visitor to your site could execute malicious commands. -debug = false - -[app:main] -use = egg:ckan - -## Development settings -ckan.devserver.host = localhost -ckan.devserver.port = 5000 - - -## Session settings -cache_dir = /tmp/%(ckan.site_id)s/ -beaker.session.key = ckan - -# This is the secret token that the beaker library uses to hash the cookie sent -# to the client. `ckan generate config` generates a unique value for this each -# time it generates a config file. -beaker.session.secret = EHU6ef6rWzspiSQutl2F4G28n - -# `ckan generate config` generates a unique value for this each time it generates -# a config file. -app_instance_uuid = 52ba6253-cf07-4f0b-ba3f-1c685b4a708c - -# repoze.who config -who.config_file = %(here)s/who.ini -who.log_level = warning -who.log_file = %(cache_dir)s/who_log.ini -# Session timeout (user logged out after period of inactivity, in seconds). -# Inactive by default, so the session doesn't expire. -# who.timeout = 86400 - -## Database Settings -sqlalchemy.url = postgresql://ckan_default:ManHat212@localhost/ckan_default - -#ckan.datastore.write_url = postgresql://ckan_default:pass@localhost/datastore_default -#ckan.datastore.read_url = postgresql://datastore_default:pass@localhost/datastore_default - -# PostgreSQL' full-text search parameters -ckan.datastore.default_fts_lang = english -ckan.datastore.default_fts_index_method = gist - - -## Site Settings - -ckan.site_url = -#ckan.use_pylons_response_cleanup_middleware = true - -## Authorization Settings - -ckan.auth.anon_create_dataset = false -ckan.auth.create_unowned_dataset = false -ckan.auth.create_dataset_if_not_in_organization = false -ckan.auth.user_create_groups = false -ckan.auth.user_create_organizations = false -ckan.auth.user_delete_groups = true -ckan.auth.user_delete_organizations = true -ckan.auth.create_user_via_api = false -ckan.auth.create_user_via_web = true -ckan.auth.roles_that_cascade_to_sub_groups = admin -ckan.auth.public_user_details = true -ckan.auth.public_activity_stream_detail = true -ckan.auth.allow_dataset_collaborators = false -ckan.auth.create_default_api_keys = false - -## API Token Settings -api_token.nbytes = 60 -api_token.jwt.encode.secret = string:EHU6ef6rWzspiSQutl2F4G28n -api_token.jwt.decode.secret = string:EHU6ef6rWzspiSQutl2F4G28n -api_token.jwt.algorithm = HS256 - -## API Token: expire_api_token plugin -expire_api_token.default_lifetime = 3600 - -## Search Settings - -ckan.site_id = default -#solr_url = http://127.0.0.1:8983/solr - - -## Redis Settings - -# URL to your Redis instance, including the database to be used. -#ckan.redis.url = redis://localhost:6379/0 - - -## CORS Settings - -# If cors.origin_allow_all is true, all origins are allowed. -# If false, the cors.origin_whitelist is used. -# ckan.cors.origin_allow_all = true -# cors.origin_whitelist is a space separated list of allowed domains. -# ckan.cors.origin_whitelist = http://example1.com http://example2.com - - -## Plugins Settings - -# Note: Add ``datastore`` to enable the CKAN DataStore -# Add ``datapusher`` to enable DataPusher -# Add ``resource_proxy`` to enable resorce proxying and get around the -# same origin policy -ckan.plugins = image_view text_view recline_view datastore datapusher envvars - -# Define which views should be created by default -# (plugins must be loaded in ckan.plugins) -ckan.views.default_views = image_view text_view recline_view - -# Customize which text formats the text_view plugin will show -#ckan.preview.json_formats = json -#ckan.preview.xml_formats = xml rdf rdf+xml owl+xml atom rss -#ckan.preview.text_formats = text plain text/plain - -# Customize which image formats the image_view plugin will show -#ckan.preview.image_formats = png jpeg jpg gif - -## Front-End Settings - -ckan.site_title = CKAN -ckan.site_logo = /base/images/ckan-logo.png -ckan.site_description = -ckan.favicon = /base/images/ckan.ico -ckan.gravatar_default = identicon -ckan.preview.direct = png jpg gif -ckan.preview.loadable = html htm rdf+xml owl+xml xml n3 n-triples turtle plain atom csv tsv rss txt json -ckan.display_timezone = server - -# package_hide_extras = for_search_index_only -#package_edit_return_url = http://another.frontend/dataset/ -#package_new_return_url = http://another.frontend/dataset/ -#ckan.recaptcha.publickey = -#ckan.recaptcha.privatekey = -#licenses_group_url = http://licenses.opendefinition.org/licenses/groups/ckan.json -# ckan.template_footer_end = - - -## Internationalisation Settings -ckan.locale_default = en -ckan.locale_order = en pt_BR ja it cs_CZ ca es fr el sv sr sr@latin no sk fi ru de pl nl bg ko_KR hu sa sl lv -ckan.locales_offered = -ckan.locales_filtered_out = en_GB - -## Feeds Settings - -ckan.feeds.authority_name = -ckan.feeds.date = -ckan.feeds.author_name = -ckan.feeds.author_link = - -## Storage Settings - -#ckan.storage_path = /var/lib/ckan -#ckan.max_resource_size = 10 -#ckan.max_image_size = 2 - -## Webassets Settings -#ckan.webassets.use_x_sendfile = false -#ckan.webassets.path = /var/lib/ckan/webassets - - -## Datapusher settings - -# Make sure you have set up the DataStore - -#ckan.datapusher.formats = csv xls xlsx tsv application/csv application/vnd.ms-excel application/vnd.openxmlformats-officedocument.spreadsheetml.sheet -#ckan.datapusher.url = http://127.0.0.1:8800/ -#ckan.datapusher.assume_task_stale_after = 3600 - -# Resource Proxy settings -# Preview size limit, default: 1MB -#ckan.resource_proxy.max_file_size = 1048576 -# Size of chunks to read/write. -#ckan.resource_proxy.chunk_size = 4096 - -## Activity Streams Settings - -#ckan.activity_streams_enabled = true -#ckan.activity_list_limit = 31 -#ckan.activity_streams_email_notifications = true -#ckan.email_notifications_since = 2 days -ckan.hide_activity_from_users = %(ckan.site_id)s - - -## Email settings - -#email_to = errors@example.com -#error_email_from = ckan-errors@example.com -#smtp.server = localhost -#smtp.starttls = False -#smtp.user = username@example.com -#smtp.password = your_password -#smtp.mail_from = -#smtp.reply_to = - -## Background Job Settings -ckan.jobs.timeout = 180 - -## Logging configuration -[loggers] -keys = root, ckan, ckanext, werkzeug - -[handlers] -keys = console - -[formatters] -keys = generic - -[logger_root] -level = WARNING -handlers = console - -[logger_werkzeug] -level = WARNING -handlers = console -qualname = werkzeug -propagate = 0 - -[logger_ckan] -level = INFO -handlers = console -qualname = ckan -propagate = 0 - -[logger_ckanext] -level = DEBUG -handlers = console -qualname = ckanext -propagate = 0 - -[handler_console] -class = StreamHandler -args = (sys.stderr,) -level = NOTSET -formatter = generic - -[formatter_generic] -format = %(asctime)s %(levelname)-5.5s [%(name)s] %(message)s diff --git a/images/ckan-base/2.7/Dockerfile b/images/ckan-base/2.7/Dockerfile index 13e079a..83b33c9 100755 --- a/images/ckan-base/2.7/Dockerfile +++ b/images/ckan-base/2.7/Dockerfile @@ -81,9 +81,11 @@ RUN addgroup -g 92 -S ckan && \ RUN mkdir -p $CKAN_STORAGE_PATH && \ chown -R ckan:ckan $CKAN_STORAGE_PATH -COPY 2.7/setup ${APP_DIR} -COPY 2.7/setup/supervisor.worker.conf /etc/supervisord.d/worker.conf -COPY 2.7/setup/uwsgi.conf /srv/app/uwsgi.conf +COPY 2.7/setup/ckan-uwsgi.ini ${APP_DIR} +COPY 2.7/setup/wsgi.py ${APP_DIR} +COPY 2.7/setup/prerun.py ${APP_DIR} +COPY 2.7/setup/start_ckan.sh ${APP_DIR} +COPY 2.7/setup/supervisord.conf /etc/supervisord.conf # Create entrypoint directory for children image scripts ONBUILD RUN mkdir /docker-entrypoint.d diff --git a/images/ckan-base/2.7/setup/ckan-uwsgi.ini b/images/ckan-base/2.7/setup/ckan-uwsgi.ini new file mode 100644 index 0000000..2361f36 --- /dev/null +++ b/images/ckan-base/2.7/setup/ckan-uwsgi.ini @@ -0,0 +1,15 @@ +[uwsgi] +http-socket = :5000 +uid = ckan +guid = ckan +plugins = python3 +wsgi-file = /srv/app/wsgi.py +virtualenv = /srv/app +module = wsgi:application +master = true +processes = 5 +pidfile = /tmp/%n.pid +harakiri = 50 +max-requests = 5000 +vacuum = true +callable = application diff --git a/images/ckan-base/2.7/setup/supervisor.worker.conf b/images/ckan-base/2.7/setup/supervisor.worker.conf deleted file mode 100755 index 0ca43ba..0000000 --- a/images/ckan-base/2.7/setup/supervisor.worker.conf +++ /dev/null @@ -1,13 +0,0 @@ -[program:ckan-worker] -command=paster --plugin=ckan jobs worker -c /srv/app/production.ini -priority=501 -autostart=true -autorestart=true -redirect_stderr=true -stdout_logfile=/dev/stdout -stdout_logfile_maxbytes=0 -stderr_logfile=/dev/stdout -stderr_logfile_maxbytes=0 -user=ckan -environment=HOME="/srv/app",USER="ckan" - diff --git a/images/ckan-base/2.7/setup/wsgi.py b/images/ckan-base/2.7/setup/wsgi.py new file mode 100644 index 0000000..b37d80e --- /dev/null +++ b/images/ckan-base/2.7/setup/wsgi.py @@ -0,0 +1,9 @@ +import os +from ckan.config.middleware import make_app +from ckan.cli import CKANConfigLoader +from logging.config import fileConfig as loggingFileConfig +config_filepath = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'ckan.ini') +abspath = os.path.join(os.path.dirname(os.path.abspath(__file__))) +loggingFileConfig(config_filepath) +config = CKANConfigLoader(config_filepath).get_config() +application = make_app(config) diff --git a/images/ckan-base/2.8/Dockerfile b/images/ckan-base/2.8/Dockerfile index e045408..e2cec51 100755 --- a/images/ckan-base/2.8/Dockerfile +++ b/images/ckan-base/2.8/Dockerfile @@ -79,9 +79,11 @@ RUN addgroup -g 92 -S ckan && \ RUN mkdir -p $CKAN_STORAGE_PATH && \ chown -R ckan:ckan $CKAN_STORAGE_PATH -COPY 2.8/setup ${APP_DIR} -COPY 2.8/setup/supervisor.worker.conf /etc/supervisord.d/worker.conf -COPY 2.8/setup/uwsgi.conf /srv/app/uwsgi.conf +COPY 2.8/setup/ckan-uwsgi.ini ${APP_DIR} +COPY 2.8/setup/wsgi.py ${APP_DIR} +COPY 2.8/setup/prerun.py ${APP_DIR} +COPY 2.8/setup/start_ckan.sh ${APP_DIR} +COPY 2.8/setup/supervisord.conf /etc/supervisord.conf # Create entrypoint directory for children image scripts ONBUILD RUN mkdir /docker-entrypoint.d diff --git a/images/ckan-base/2.8/setup/ckan-uwsgi.ini b/images/ckan-base/2.8/setup/ckan-uwsgi.ini new file mode 100644 index 0000000..2361f36 --- /dev/null +++ b/images/ckan-base/2.8/setup/ckan-uwsgi.ini @@ -0,0 +1,15 @@ +[uwsgi] +http-socket = :5000 +uid = ckan +guid = ckan +plugins = python3 +wsgi-file = /srv/app/wsgi.py +virtualenv = /srv/app +module = wsgi:application +master = true +processes = 5 +pidfile = /tmp/%n.pid +harakiri = 50 +max-requests = 5000 +vacuum = true +callable = application diff --git a/images/ckan-base/2.8/setup/supervisor.worker.conf b/images/ckan-base/2.8/setup/supervisor.worker.conf deleted file mode 100755 index 0ca43ba..0000000 --- a/images/ckan-base/2.8/setup/supervisor.worker.conf +++ /dev/null @@ -1,13 +0,0 @@ -[program:ckan-worker] -command=paster --plugin=ckan jobs worker -c /srv/app/production.ini -priority=501 -autostart=true -autorestart=true -redirect_stderr=true -stdout_logfile=/dev/stdout -stdout_logfile_maxbytes=0 -stderr_logfile=/dev/stdout -stderr_logfile_maxbytes=0 -user=ckan -environment=HOME="/srv/app",USER="ckan" - diff --git a/images/ckan-base/2.8/setup/wsgi.py b/images/ckan-base/2.8/setup/wsgi.py new file mode 100644 index 0000000..b37d80e --- /dev/null +++ b/images/ckan-base/2.8/setup/wsgi.py @@ -0,0 +1,9 @@ +import os +from ckan.config.middleware import make_app +from ckan.cli import CKANConfigLoader +from logging.config import fileConfig as loggingFileConfig +config_filepath = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'ckan.ini') +abspath = os.path.join(os.path.dirname(os.path.abspath(__file__))) +loggingFileConfig(config_filepath) +config = CKANConfigLoader(config_filepath).get_config() +application = make_app(config) diff --git a/images/ckan-base/2.9/Dockerfile b/images/ckan-base/2.9/Dockerfile index 4cda64c..a0d28d4 100755 --- a/images/ckan-base/2.9/Dockerfile +++ b/images/ckan-base/2.9/Dockerfile @@ -63,7 +63,7 @@ RUN cd ${APP_DIR} && \ # Virtual environment binaries/scripts to be used first ENV PATH=${APP_DIR}/bin:${PATH} -# Install CKAN and uwsgi +# Install CKAN, uwsgi, ckanext-envvars and generate CKAN config file RUN pip3 install -e git+${GIT_URL}@${GIT_BRANCH}#egg=ckan && \ pip3 install uwsgi && \ cd ${SRC_DIR}/ckan && \ @@ -80,8 +80,10 @@ RUN pip3 install supervisor && \ mkdir /etc/supervisord.d # Copy all setup files -COPY 2.9/setup ${APP_DIR} -COPY 2.9/setup/supervisor.worker.conf /etc/supervisord.d/worker.conf +COPY 2.9/setup/ckan-uwsgi.ini ${APP_DIR} +COPY 2.9/setup/wsgi.py ${APP_DIR} +COPY 2.9/setup/prerun.py ${APP_DIR} +COPY 2.9/setup/start_ckan.sh ${APP_DIR} COPY 2.9/setup/supervisord.conf /etc/supervisord.conf # Create a local user and group to run the app diff --git a/images/ckan-base/2.9/setup/supervisor.worker.conf b/images/ckan-base/2.9/setup/supervisor.worker.conf deleted file mode 100644 index 9d46f37..0000000 --- a/images/ckan-base/2.9/setup/supervisor.worker.conf +++ /dev/null @@ -1,12 +0,0 @@ -[program:ckan-worker] -command=ckan -c /srv/app/ckan.ini jobs worker -priority=501 -autostart=true -autorestart=true -redirect_stderr=true -stdout_logfile=/dev/stdout -stdout_logfile_maxbytes=0 -stderr_logfile=/dev/stdout -stderr_logfile_maxbytes=0 -user=ckan -environment=HOME="/srv/app",USER="ckan" \ No newline at end of file From 5cc6d1af4ba7d9e782284efa4b89803bd8b33ce1 Mon Sep 17 00:00:00 2001 From: Brett Date: Wed, 12 Jan 2022 15:57:17 +0100 Subject: [PATCH 14/14] More README changes --- README.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.txt b/README.txt index a769c0e..022f23c 100644 --- a/README.txt +++ b/README.txt @@ -16,6 +16,8 @@ out of the build and hence will not show with an "inspect image" command Do we pre-build a CKAN image and use that (and extend) as the base image OR just build it from scratch and while saving to storage what we need to +To use local storage for the ckan.ini file and the CKAN src code, do the following: + # docker-compose up -d --build # mkdir local # docker cp ckan:/srv/app/ckan.ini ./local/ckan.ini