From 329557ca3c46fd53ceece11053b4ad312f60d84c Mon Sep 17 00:00:00 2001 From: Brett <54408245+kowh-ai@users.noreply.github.com> Date: Fri, 15 Jan 2021 16:05:22 +0100 Subject: [PATCH] Dockerfile changes etc --- .env | 46 +++++++++++++++++++++++++++++++++++++++++++ README.txt | 16 +++++++++++++-- postgresql/Dockerfile | 3 +-- solr/Dockerfile | 41 +++++++++++++++++++++----------------- solr/solrconfig.xml | 4 +++- 5 files changed, 87 insertions(+), 23 deletions(-) create mode 100644 .env diff --git a/.env b/.env new file mode 100644 index 0000000..a3d9300 --- /dev/null +++ b/.env @@ -0,0 +1,46 @@ +# CKAN databases +POSTGRES_USER=ckan +POSTGRES_PASSWORD=ckan +DATASTORE_READONLY_USER=datastore_ro +DATASTORE_READONLY_PASSWORD=datastore +POSTGRES_HOST=db +CKAN_SQLALCHEMY_URL=postgresql://ckan:ckan@db/ckan +CKAN_DATASTORE_WRITE_URL=postgresql://ckan:ckan@db/datastore +CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:datastore@db/datastore + +# Test database connections +TEST_CKAN_SQLALCHEMY_URL=postgres://ckan:ckan@db/ckan_test +TEST_CKAN_DATASTORE_WRITE_URL=postgresql://ckan:ckan@db/datastore_test +TEST_CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:datastore@db/datastore_test + +# CKAN core +CKAN_SITE_ID=default +CKAN_SITE_URL=http://ckan:5000 +CKAN_PORT=5000 +CKAN_SYSADMIN_NAME=ckan_admin +CKAN_SYSADMIN_PASSWORD=test1234 +CKAN_SYSADMIN_EMAIL=your_email@example.com +CKAN_STORAGE_PATH=/var/lib/ckan +CKAN_SMTP_SERVER=smtp.corporateict.domain:25 +CKAN_SMTP_STARTTLS=True +CKAN_SMTP_USER=user +CKAN_SMTP_PASSWORD=pass +CKAN_SMTP_MAIL_FROM=ckan@localhost +TZ=UTC + +# Other services +CKAN_SOLR_URL=http://solr:8983/solr/ckan +CKAN_REDIS_URL=redis://redis:6379/1 +CKAN_DATAPUSHER_URL=http://datapusher:8801 +CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://ckan:5000 + +# test connections +TEST_CKAN_SOLR_URL=http://solr:8983/solr/ckan +TEST_CKAN_REDIS_URL=redis://redis:6379/1 + +# Extensions +CKAN__PLUGINS=envvars image_view text_view recline_view datastore datapusher +CKAN__HARVEST__MQ__TYPE=redis +CKAN__HARVEST__MQ__HOSTNAME=redis +CKAN__HARVEST__MQ__PORT=6379 +CKAN__HARVEST__MQ__REDIS_DB=1 \ No newline at end of file diff --git a/README.txt b/README.txt index c24fed2..aaeeb44 100644 --- a/README.txt +++ b/README.txt @@ -6,13 +6,25 @@ Potential ideas/investigations from Keitaro (Marko Bocevski) : https://github.co - Maybe use build/up time variables loaded via the .env file, runtime variables loaded via a .ckan-env file - both located at the root directory -- Should we use wheels too? + +- Should we use wheels too? + +- Use Multi Stage Docker builds? + run some tests on image/container size with the different options + a smaller production build vs a development build with all the dev tools + +- Use SSL on the NGINX container port + - 2 networks: 1) Frontend 2) Backend + - Create/access a CKAN datapusher image rather than build one? + - all username/passwords as environment variables rather than hardcoded + - "/images" is a good folder name to use to be able to build all versions of CKAN and DataPusher images -- include an "/examples" folder for examples of adding extentions xloader, harvester, etc - make sure the "development mode" path is taken cared of with any changes + - Create an admin user during the container deployment + - should there be a datapusher container built? maybe an xloader container? diff --git a/postgresql/Dockerfile b/postgresql/Dockerfile index f221611..e912383 100755 --- a/postgresql/Dockerfile +++ b/postgresql/Dockerfile @@ -1,5 +1,4 @@ -FROM postgres:9.6-alpine -MAINTAINER Open Knowledge International +FROM postgres:12-alpine # Allow connections; we don't map out any ports so only linked docker containers can connect RUN echo "host all all 0.0.0.0/0 md5" >> /var/lib/postgresql/data/pg_hba.conf diff --git a/solr/Dockerfile b/solr/Dockerfile index 338fc15..8922342 100755 --- a/solr/Dockerfile +++ b/solr/Dockerfile @@ -1,29 +1,34 @@ -FROM solr:6.2-alpine -MAINTAINER Open Knowledge +FROM solr:6.6.6 -# Enviroment +# Enviroment variables ENV SOLR_CORE ckan -ENV CKAN_VERSION dev-v2.9 +ENV SOLR_VERSION 6.6.6 +ENV CKAN_VERSION 2.9.1 +###TODO!!! CKAN_VERSION to be passed in as an ARG -# User +# root user for initial config USER root -# Create Directories -RUN mkdir -p /opt/solr/server/solr/$SOLR_CORE/conf -RUN mkdir -p /opt/solr/server/solr/$SOLR_CORE/data +# Create directories +RUN mkdir -p /opt/solr/server/solr/${SOLR_CORE}/conf && \ + mkdir -p /opt/solr/server/solr/${SOLR_CORE}/data && \ + mkdir -p /opt/solr/server/solr/${SOLR_CORE}/data/index -# Adding Files +# Add files ADD solrconfig.xml \ -https://raw.githubusercontent.com/ckan/ckan/$CKAN_VERSION/ckan/config/solr/schema.xml \ -https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.0.0/solr/server/solr/configsets/basic_configs/conf/currency.xml \ -https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.0.0/solr/server/solr/configsets/basic_configs/conf/synonyms.txt \ -https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.0.0/solr/server/solr/configsets/basic_configs/conf/stopwords.txt \ -https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.0.0/solr/server/solr/configsets/basic_configs/conf/protwords.txt \ -https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.0.0/solr/server/solr/configsets/data_driven_schema_configs/conf/elevate.xml \ +https://raw.githubusercontent.com/ckan/ckan/ckan-${CKAN_VERSION}/ckan/config/solr/schema.xml \ +https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/$SOLR_VERSION/solr/server/solr/configsets/basic_configs/conf/currency.xml \ +https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/$SOLR_VERSION/solr/server/solr/configsets/basic_configs/conf/synonyms.txt \ +https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/$SOLR_VERSION/solr/server/solr/configsets/basic_configs/conf/stopwords.txt \ +https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/$SOLR_VERSION/solr/server/solr/configsets/basic_configs/conf/protwords.txt \ +https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/$SOLR_VERSION/solr/server/solr/configsets/data_driven_schema_configs/conf/elevate.xml \ /opt/solr/server/solr/$SOLR_CORE/conf/ -# Create Core.properties -RUN echo name=$SOLR_CORE > /opt/solr/server/solr/$SOLR_CORE/core.properties +# Create core.properties +RUN echo name=${SOLR_CORE} > /opt/solr/server/solr/${SOLR_CORE}/core.properties # Giving ownership to Solr -RUN chown -R $SOLR_USER:$SOLR_USER /opt/solr/server/solr/$SOLR_CORE +RUN chown -R ${SOLR_USER}:${SOLR_USER} /opt/solr/server/solr/${SOLR_CORE} + +# non-root user for runtime +USER ${SOLR_USER}:${SOLR_USER} diff --git a/solr/solrconfig.xml b/solr/solrconfig.xml index 9ac620c..8a3eade 100755 --- a/solr/solrconfig.xml +++ b/solr/solrconfig.xml @@ -288,7 +288,9 @@ - + + true +