Dockerfile changes etc

.env

@@ -0,0 +1,46 @@
+# CKAN databases
+POSTGRES_USER=ckan
+POSTGRES_PASSWORD=ckan
+DATASTORE_READONLY_USER=datastore_ro
+DATASTORE_READONLY_PASSWORD=datastore
+POSTGRES_HOST=db
+CKAN_SQLALCHEMY_URL=postgresql://ckan:ckan@db/ckan
+CKAN_DATASTORE_WRITE_URL=postgresql://ckan:ckan@db/datastore
+CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:datastore@db/datastore
+
+# Test database connections
+TEST_CKAN_SQLALCHEMY_URL=postgres://ckan:ckan@db/ckan_test
+TEST_CKAN_DATASTORE_WRITE_URL=postgresql://ckan:ckan@db/datastore_test
+TEST_CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:datastore@db/datastore_test
+
+# CKAN core
+CKAN_SITE_ID=default
+CKAN_SITE_URL=http://ckan:5000
+CKAN_PORT=5000
+CKAN_SYSADMIN_NAME=ckan_admin
+CKAN_SYSADMIN_PASSWORD=test1234
+CKAN_SYSADMIN_EMAIL=your_email@example.com
+CKAN_STORAGE_PATH=/var/lib/ckan
+CKAN_SMTP_SERVER=smtp.corporateict.domain:25
+CKAN_SMTP_STARTTLS=True
+CKAN_SMTP_USER=user
+CKAN_SMTP_PASSWORD=pass
+CKAN_SMTP_MAIL_FROM=ckan@localhost
+TZ=UTC
+
+# Other services
+CKAN_SOLR_URL=http://solr:8983/solr/ckan
+CKAN_REDIS_URL=redis://redis:6379/1
+CKAN_DATAPUSHER_URL=http://datapusher:8801
+CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://ckan:5000
+
+# test connections
+TEST_CKAN_SOLR_URL=http://solr:8983/solr/ckan
+TEST_CKAN_REDIS_URL=redis://redis:6379/1
+
+# Extensions
+CKAN__PLUGINS=envvars image_view text_view recline_view datastore datapusher
+CKAN__HARVEST__MQ__TYPE=redis
+CKAN__HARVEST__MQ__HOSTNAME=redis
+CKAN__HARVEST__MQ__PORT=6379
+CKAN__HARVEST__MQ__REDIS_DB=1

README.txt

@@ -6,13 +6,25 @@ Potential ideas/investigations from Keitaro (Marko Bocevski) : https://github.co
 
 - Maybe use build/up time variables loaded via the .env file, runtime variables loaded via a .ckan-env file - both
 located at the root directory
+
 - Should we use wheels too?
+
+- Use Multi Stage Docker builds? 
+    run some tests on image/container size with the different options
+    a smaller production build vs a development build with all the dev tools
+
+- Use SSL on the NGINX container port
+
 - 2 networks: 1) Frontend 2) Backend
+
 - Create/access a CKAN datapusher image rather than build one?
+
 - all username/passwords as environment variables rather than hardcoded
+
 - "/images" is a good folder name to use to be able to build all versions of CKAN and DataPusher images
-- include an "/examples" folder for examples of adding extentions xloader, harvester, etc
 
 - make sure the "development mode" path is taken cared of with any changes
+
 - Create an admin user during the container deployment
+
 - should there be a datapusher container built? maybe an xloader container?

postgresql/Dockerfile

@@ -1,5 +1,4 @@
-FROM postgres:9.6-alpine
+FROM postgres:12-alpine
-MAINTAINER Open Knowledge International
 
 # Allow connections; we don't map out any ports so only linked docker containers can connect
 RUN echo "host all  all    0.0.0.0/0  md5" >> /var/lib/postgresql/data/pg_hba.conf

solr/Dockerfile

@@ -1,29 +1,34 @@
-FROM solr:6.2-alpine
+FROM solr:6.6.6
-MAINTAINER Open Knowledge
 
-# Enviroment
+# Enviroment variables
 ENV SOLR_CORE ckan
-ENV CKAN_VERSION dev-v2.9
+ENV SOLR_VERSION 6.6.6
+ENV CKAN_VERSION 2.9.1
+###TODO!!! CKAN_VERSION to be passed in as an ARG
 
-# User
+# root user for initial config
 USER root
 
-# Create Directories
+# Create directories
-RUN mkdir -p /opt/solr/server/solr/$SOLR_CORE/conf
+RUN mkdir -p /opt/solr/server/solr/${SOLR_CORE}/conf && \
-RUN mkdir -p /opt/solr/server/solr/$SOLR_CORE/data
+    mkdir -p /opt/solr/server/solr/${SOLR_CORE}/data && \
+    mkdir -p /opt/solr/server/solr/${SOLR_CORE}/data/index
 
-# Adding Files
+# Add files
 ADD solrconfig.xml \
-https://raw.githubusercontent.com/ckan/ckan/$CKAN_VERSION/ckan/config/solr/schema.xml \
+https://raw.githubusercontent.com/ckan/ckan/ckan-${CKAN_VERSION}/ckan/config/solr/schema.xml \
-https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.0.0/solr/server/solr/configsets/basic_configs/conf/currency.xml \
+https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/$SOLR_VERSION/solr/server/solr/configsets/basic_configs/conf/currency.xml \
-https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.0.0/solr/server/solr/configsets/basic_configs/conf/synonyms.txt \
+https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/$SOLR_VERSION/solr/server/solr/configsets/basic_configs/conf/synonyms.txt \
-https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.0.0/solr/server/solr/configsets/basic_configs/conf/stopwords.txt \
+https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/$SOLR_VERSION/solr/server/solr/configsets/basic_configs/conf/stopwords.txt \
-https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.0.0/solr/server/solr/configsets/basic_configs/conf/protwords.txt \
+https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/$SOLR_VERSION/solr/server/solr/configsets/basic_configs/conf/protwords.txt \
-https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.0.0/solr/server/solr/configsets/data_driven_schema_configs/conf/elevate.xml \
+https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/$SOLR_VERSION/solr/server/solr/configsets/data_driven_schema_configs/conf/elevate.xml \
 /opt/solr/server/solr/$SOLR_CORE/conf/
 
-# Create Core.properties
+# Create core.properties
-RUN echo name=$SOLR_CORE > /opt/solr/server/solr/$SOLR_CORE/core.properties
+RUN echo name=${SOLR_CORE} > /opt/solr/server/solr/${SOLR_CORE}/core.properties
 
 # Giving ownership to Solr
-RUN chown -R $SOLR_USER:$SOLR_USER /opt/solr/server/solr/$SOLR_CORE
+RUN chown -R ${SOLR_USER}:${SOLR_USER} /opt/solr/server/solr/${SOLR_CORE}
+
+# non-root user for runtime
+USER ${SOLR_USER}:${SOLR_USER}

solr/solrconfig.xml

@@ -288,7 +288,9 @@
 		
 	</searchComponent>
 	
-	<schemaFactory class="ClassicIndexSchemaFactory" />
+	<schemaFactory class="ManagedIndexSchemaFactory">
+		<bool name="mutable">true</bool>
+	</schemaFactory>
 	
 	<updateRequestProcessorChain name="add-unknown-fields-to-the-schema">
 		<processor class="solr.UUIDUpdateProcessorFactory" />