From 6a8c9ecc5cfc0b824d5b145b17ee46aaccb004e9 Mon Sep 17 00:00:00 2001
From: Brett <brett@kowh.ai>
Date: Fri, 29 Oct 2021 10:58:20 +0200
Subject: [PATCH 01/10] Commit new images directory

---
 README                                     |  10 ++
 ckan-base/2.7/Dockerfile                   | 101 +++++++++++
 ckan-base/2.7/setup/prerun.py              | 197 +++++++++++++++++++++
 ckan-base/2.7/setup/start_ckan.sh          |  47 +++++
 ckan-base/2.7/setup/supervisor.worker.conf |  13 ++
 ckan-base/2.7/setup/supervisord.conf       |  23 +++
 ckan-base/2.7/setup/uwsgi.conf             |   2 +
 ckan-base/2.8/Dockerfile                   |  99 +++++++++++
 ckan-base/2.8/setup/prerun.py              | 197 +++++++++++++++++++++
 ckan-base/2.8/setup/start_ckan.sh          |  47 +++++
 ckan-base/2.8/setup/supervisor.worker.conf |  13 ++
 ckan-base/2.8/setup/supervisord.conf       |  23 +++
 ckan-base/2.8/setup/uwsgi.conf             |   2 +
 ckan-base/2.9/Dockerfile                   | 109 ++++++++++++
 ckan-base/2.9/setup/ckan-uwsgi.ini         |  15 ++
 ckan-base/2.9/setup/prerun.py              | 194 ++++++++++++++++++++
 ckan-base/2.9/setup/start_ckan.sh          |  46 +++++
 ckan-base/2.9/setup/supervisor.worker.conf |  12 ++
 ckan-base/2.9/setup/supervisord.conf       |  23 +++
 ckan-base/2.9/setup/wsgi.py                |   9 +
 ckan-dev/2.7/Dockerfile                    |  20 +++
 ckan-dev/2.8/Dockerfile                    |  20 +++
 ckan-dev/2.9/Dockerfile                    |  29 +++
 ckan-dev/setup/start_ckan_development.sh   |  82 +++++++++
 24 files changed, 1333 insertions(+)
 create mode 100755 README
 create mode 100755 ckan-base/2.7/Dockerfile
 create mode 100755 ckan-base/2.7/setup/prerun.py
 create mode 100755 ckan-base/2.7/setup/start_ckan.sh
 create mode 100755 ckan-base/2.7/setup/supervisor.worker.conf
 create mode 100755 ckan-base/2.7/setup/supervisord.conf
 create mode 100755 ckan-base/2.7/setup/uwsgi.conf
 create mode 100755 ckan-base/2.8/Dockerfile
 create mode 100755 ckan-base/2.8/setup/prerun.py
 create mode 100755 ckan-base/2.8/setup/start_ckan.sh
 create mode 100755 ckan-base/2.8/setup/supervisor.worker.conf
 create mode 100755 ckan-base/2.8/setup/supervisord.conf
 create mode 100755 ckan-base/2.8/setup/uwsgi.conf
 create mode 100755 ckan-base/2.9/Dockerfile
 create mode 100644 ckan-base/2.9/setup/ckan-uwsgi.ini
 create mode 100755 ckan-base/2.9/setup/prerun.py
 create mode 100755 ckan-base/2.9/setup/start_ckan.sh
 create mode 100644 ckan-base/2.9/setup/supervisor.worker.conf
 create mode 100644 ckan-base/2.9/setup/supervisord.conf
 create mode 100644 ckan-base/2.9/setup/wsgi.py
 create mode 100755 ckan-dev/2.7/Dockerfile
 create mode 100755 ckan-dev/2.8/Dockerfile
 create mode 100755 ckan-dev/2.9/Dockerfile
 create mode 100755 ckan-dev/setup/start_ckan_development.sh

diff --git a/README b/README
new file mode 100755
index 0000000..703c891
--- /dev/null
+++ b/README
@@ -0,0 +1,10 @@
+Build the images from this directory using:
+
+    cd $REPO_HOME
+    cd images/ckan-base
+    docker build -t ckan/ckan-base:testing-only-2.9 -f 2.9/Dockerfile .
+    docker push ckan/ckan-base:testing-only-2.9
+    docker build -t ckan/ckan-base:testing-only-2.8 -f 2.8/Dockerfile .
+    docker push ckan/ckan-base:testing-only-2.8
+    docker build -t ckan/ckan-base:testing-only-2.7 -f 2.7/Dockerfile .
+    docker push ckan/ckan-base:testing-only-2.7
diff --git a/ckan-base/2.7/Dockerfile b/ckan-base/2.7/Dockerfile
new file mode 100755
index 0000000..4eb4332
--- /dev/null
+++ b/ckan-base/2.7/Dockerfile
@@ -0,0 +1,101 @@
+FROM alpine:3.13
+
+# Internals, you probably don't need to change these
+ENV APP_DIR=/srv/app
+ENV SRC_DIR=/srv/app/src
+ENV CKAN_INI=${APP_DIR}/production.ini
+ENV PIP_SRC=${SRC_DIR}
+ENV CKAN_STORAGE_PATH=/var/lib/ckan
+ENV GIT_URL=https://github.com/ckan/ckan.git
+# CKAN version to build
+ENV GIT_BRANCH=ckan-2.7.11
+# Customize these on the .env file if needed
+ENV CKAN_SITE_URL=http://localhost:5000
+ENV CKAN__PLUGINS image_view text_view recline_view datastore datapusher envvars
+
+WORKDIR ${APP_DIR}
+
+# Install necessary packages to run CKAN
+RUN apk add --no-cache tzdata \
+        git \
+        gettext \
+        postgresql-client \
+        python \
+        apache2-utils \
+        libxml2 \
+        libxslt \
+        musl-dev \
+        uwsgi \
+        uwsgi-http \
+        uwsgi-corerouter \
+        uwsgi-python \
+        py2-gevent \
+        uwsgi-gevent \
+        libmagic \
+        curl \
+        sudo && \
+    # Packages to build CKAN requirements and plugins
+    apk add --no-cache --virtual .build-deps \
+        postgresql-dev \
+        gcc \
+        make \
+        g++ \
+        autoconf \
+        automake \
+	libtool \
+        python-dev \
+        libxml2-dev \
+        libxslt-dev \
+        linux-headers
+
+# Create SRC_DIR
+RUN mkdir -p ${SRC_DIR}
+
+# Install pip
+RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/pip/2.7/get-pip.py && \
+    python ${SRC_DIR}/get-pip.py 'pip==20.3.3'
+
+# Install pip, supervisord and uwsgi
+RUN pip install supervisor && \
+    mkdir /etc/supervisord.d && \
+    #pip wheel --wheel-dir=/wheels uwsgi gevent && \
+    rm -rf ${SRC_DIR}/get-pip.py
+
+COPY 2.7/setup/supervisord.conf /etc
+
+# Install CKAN
+RUN pip install -e git+${GIT_URL}@${GIT_BRANCH}#egg=ckan && \
+    cd ${SRC_DIR}/ckan && \
+    cp who.ini ${APP_DIR} && \
+    # Workaround to solve https://github.com/psycopg/psycopg2/issues/594 in Alpine 3.7
+	sed -i -e "s/psycopg2==2.4.5/psycopg2==2.7.3.2/g" requirements.txt && \
+    pip install --no-binary :all: -r requirements.txt && \
+    # Install CKAN envvars to support loading config from environment variables
+    pip install -e git+https://github.com/okfn/ckanext-envvars.git#egg=ckanext-envvars && \
+    # Create and update CKAN config
+    paster --plugin=ckan make-config ckan ${CKAN_INI} && \
+    paster --plugin=ckan config-tool ${CKAN_INI} "ckan.plugins = ${CKAN__PLUGINS}" && \
+    paster --plugin=ckan config-tool ${CKAN_INI} "ckan.site_url = ${CKAN__SITE_URL}"
+
+# Create a local user and group to run the app
+RUN addgroup -g 92 -S ckan && \
+    adduser -u 92 -h /srv/app -H -D -S -G ckan ckan
+
+# Create local storage folder
+RUN mkdir -p $CKAN_STORAGE_PATH && \
+    chown -R ckan:ckan $CKAN_STORAGE_PATH
+
+COPY 2.7/setup ${APP_DIR}
+COPY 2.7/setup/supervisor.worker.conf /etc/supervisord.d/worker.conf
+COPY 2.7/setup/uwsgi.conf /srv/app/uwsgi.conf
+
+# Create entrypoint directory for children image scripts
+ONBUILD RUN mkdir /docker-entrypoint.d
+
+RUN chown ckan -R /srv/app
+
+EXPOSE 5000
+
+HEALTHCHECK --interval=10s --timeout=5s --retries=5 CMD curl --fail http://localhost:5000/api/3/action/status_show || exit 1
+
+CMD ["/srv/app/start_ckan.sh"]
diff --git a/ckan-base/2.7/setup/prerun.py b/ckan-base/2.7/setup/prerun.py
new file mode 100755
index 0000000..7ebacfa
--- /dev/null
+++ b/ckan-base/2.7/setup/prerun.py
@@ -0,0 +1,197 @@
+import os
+import sys
+import subprocess
+import psycopg2
+import urllib2
+import time
+import re
+
+ckan_ini = os.environ.get('CKAN_INI', '/srv/app/production.ini')
+
+RETRY = 5
+
+def update_plugins():
+
+    plugins = os.environ.get('CKAN__PLUGINS', '')
+    print('[prerun] Setting the following plugins in {}:'.format(ckan_ini))
+    print(plugins)
+    cmd = ['paster', '--plugin=ckan', 'config-tool',
+           ckan_ini, 'ckan.plugins = {}'.format(plugins)]
+    subprocess.check_output(cmd, stderr=subprocess.STDOUT)
+    print '[prerun] Plugins set.'
+
+    
+def check_main_db_connection(retry=None):
+
+    conn_str = os.environ.get('CKAN_SQLALCHEMY_URL')
+    if not conn_str:
+        print '[prerun] CKAN_SQLALCHEMY_URL not defined, not checking db'
+    return check_db_connection(conn_str, retry)
+
+
+def check_datastore_db_connection(retry=None):
+
+    conn_str = os.environ.get('CKAN_DATASTORE_WRITE_URL')
+    if not conn_str:
+        print '[prerun] CKAN_DATASTORE_WRITE_URL not defined, not checking db'
+    return check_db_connection(conn_str, retry)
+
+
+def check_db_connection(conn_str, retry=None):
+
+    if retry is None:
+        retry = RETRY
+    elif retry == 0:
+        print '[prerun] Giving up after 5 tries...'
+        sys.exit(1)
+
+    try:
+        connection = psycopg2.connect(conn_str)
+
+    except psycopg2.Error as e:
+        print str(e)
+        print '[prerun] Unable to connect to the database, waiting...'
+        time.sleep(10)
+        check_db_connection(conn_str, retry=retry - 1)
+    else:
+        connection.close()
+
+        
+def check_solr_connection(retry=None):
+
+    if retry is None:
+        retry = RETRY
+    elif retry == 0:
+        print '[prerun] Giving up after 5 tries...'
+        sys.exit(1)
+
+    url = os.environ.get('CKAN_SOLR_URL', '')
+    search_url = '{url}/select/?q=*&wt=json'.format(url=url)
+
+    try:
+        connection = urllib2.urlopen(search_url)
+    except urllib2.URLError as e:
+        print str(e)
+        print '[prerun] Unable to connect to solr, waiting...'
+        time.sleep(10)
+        check_solr_connection(retry=retry - 1)
+    else:
+        eval(connection.read())
+
+
+def init_db():
+
+    db_command = ['paster', '--plugin=ckan', 'db',
+                  'init', '-c', ckan_ini]
+    print '[prerun] Initializing or upgrading db - start'
+    try:
+        subprocess.check_output(db_command, stderr=subprocess.STDOUT)
+        print '[prerun] Initializing or upgrading db - end'
+    except subprocess.CalledProcessError, e:
+        if 'OperationalError' in e.output:
+            print e.output
+            print '[prerun] Database not ready, waiting a bit before exit...'
+            time.sleep(5)
+            sys.exit(1)
+        else:
+            print e.output
+            raise e
+
+
+def init_datastore_db():
+
+    conn_str = os.environ.get('CKAN_DATASTORE_WRITE_URL')
+    if not conn_str:
+        print '[prerun] Skipping datastore initialization'
+        return
+
+    datastore_perms_command = ['paster', '--plugin=ckan', 'datastore',
+                               'set-permissions', '-c', ckan_ini]
+
+    connection = psycopg2.connect(conn_str)
+    cursor = connection.cursor()
+
+    print '[prerun] Initializing datastore db - start'
+    try:
+        datastore_perms = subprocess.Popen(
+            datastore_perms_command,
+            stdout=subprocess.PIPE)
+
+        perms_sql = datastore_perms.stdout.read()
+        # Remove internal pg command as psycopg2 does not like it
+        perms_sql = re.sub('\\\\connect \"(.*)\"', '', perms_sql)
+        cursor.execute(perms_sql)
+        for notice in connection.notices:
+            print notice
+
+        connection.commit()
+
+        print '[prerun] Initializing datastore db - end'
+        print datastore_perms.stdout.read()
+    except psycopg2.Error as e:
+        print '[prerun] Could not initialize datastore'
+        print str(e)
+
+    except subprocess.CalledProcessError, e:
+        if 'OperationalError' in e.output:
+            print e.output
+            print '[prerun] Database not ready, waiting a bit before exit...'
+            time.sleep(5)
+            sys.exit(1)
+        else:
+            print e.output
+            raise e
+    finally:
+        cursor.close()
+        connection.close()
+
+
+def create_sysadmin():
+
+    name = os.environ.get('CKAN_SYSADMIN_NAME')
+    password = os.environ.get('CKAN_SYSADMIN_PASSWORD')
+    email = os.environ.get('CKAN_SYSADMIN_EMAIL')
+
+    if name and password and email:
+
+        # Check if user exists
+        command = ['paster', '--plugin=ckan', 'user', name, '-c', ckan_ini]
+
+        out = subprocess.check_output(command)
+        if 'User:None' not in re.sub(r'\s', '', out):
+            print '[prerun] Sysadmin user exists, skipping creation'
+            return
+
+        # Create user
+        command = ['paster', '--plugin=ckan', 'user', 'add',
+                   name,
+                   'password=' + password,
+                   'email=' + email,
+                   '-c', ckan_ini]
+
+        subprocess.call(command)
+        print '[prerun] Created user {0}'.format(name)
+
+        # Make it sysadmin
+        command = ['paster', '--plugin=ckan', 'sysadmin', 'add',
+                   name,
+                   '-c', ckan_ini]
+
+        subprocess.call(command)
+        print '[prerun] Made user {0} a sysadmin'.format(name)
+
+
+if __name__ == '__main__':
+
+    maintenance = os.environ.get('MAINTENANCE_MODE', '').lower() == 'true'
+
+    if maintenance:
+        print '[prerun] Maintenance mode, skipping setup...'
+    else:
+        check_main_db_connection()
+        init_db()
+        update_plugins()
+        check_datastore_db_connection()
+        init_datastore_db()
+        check_solr_connection()
+        create_sysadmin()
\ No newline at end of file
diff --git a/ckan-base/2.7/setup/start_ckan.sh b/ckan-base/2.7/setup/start_ckan.sh
new file mode 100755
index 0000000..d1c6726
--- /dev/null
+++ b/ckan-base/2.7/setup/start_ckan.sh
@@ -0,0 +1,47 @@
+#!/bin/bash
+
+# Run the prerun script to init CKAN and create the default admin user
+sudo -u ckan -EH python prerun.py
+
+# Run any startup scripts provided by images extending this one
+if [[ -d "/docker-entrypoint.d" ]]
+then
+    for f in /docker-entrypoint.d/*; do
+        case "$f" in
+            *.sh)     echo "$0: Running init file $f"; . "$f" ;;
+            *.py)     echo "$0: Running init file $f"; python "$f"; echo ;;
+            *)        echo "$0: Ignoring $f (not an sh or py file)" ;;
+        esac
+        echo
+    done
+fi
+
+# Set the common uwsgi options
+UWSGI_OPTS="--plugins http,python,gevent --socket /tmp/uwsgi.sock --uid 92 --gid 92 --http :5000 --master --enable-threads --paste config:/srv/app/production.ini --paste-logger --lazy-apps --gevent 2000 -p 2 -L"
+
+# Check whether http basic auth password protection is enabled and enable basicauth routing on uwsgi respecfully
+if [ $? -eq 0 ]
+then
+  if [ "$PASSWORD_PROTECT" = true ]
+  then
+    if [ "$HTPASSWD_USER" ] || [ "$HTPASSWD_PASSWORD" ]
+    then
+      # Generate htpasswd file for basicauth
+      htpasswd -d -b -c /srv/app/.htpasswd $HTPASSWD_USER $HTPASSWD_PASSWORD
+      # Start supervisord
+      supervisord --configuration /etc/supervisord.conf &
+      # Start uwsgi with basicauth
+      sudo -u ckan -EH uwsgi --ini /srv/app/uwsgi.conf --pcre-jit $UWSGI_OPTS
+    else
+      echo "Missing HTPASSWD_USER or HTPASSWD_PASSWORD environment variables. Exiting..."
+      exit 1
+    fi
+  else
+    # Start supervisord
+    supervisord --configuration /etc/supervisord.conf &
+    # Start uwsgi
+    sudo -u ckan -EH uwsgi $UWSGI_OPTS
+  fi
+else
+  echo "[prerun] failed...not starting CKAN."
+fi
diff --git a/ckan-base/2.7/setup/supervisor.worker.conf b/ckan-base/2.7/setup/supervisor.worker.conf
new file mode 100755
index 0000000..0ca43ba
--- /dev/null
+++ b/ckan-base/2.7/setup/supervisor.worker.conf
@@ -0,0 +1,13 @@
+[program:ckan-worker]
+command=paster --plugin=ckan jobs worker -c /srv/app/production.ini
+priority=501
+autostart=true
+autorestart=true
+redirect_stderr=true
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stdout
+stderr_logfile_maxbytes=0
+user=ckan
+environment=HOME="/srv/app",USER="ckan"
+
diff --git a/ckan-base/2.7/setup/supervisord.conf b/ckan-base/2.7/setup/supervisord.conf
new file mode 100755
index 0000000..a3f6671
--- /dev/null
+++ b/ckan-base/2.7/setup/supervisord.conf
@@ -0,0 +1,23 @@
+[unix_http_server]
+file = /tmp/supervisor.sock
+chmod = 0777
+chown = nobody:nogroup
+
+[supervisord]
+logfile = /tmp/supervisord.log
+logfile_maxbytes = 50MB
+logfile_backups=10
+loglevel = info
+pidfile = /tmp/supervisord.pid
+nodaemon = true
+umask = 022
+identifier = supervisor
+
+[supervisorctl]
+serverurl = unix:///tmp/supervisor.sock
+
+[rpcinterface:supervisor]
+supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
+
+[include]
+files = /etc/supervisord.d/*.conf
diff --git a/ckan-base/2.7/setup/uwsgi.conf b/ckan-base/2.7/setup/uwsgi.conf
new file mode 100755
index 0000000..6321d6d
--- /dev/null
+++ b/ckan-base/2.7/setup/uwsgi.conf
@@ -0,0 +1,2 @@
+[uwsgi]
+route = ^(?!/api).*$ basicauth:Restricted,/srv/app/.htpasswd
diff --git a/ckan-base/2.8/Dockerfile b/ckan-base/2.8/Dockerfile
new file mode 100755
index 0000000..59904a3
--- /dev/null
+++ b/ckan-base/2.8/Dockerfile
@@ -0,0 +1,99 @@
+FROM alpine:3.13
+
+# Internals, you probably don't need to change these
+ENV APP_DIR=/srv/app
+ENV SRC_DIR=/srv/app/src
+ENV CKAN_INI=${APP_DIR}/production.ini
+ENV PIP_SRC=${SRC_DIR}
+ENV CKAN_STORAGE_PATH=/var/lib/ckan
+ENV GIT_URL=https://github.com/ckan/ckan.git
+# CKAN version to build
+ENV GIT_BRANCH=ckan-2.8.8
+# Customize these on the .env file if needed
+ENV CKAN_SITE_URL=http://localhost:5000
+ENV CKAN__PLUGINS image_view text_view recline_view datastore datapusher envvars
+
+WORKDIR ${APP_DIR}
+
+# Install necessary packages to run CKAN
+RUN apk add --no-cache tzdata \
+        git \
+        gettext \
+        postgresql-client \
+        python \
+        apache2-utils \
+        libxml2 \
+        libxslt \
+        musl-dev \
+        uwsgi \
+        uwsgi-http \
+        uwsgi-corerouter \
+        uwsgi-python \
+        py2-gevent \
+        uwsgi-gevent \
+        libmagic \
+        curl \
+        sudo && \
+    # Packages to build CKAN requirements and plugins
+    apk add --no-cache --virtual .build-deps \
+        postgresql-dev \
+        gcc \
+        make \
+        g++ \
+        autoconf \
+        automake \
+	    libtool \
+        python-dev \
+        libxml2-dev \
+        libxslt-dev \
+        linux-headers
+
+# Create SRC_DIR
+RUN mkdir -p ${SRC_DIR}
+    
+# Install pip
+RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/pip/2.7/get-pip.py && \
+    python ${SRC_DIR}/get-pip.py 'pip==20.3.3'
+
+# Install supervisord and uwsgi
+RUN pip install supervisor && \
+    mkdir /etc/supervisord.d && \
+    #pip wheel --wheel-dir=/wheels uwsgi gevent && \
+    rm -rf ${SRC_DIR}/get-pip.py
+
+COPY 2.8/setup/supervisord.conf /etc
+
+# Install CKAN
+RUN pip install -e git+${GIT_URL}@${GIT_BRANCH}#egg=ckan && \
+    cd ${SRC_DIR}/ckan && \
+    cp who.ini ${APP_DIR} && \
+    pip install --no-binary :all: -r requirements.txt && \
+    # Install CKAN envvars to support loading config from environment variables
+    pip install -e git+https://github.com/okfn/ckanext-envvars.git#egg=ckanext-envvars && \
+    # Create and update CKAN config
+    paster --plugin=ckan make-config ckan ${CKAN_INI} && \
+    paster --plugin=ckan config-tool ${CKAN_INI} "ckan.plugins = ${CKAN__PLUGINS}" && \
+    paster --plugin=ckan config-tool ${CKAN_INI} "ckan.site_url = ${CKAN__SITE_URL}"
+
+# Create a local user and group to run the app
+RUN addgroup -g 92 -S ckan && \
+    adduser -u 92 -h /srv/app -H -D -S -G ckan ckan
+
+# Create local storage folder
+RUN mkdir -p $CKAN_STORAGE_PATH && \
+    chown -R ckan:ckan $CKAN_STORAGE_PATH
+
+COPY 2.8/setup ${APP_DIR}
+COPY 2.8/setup/supervisor.worker.conf /etc/supervisord.d/worker.conf
+COPY 2.8/setup/uwsgi.conf /srv/app/uwsgi.conf
+
+# Create entrypoint directory for children image scripts
+ONBUILD RUN mkdir /docker-entrypoint.d
+
+RUN chown ckan -R /srv/app
+
+EXPOSE 5000
+
+HEALTHCHECK --interval=10s --timeout=5s --retries=5 CMD curl --fail http://localhost:5000/api/3/action/status_show || exit 1
+
+CMD ["/srv/app/start_ckan.sh"]
diff --git a/ckan-base/2.8/setup/prerun.py b/ckan-base/2.8/setup/prerun.py
new file mode 100755
index 0000000..7ebacfa
--- /dev/null
+++ b/ckan-base/2.8/setup/prerun.py
@@ -0,0 +1,197 @@
+import os
+import sys
+import subprocess
+import psycopg2
+import urllib2
+import time
+import re
+
+ckan_ini = os.environ.get('CKAN_INI', '/srv/app/production.ini')
+
+RETRY = 5
+
+def update_plugins():
+
+    plugins = os.environ.get('CKAN__PLUGINS', '')
+    print('[prerun] Setting the following plugins in {}:'.format(ckan_ini))
+    print(plugins)
+    cmd = ['paster', '--plugin=ckan', 'config-tool',
+           ckan_ini, 'ckan.plugins = {}'.format(plugins)]
+    subprocess.check_output(cmd, stderr=subprocess.STDOUT)
+    print '[prerun] Plugins set.'
+
+    
+def check_main_db_connection(retry=None):
+
+    conn_str = os.environ.get('CKAN_SQLALCHEMY_URL')
+    if not conn_str:
+        print '[prerun] CKAN_SQLALCHEMY_URL not defined, not checking db'
+    return check_db_connection(conn_str, retry)
+
+
+def check_datastore_db_connection(retry=None):
+
+    conn_str = os.environ.get('CKAN_DATASTORE_WRITE_URL')
+    if not conn_str:
+        print '[prerun] CKAN_DATASTORE_WRITE_URL not defined, not checking db'
+    return check_db_connection(conn_str, retry)
+
+
+def check_db_connection(conn_str, retry=None):
+
+    if retry is None:
+        retry = RETRY
+    elif retry == 0:
+        print '[prerun] Giving up after 5 tries...'
+        sys.exit(1)
+
+    try:
+        connection = psycopg2.connect(conn_str)
+
+    except psycopg2.Error as e:
+        print str(e)
+        print '[prerun] Unable to connect to the database, waiting...'
+        time.sleep(10)
+        check_db_connection(conn_str, retry=retry - 1)
+    else:
+        connection.close()
+
+        
+def check_solr_connection(retry=None):
+
+    if retry is None:
+        retry = RETRY
+    elif retry == 0:
+        print '[prerun] Giving up after 5 tries...'
+        sys.exit(1)
+
+    url = os.environ.get('CKAN_SOLR_URL', '')
+    search_url = '{url}/select/?q=*&wt=json'.format(url=url)
+
+    try:
+        connection = urllib2.urlopen(search_url)
+    except urllib2.URLError as e:
+        print str(e)
+        print '[prerun] Unable to connect to solr, waiting...'
+        time.sleep(10)
+        check_solr_connection(retry=retry - 1)
+    else:
+        eval(connection.read())
+
+
+def init_db():
+
+    db_command = ['paster', '--plugin=ckan', 'db',
+                  'init', '-c', ckan_ini]
+    print '[prerun] Initializing or upgrading db - start'
+    try:
+        subprocess.check_output(db_command, stderr=subprocess.STDOUT)
+        print '[prerun] Initializing or upgrading db - end'
+    except subprocess.CalledProcessError, e:
+        if 'OperationalError' in e.output:
+            print e.output
+            print '[prerun] Database not ready, waiting a bit before exit...'
+            time.sleep(5)
+            sys.exit(1)
+        else:
+            print e.output
+            raise e
+
+
+def init_datastore_db():
+
+    conn_str = os.environ.get('CKAN_DATASTORE_WRITE_URL')
+    if not conn_str:
+        print '[prerun] Skipping datastore initialization'
+        return
+
+    datastore_perms_command = ['paster', '--plugin=ckan', 'datastore',
+                               'set-permissions', '-c', ckan_ini]
+
+    connection = psycopg2.connect(conn_str)
+    cursor = connection.cursor()
+
+    print '[prerun] Initializing datastore db - start'
+    try:
+        datastore_perms = subprocess.Popen(
+            datastore_perms_command,
+            stdout=subprocess.PIPE)
+
+        perms_sql = datastore_perms.stdout.read()
+        # Remove internal pg command as psycopg2 does not like it
+        perms_sql = re.sub('\\\\connect \"(.*)\"', '', perms_sql)
+        cursor.execute(perms_sql)
+        for notice in connection.notices:
+            print notice
+
+        connection.commit()
+
+        print '[prerun] Initializing datastore db - end'
+        print datastore_perms.stdout.read()
+    except psycopg2.Error as e:
+        print '[prerun] Could not initialize datastore'
+        print str(e)
+
+    except subprocess.CalledProcessError, e:
+        if 'OperationalError' in e.output:
+            print e.output
+            print '[prerun] Database not ready, waiting a bit before exit...'
+            time.sleep(5)
+            sys.exit(1)
+        else:
+            print e.output
+            raise e
+    finally:
+        cursor.close()
+        connection.close()
+
+
+def create_sysadmin():
+
+    name = os.environ.get('CKAN_SYSADMIN_NAME')
+    password = os.environ.get('CKAN_SYSADMIN_PASSWORD')
+    email = os.environ.get('CKAN_SYSADMIN_EMAIL')
+
+    if name and password and email:
+
+        # Check if user exists
+        command = ['paster', '--plugin=ckan', 'user', name, '-c', ckan_ini]
+
+        out = subprocess.check_output(command)
+        if 'User:None' not in re.sub(r'\s', '', out):
+            print '[prerun] Sysadmin user exists, skipping creation'
+            return
+
+        # Create user
+        command = ['paster', '--plugin=ckan', 'user', 'add',
+                   name,
+                   'password=' + password,
+                   'email=' + email,
+                   '-c', ckan_ini]
+
+        subprocess.call(command)
+        print '[prerun] Created user {0}'.format(name)
+
+        # Make it sysadmin
+        command = ['paster', '--plugin=ckan', 'sysadmin', 'add',
+                   name,
+                   '-c', ckan_ini]
+
+        subprocess.call(command)
+        print '[prerun] Made user {0} a sysadmin'.format(name)
+
+
+if __name__ == '__main__':
+
+    maintenance = os.environ.get('MAINTENANCE_MODE', '').lower() == 'true'
+
+    if maintenance:
+        print '[prerun] Maintenance mode, skipping setup...'
+    else:
+        check_main_db_connection()
+        init_db()
+        update_plugins()
+        check_datastore_db_connection()
+        init_datastore_db()
+        check_solr_connection()
+        create_sysadmin()
\ No newline at end of file
diff --git a/ckan-base/2.8/setup/start_ckan.sh b/ckan-base/2.8/setup/start_ckan.sh
new file mode 100755
index 0000000..d1c6726
--- /dev/null
+++ b/ckan-base/2.8/setup/start_ckan.sh
@@ -0,0 +1,47 @@
+#!/bin/bash
+
+# Run the prerun script to init CKAN and create the default admin user
+sudo -u ckan -EH python prerun.py
+
+# Run any startup scripts provided by images extending this one
+if [[ -d "/docker-entrypoint.d" ]]
+then
+    for f in /docker-entrypoint.d/*; do
+        case "$f" in
+            *.sh)     echo "$0: Running init file $f"; . "$f" ;;
+            *.py)     echo "$0: Running init file $f"; python "$f"; echo ;;
+            *)        echo "$0: Ignoring $f (not an sh or py file)" ;;
+        esac
+        echo
+    done
+fi
+
+# Set the common uwsgi options
+UWSGI_OPTS="--plugins http,python,gevent --socket /tmp/uwsgi.sock --uid 92 --gid 92 --http :5000 --master --enable-threads --paste config:/srv/app/production.ini --paste-logger --lazy-apps --gevent 2000 -p 2 -L"
+
+# Check whether http basic auth password protection is enabled and enable basicauth routing on uwsgi respecfully
+if [ $? -eq 0 ]
+then
+  if [ "$PASSWORD_PROTECT" = true ]
+  then
+    if [ "$HTPASSWD_USER" ] || [ "$HTPASSWD_PASSWORD" ]
+    then
+      # Generate htpasswd file for basicauth
+      htpasswd -d -b -c /srv/app/.htpasswd $HTPASSWD_USER $HTPASSWD_PASSWORD
+      # Start supervisord
+      supervisord --configuration /etc/supervisord.conf &
+      # Start uwsgi with basicauth
+      sudo -u ckan -EH uwsgi --ini /srv/app/uwsgi.conf --pcre-jit $UWSGI_OPTS
+    else
+      echo "Missing HTPASSWD_USER or HTPASSWD_PASSWORD environment variables. Exiting..."
+      exit 1
+    fi
+  else
+    # Start supervisord
+    supervisord --configuration /etc/supervisord.conf &
+    # Start uwsgi
+    sudo -u ckan -EH uwsgi $UWSGI_OPTS
+  fi
+else
+  echo "[prerun] failed...not starting CKAN."
+fi
diff --git a/ckan-base/2.8/setup/supervisor.worker.conf b/ckan-base/2.8/setup/supervisor.worker.conf
new file mode 100755
index 0000000..0ca43ba
--- /dev/null
+++ b/ckan-base/2.8/setup/supervisor.worker.conf
@@ -0,0 +1,13 @@
+[program:ckan-worker]
+command=paster --plugin=ckan jobs worker -c /srv/app/production.ini
+priority=501
+autostart=true
+autorestart=true
+redirect_stderr=true
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stdout
+stderr_logfile_maxbytes=0
+user=ckan
+environment=HOME="/srv/app",USER="ckan"
+
diff --git a/ckan-base/2.8/setup/supervisord.conf b/ckan-base/2.8/setup/supervisord.conf
new file mode 100755
index 0000000..a3f6671
--- /dev/null
+++ b/ckan-base/2.8/setup/supervisord.conf
@@ -0,0 +1,23 @@
+[unix_http_server]
+file = /tmp/supervisor.sock
+chmod = 0777
+chown = nobody:nogroup
+
+[supervisord]
+logfile = /tmp/supervisord.log
+logfile_maxbytes = 50MB
+logfile_backups=10
+loglevel = info
+pidfile = /tmp/supervisord.pid
+nodaemon = true
+umask = 022
+identifier = supervisor
+
+[supervisorctl]
+serverurl = unix:///tmp/supervisor.sock
+
+[rpcinterface:supervisor]
+supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
+
+[include]
+files = /etc/supervisord.d/*.conf
diff --git a/ckan-base/2.8/setup/uwsgi.conf b/ckan-base/2.8/setup/uwsgi.conf
new file mode 100755
index 0000000..6321d6d
--- /dev/null
+++ b/ckan-base/2.8/setup/uwsgi.conf
@@ -0,0 +1,2 @@
+[uwsgi]
+route = ^(?!/api).*$ basicauth:Restricted,/srv/app/.htpasswd
diff --git a/ckan-base/2.9/Dockerfile b/ckan-base/2.9/Dockerfile
new file mode 100755
index 0000000..45b5aae
--- /dev/null
+++ b/ckan-base/2.9/Dockerfile
@@ -0,0 +1,109 @@
+FROM alpine:3.13
+
+# Internal environment variables
+ENV APP_DIR=/srv/app
+ENV SRC_DIR=/srv/app/src
+ENV CKAN_INI=${APP_DIR}/ckan.ini
+ENV PIP_SRC=${SRC_DIR}
+ENV CKAN_STORAGE_PATH=/var/lib/ckan
+ENV GIT_URL=https://github.com/ckan/ckan.git
+# CKAN version to build
+ENV GIT_BRANCH=ckan-2.9.4
+# Customize these on the .env file if needed
+ENV CKAN_SITE_URL=http://localhost:5000
+ENV CKAN__PLUGINS image_view text_view recline_view datastore datapusher envvars
+
+WORKDIR ${APP_DIR}
+
+# Install necessary packages to run CKAN
+RUN apk add --no-cache tzdata \
+        git \
+        gettext \
+        postgresql-client \
+        python3 \
+        apache2-utils \
+        libxml2 \
+        libxslt \
+        musl-dev \
+        uwsgi-http \
+        uwsgi-corerouter \
+        uwsgi-python3 \
+        py3-gevent \
+        uwsgi-gevent \
+        libmagic \
+        curl \
+        sudo && \
+    # Packages to build CKAN requirements and plugins
+    apk add --no-cache --virtual .build-deps \
+        postgresql-dev \
+        gcc \
+        make \
+        g++ \
+        autoconf \
+        automake \
+	    libtool \
+        python3-dev \
+        py3-virtualenv \
+        libxml2-dev \
+        libxslt-dev \
+        linux-headers
+
+# Create src directory (SRC_DIR)
+RUN mkdir -p ${SRC_DIR}
+
+# Install pip
+RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/get-pip.py && \
+    python3 ${SRC_DIR}/get-pip.py
+
+# Set up Python3 virtual environment
+RUN cd ${APP_DIR} && \
+    python3 -m venv ${APP_DIR} && \
+    source ${APP_DIR}/bin/activate
+
+# Virtual environment binaries/scripts to be used first
+ENV PATH=${APP_DIR}/bin:${PATH}   
+    
+# Install CKAN code
+RUN pip3 install -e git+${GIT_URL}@${GIT_BRANCH}#egg=ckan
+
+# Install uwsgi
+RUN pip3 install uwsgi
+
+# Complete CKAN install
+RUN cd ${SRC_DIR}/ckan && \
+    cp who.ini ${APP_DIR} && \
+    pip3 install -r requirement-setuptools.txt && \
+    pip3 install --no-binary :all: -r requirements.txt && \
+    # Install CKAN envvars to support loading config from environment variables
+    pip3 install -e git+https://github.com/okfn/ckanext-envvars.git#egg=ckanext-envvars
+    
+# Create and update CKAN config
+RUN ckan generate config ${CKAN_INI}
+
+# Install and configure supervisor
+RUN pip3 install supervisor && \
+mkdir /etc/supervisord.d
+
+# Copy all setup files
+COPY 2.9/setup ${APP_DIR}
+COPY 2.9/setup/supervisor.worker.conf /etc/supervisord.d/worker.conf
+COPY 2.9/setup/supervisord.conf /etc/supervisord.conf
+
+# Create a local user and group to run the app
+RUN addgroup -g 92 -S ckan && \
+    adduser -u 92 -h /srv/app -H -D -S -G ckan ckan
+
+# Create local storage folder
+RUN mkdir -p $CKAN_STORAGE_PATH && \
+    chown -R ckan:ckan $CKAN_STORAGE_PATH
+
+# Create entrypoint directory for children image scripts
+ONBUILD RUN mkdir /docker-entrypoint.d
+
+RUN chown ckan -R /srv/app
+
+EXPOSE 5000
+
+HEALTHCHECK --interval=10s --timeout=5s --retries=5 CMD curl --fail http://localhost:5000/api/3/action/status_show || exit 1
+
+CMD ["/srv/app/start_ckan.sh"] 
\ No newline at end of file
diff --git a/ckan-base/2.9/setup/ckan-uwsgi.ini b/ckan-base/2.9/setup/ckan-uwsgi.ini
new file mode 100644
index 0000000..2361f36
--- /dev/null
+++ b/ckan-base/2.9/setup/ckan-uwsgi.ini
@@ -0,0 +1,15 @@
+[uwsgi]
+http-socket     =  :5000
+uid             =  ckan
+guid            =  ckan
+plugins         =  python3
+wsgi-file       =  /srv/app/wsgi.py
+virtualenv      =  /srv/app
+module          =  wsgi:application
+master          =  true
+processes       =  5
+pidfile         =  /tmp/%n.pid
+harakiri        =  50
+max-requests    =  5000
+vacuum          =  true
+callable        =  application
diff --git a/ckan-base/2.9/setup/prerun.py b/ckan-base/2.9/setup/prerun.py
new file mode 100755
index 0000000..f8dd693
--- /dev/null
+++ b/ckan-base/2.9/setup/prerun.py
@@ -0,0 +1,194 @@
+import os
+import sys
+import subprocess
+import psycopg2
+import urllib3
+import time
+import re
+
+ckan_ini = os.environ.get('CKAN_INI', '/srv/app/ckan.ini')
+
+RETRY = 5
+
+def update_plugins():
+
+    plugins = os.environ.get('CKAN__PLUGINS', '')
+    print('[prerun] Setting the following plugins in {}:'.format(ckan_ini))
+    print(plugins)
+    cmd = ['ckan', 'config-tool', ckan_ini,
+           'ckan.plugins = {}'.format(plugins)]
+    subprocess.check_output(cmd, stderr=subprocess.STDOUT)
+    print('[prerun] Plugins set.')
+
+    
+def check_main_db_connection(retry=None):
+
+    conn_str = os.environ.get('CKAN_SQLALCHEMY_URL')
+    if not conn_str:
+        print('[prerun] CKAN_SQLALCHEMY_URL not defined, not checking db')
+    return check_db_connection(conn_str, retry)
+
+
+def check_datastore_db_connection(retry=None):
+
+    conn_str = os.environ.get('CKAN_DATASTORE_WRITE_URL')
+    if not conn_str:
+        print('[prerun] CKAN_DATASTORE_WRITE_URL not defined, not checking db')
+    return check_db_connection(conn_str, retry)
+
+
+def check_db_connection(conn_str, retry=None):
+
+    if retry is None:
+        retry = RETRY
+    elif retry == 0:
+        print('[prerun] Giving up after 5 tries...')
+        sys.exit(1)
+
+    try:
+        connection = psycopg2.connect(conn_str)
+
+    except psycopg2.Error as e:
+        print(str(e))
+        print('[prerun] Unable to connect to the database, waiting...')
+        time.sleep(10)
+        check_db_connection(conn_str, retry=retry - 1)
+    else:
+        connection.close()
+
+        
+def check_solr_connection(retry=None):
+
+    if retry is None:
+        retry = RETRY
+    elif retry == 0:
+        print('[prerun] Giving up after 5 tries...')
+        sys.exit(1)
+
+    url = os.environ.get('CKAN_SOLR_URL', '')
+    search_url = '{url}/select/?q=*&wt=json'.format(url=url)
+    http = urllib3.PoolManager()
+    try:
+        r = http.request('GET', search_url)
+    except urllib3.exceptions.ConnectionError as e:
+        print(str(e))
+        print('[prerun] Unable to connect to solr, waiting...')
+        time.sleep(10)
+        check_solr_connection(retry=retry - 1)
+    else:
+        print('[prerun] Connection Status from SOLR is ', (r.status))
+
+def init_db():
+
+    db_command = ['ckan', '-c', ckan_ini, 
+                  'db', 'init']
+    print('[prerun] Initializing or upgrading db - start')
+    try:
+        subprocess.check_output(db_command, stderr=subprocess.STDOUT)
+        print('[prerun] Initializing or upgrading db - end')
+    except subprocess.CalledProcessError as e:
+        if 'OperationalError' in e.output:
+            print(e.output)
+            print('[prerun] Database not ready, waiting a bit before exit...')
+            time.sleep(5)
+            sys.exit(1)
+        else:
+            print(e.output)
+            raise e
+
+
+def init_datastore_db():
+
+    conn_str = os.environ.get('CKAN_DATASTORE_WRITE_URL')
+    if not conn_str:
+        print('[prerun] Skipping datastore initialization')
+        return
+
+    datastore_perms_command = ['ckan', '-c', ckan_ini, 
+                               'datastore', 'set-permissions']
+
+    connection = psycopg2.connect(conn_str)
+    cursor = connection.cursor()
+
+    print('[prerun] Initializing datastore db - start')
+    try:
+        datastore_perms = subprocess.Popen(
+            datastore_perms_command,
+            stdout=subprocess.PIPE)
+
+        perms_sql = datastore_perms.stdout.read().decode('utf-8')
+        # Remove internal pg command as psycopg2 does not like it
+        perms_sql = re.sub('\\\\connect \"(.*)\"', '', perms_sql)
+        cursor.execute(perms_sql)
+        for notice in connection.notices:
+            print(notice)
+
+        connection.commit()
+
+        print('[prerun] Initializing datastore db - end')
+        print(datastore_perms.stdout.read().decode('utf-8'))
+    except psycopg2.Error as e:
+        print('[prerun] Could not initialize datastore')
+        print(str(e))
+
+    except subprocess.CalledProcessError as e:
+        if 'OperationalError' in e.output:
+            print(e.output)
+            print('[prerun] Database not ready, waiting a bit before exit...')
+            time.sleep(5)
+            sys.exit(1)
+        else:
+            print(e.output)
+            raise e
+    finally:
+        cursor.close()
+        connection.close()
+
+
+def create_sysadmin():
+
+    name = os.environ.get('CKAN_SYSADMIN_NAME')
+    password = os.environ.get('CKAN_SYSADMIN_PASSWORD')
+    email = os.environ.get('CKAN_SYSADMIN_EMAIL')
+
+    if name and password and email:
+
+        # Check if user exists
+        command = ['ckan', '-c', ckan_ini, 'user', 'show', name,]
+
+        out = subprocess.check_output(command)
+        if 'User:None' not in re.sub(r'\s', '', out.decode()):
+            print('[prerun] Sysadmin user exists, skipping creation')
+            return
+
+        # Create user
+        command = ['ckan', '-c', ckan_ini, 'user', 'add',
+                   name,
+                   'password=' + password,
+                   'email=' + email]
+
+        subprocess.call(command)
+        print('[prerun] Created user {0}'.format(name))
+
+        # Make it sysadmin
+        command = ['ckan', '-c', ckan_ini, 'sysadmin', 'add',
+                   name]
+
+        subprocess.call(command)
+        print('[prerun] Made user {0} a sysadmin'.format(name))
+
+
+if __name__ == '__main__':
+
+    maintenance = os.environ.get('MAINTENANCE_MODE', '').lower() == 'true'
+
+    if maintenance:
+        print('[prerun] Maintenance mode, skipping setup...')
+    else:
+        check_main_db_connection()
+        init_db()
+        update_plugins()
+        check_datastore_db_connection()
+        init_datastore_db()
+        check_solr_connection()
+        create_sysadmin()
diff --git a/ckan-base/2.9/setup/start_ckan.sh b/ckan-base/2.9/setup/start_ckan.sh
new file mode 100755
index 0000000..5a119e6
--- /dev/null
+++ b/ckan-base/2.9/setup/start_ckan.sh
@@ -0,0 +1,46 @@
+#!/bin/bash
+
+# Run the prerun script to init CKAN and create the default admin user
+sudo -u ckan -EH python3 prerun.py
+
+# Run any startup scripts provided by images extending this one
+if [[ -d "/docker-entrypoint.d" ]]
+then
+    for f in /docker-entrypoint.d/*; do
+        case "$f" in
+            *.sh)     echo "$0: Running init file $f"; . "$f" ;;
+            *.py)     echo "$0: Running init file $f"; python "$f"; echo ;;
+            *)        echo "$0: Ignoring $f (not an sh or py file)" ;;
+        esac
+        echo
+    done
+fi
+
+
+# Check whether http basic auth password protection is enabled and enable basicauth routing on uwsgi respecfully
+if [ $? -eq 0 ]
+then
+  if [ "$PASSWORD_PROTECT" = true ]
+  then
+    if [ "$HTPASSWD_USER" ] || [ "$HTPASSWD_PASSWORD" ]
+    then
+      # Generate htpasswd file for basicauth
+      htpasswd -d -b -c /srv/app/.htpasswd $HTPASSWD_USER $HTPASSWD_PASSWORD
+      # Start supervisord
+      supervisord --configuration /etc/supervisord.conf &
+      # Start uwsgi with basicauth
+      sudo -u ckan -EH uwsgi -i ckan-uwsgi.ini
+    else
+      echo "Missing HTPASSWD_USER or HTPASSWD_PASSWORD environment variables. Exiting..."
+      exit 1
+    fi
+  else
+    # Start supervisord
+    supervisord --configuration /etc/supervisord.conf &
+    # Start uwsgi
+    sudo -u ckan -EH uwsgi -i ckan-uwsgi.ini
+  fi
+else
+  echo "[prerun] failed...not starting CKAN."
+fi
+
diff --git a/ckan-base/2.9/setup/supervisor.worker.conf b/ckan-base/2.9/setup/supervisor.worker.conf
new file mode 100644
index 0000000..9d46f37
--- /dev/null
+++ b/ckan-base/2.9/setup/supervisor.worker.conf
@@ -0,0 +1,12 @@
+[program:ckan-worker]
+command=ckan -c /srv/app/ckan.ini jobs worker
+priority=501
+autostart=true
+autorestart=true
+redirect_stderr=true
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stdout
+stderr_logfile_maxbytes=0
+user=ckan
+environment=HOME="/srv/app",USER="ckan"
\ No newline at end of file
diff --git a/ckan-base/2.9/setup/supervisord.conf b/ckan-base/2.9/setup/supervisord.conf
new file mode 100644
index 0000000..052dbc5
--- /dev/null
+++ b/ckan-base/2.9/setup/supervisord.conf
@@ -0,0 +1,23 @@
+[unix_http_server]
+file = /tmp/supervisor.sock
+chmod = 0777
+chown = nobody:nogroup
+
+[supervisord]
+logfile = /tmp/supervisord.log
+logfile_maxbytes = 50MB
+logfile_backups=10
+loglevel = info
+pidfile = /tmp/supervisord.pid
+nodaemon = true
+umask = 022
+identifier = supervisor
+
+[supervisorctl]
+serverurl = unix:///tmp/supervisor.sock
+
+[rpcinterface:supervisor]
+supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
+
+[include]
+files = /etc/supervisord.d/*.conf
\ No newline at end of file
diff --git a/ckan-base/2.9/setup/wsgi.py b/ckan-base/2.9/setup/wsgi.py
new file mode 100644
index 0000000..b37d80e
--- /dev/null
+++ b/ckan-base/2.9/setup/wsgi.py
@@ -0,0 +1,9 @@
+import os
+from ckan.config.middleware import make_app
+from ckan.cli import CKANConfigLoader
+from logging.config import fileConfig as loggingFileConfig
+config_filepath = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'ckan.ini')
+abspath = os.path.join(os.path.dirname(os.path.abspath(__file__)))
+loggingFileConfig(config_filepath)
+config = CKANConfigLoader(config_filepath).get_config()
+application = make_app(config)
diff --git a/ckan-dev/2.7/Dockerfile b/ckan-dev/2.7/Dockerfile
new file mode 100755
index 0000000..a421a59
--- /dev/null
+++ b/ckan-dev/2.7/Dockerfile
@@ -0,0 +1,20 @@
+FROM ckan/ckan-base:testing-only.2.7
+
+LABEL maintainer="brett@kowh.ai"
+
+ENV APP_DIR=/srv/app
+ENV SRC_EXTENSIONS_DIR=/srv/app/src_extensions
+
+# Install packages needed by the dev requirements
+RUN apk add --no-cache libffi-dev
+
+# Install CKAN dev requirements
+RUN pip install --no-binary :all: -r https://raw.githubusercontent.com/ckan/ckan/${GIT_BRANCH}/dev-requirements.txt
+
+# Create folder for local extensions sources
+RUN mkdir $SRC_EXTENSIONS_DIR
+
+COPY setup/start_ckan_development.sh ${APP_DIR}
+
+
+CMD ["/srv/app/start_ckan_development.sh"]
diff --git a/ckan-dev/2.8/Dockerfile b/ckan-dev/2.8/Dockerfile
new file mode 100755
index 0000000..6142a8d
--- /dev/null
+++ b/ckan-dev/2.8/Dockerfile
@@ -0,0 +1,20 @@
+FROM ckan/ckan-base:testing-only.2.8
+
+LABEL maintainer="brett@kowh.ai"
+
+ENV APP_DIR=/srv/app
+ENV SRC_EXTENSIONS_DIR=/srv/app/src_extensions
+
+# Install packages needed by the dev requirements
+RUN apk add --no-cache libffi-dev
+
+# Install CKAN dev requirements
+RUN pip install --no-binary :all: -r https://raw.githubusercontent.com/ckan/ckan/${GIT_BRANCH}/dev-requirements.txt
+
+# Create folder for local extensions sources
+RUN mkdir $SRC_EXTENSIONS_DIR
+
+COPY setup/start_ckan_development.sh ${APP_DIR}
+
+
+CMD ["/srv/app/start_ckan_development.sh"]
diff --git a/ckan-dev/2.9/Dockerfile b/ckan-dev/2.9/Dockerfile
new file mode 100755
index 0000000..fc170ee
--- /dev/null
+++ b/ckan-dev/2.9/Dockerfile
@@ -0,0 +1,29 @@
+FROM ckan/ckan-base:testing-only.2.9
+
+LABEL maintainer="brett@kowh.ai"
+
+ENV APP_DIR=/srv/app
+ENV SRC_EXTENSIONS_DIR=/srv/app/src_extensions
+
+# Install packages needed by the dev requirements
+RUN apk add --no-cache libffi-dev
+
+# Set up Python3 virtual environment
+RUN cd ${APP_DIR} && \
+    source ${APP_DIR}/bin/activate
+
+# Virtual environment binaries/scripts to be used first
+ENV PATH=${APP_DIR}/bin:${PATH} 
+
+# Install CKAN dev requirements
+# Will need to change this eventually - when CKAN 2.9 is out
+# wget https://raw.githubusercontent.com/ckan/ckan/master/dev-requirements.txt
+# RUN pip3 install --no-binary :all: -r https://raw.githubusercontent.com/ckan/ckan/master/dev-requirements.txt
+RUN pip3 install -r https://raw.githubusercontent.com/ckan/ckan/master/dev-requirements.txt
+
+# Create folder for local extensions sources
+RUN mkdir $SRC_EXTENSIONS_DIR
+
+COPY setup/start_ckan_development.sh ${APP_DIR}
+
+CMD ["/srv/app/start_ckan_development.sh"]
diff --git a/ckan-dev/setup/start_ckan_development.sh b/ckan-dev/setup/start_ckan_development.sh
new file mode 100755
index 0000000..331266c
--- /dev/null
+++ b/ckan-dev/setup/start_ckan_development.sh
@@ -0,0 +1,82 @@
+#!/bin/bash
+
+# Install any local extensions in the src_extensions volume
+echo "Looking for local extensions to install..."
+echo "Extension dir contents:"
+ls -la $SRC_EXTENSIONS_DIR
+for i in $SRC_EXTENSIONS_DIR/*
+do
+    if [ -d $i ];
+    then
+
+        if [ -f $i/pip-requirements.txt ];
+        then
+            pip install -r $i/pip-requirements.txt
+            echo "Found requirements file in $i"
+        fi
+        if [ -f $i/requirements.txt ];
+        then
+            pip install -r $i/requirements.txt
+            echo "Found requirements file in $i"
+        fi
+        if [ -f $i/dev-requirements.txt ];
+        then
+            pip install -r $i/dev-requirements.txt
+            echo "Found dev-requirements file in $i"
+        fi
+        if [ -f $i/setup.py ];
+        then
+            cd $i
+            python $i/setup.py develop
+            echo "Found setup.py file in $i"
+            cd $APP_DIR
+        fi
+
+        # Point `use` in test.ini to location of `test-core.ini`
+        if [ -f $i/test.ini ];
+        then
+            echo "Updating \`test.ini\` reference to \`test-core.ini\` for plugin $i"
+            paster --plugin=ckan config-tool $i/test.ini "use = config:../../src/ckan/test-core.ini"
+        fi
+    fi
+done
+
+# Set debug to true
+echo "Enabling debug mode"
+ckan config-tool $CKAN_INI -s DEFAULT "debug = true"
+
+# Update the plugins setting in the ini file with the values defined in the env var
+echo "Loading the following plugins: $CKAN__PLUGINS"
+ckan config-tool $CKAN_INI "ckan.plugins = $CKAN__PLUGINS"
+
+# Update test-core.ini DB, SOLR & Redis settings
+echo "Loading test settings into test-core.ini"
+ckan config-tool $SRC_DIR/ckan/test-core.ini \
+    "sqlalchemy.url = $TEST_CKAN_SQLALCHEMY_URL" \
+    "ckan.datstore.write_url = $TEST_CKAN_DATASTORE_WRITE_URL" \
+    "ckan.datstore.read_url = $TEST_CKAN_DATASTORE_READ_URL" \
+    "solr_url = $TEST_CKAN_SOLR_URL" \
+    "ckan.redis_url = $TEST_CKAN_REDIS_URL"
+
+# Run the prerun script to init CKAN and create the default admin user
+sudo -u ckan -EH python prerun.py
+
+# Run any startup scripts provided by images extending this one
+if [[ -d "/docker-entrypoint.d" ]]
+then
+    for f in /docker-entrypoint.d/*; do
+        case "$f" in
+            *.sh)     echo "$0: Running init file $f"; . "$f" ;;
+            *.py)     echo "$0: Running init file $f"; python "$f"; echo ;;
+            *)        echo "$0: Ignoring $f (not an sh or py file)" ;;
+        esac
+        echo
+    done
+fi
+
+# Start supervisord
+supervisord --configuration /etc/supervisord.conf &
+
+# Start the development server with automatic reload
+# Check the --reloader options sudo -u ckan -EH ckan -c $CKAN_INI run --reloader <TEXT>
+sudo -u ckan -EH ckan -c $CKAN_INI run

From e7075b9c345802c46b817e65986435cfd4455987 Mon Sep 17 00:00:00 2001
From: Brett <brett@kowh.ai>
Date: Fri, 29 Oct 2021 13:55:58 +0200
Subject: [PATCH 02/10] commit ckan directory changes

---
 .gitkeep                                      |  0
 Dockerfile                                    | 55 +++++++++++++++++++
 Dockerfile.dev                                | 37 +++++++++++++
 patches/00_patch_sql_url.patch                | 11 ++++
 .../01_patch_resource_replace_upload.patch    | 11 ++++
 .../02_patch_postgres_username_split.patch    | 11 ++++
 scripts/apply_ckan_patches.sh                 |  5 ++
 7 files changed, 130 insertions(+)
 create mode 100644 .gitkeep
 create mode 100644 Dockerfile
 create mode 100644 Dockerfile.dev
 create mode 100644 patches/00_patch_sql_url.patch
 create mode 100644 patches/01_patch_resource_replace_upload.patch
 create mode 100644 patches/02_patch_postgres_username_split.patch
 create mode 100755 scripts/apply_ckan_patches.sh

diff --git a/.gitkeep b/.gitkeep
new file mode 100644
index 0000000..e69de29
diff --git a/Dockerfile b/Dockerfile
new file mode 100644
index 0000000..e13b160
--- /dev/null
+++ b/Dockerfile
@@ -0,0 +1,55 @@
+FROM ckan/ckan-base:testing-only.2.9
+
+LABEL maintainer="brett@kowh.ai"
+
+#RUN apk update \
+#    && apk upgrade \
+#    && apk add --no-cache libffi-dev \
+#    libmagic
+    
+
+# Set up environment variables
+ENV APP_DIR=/srv/app
+ENV TZ=UTC
+ENV CRYPTOGRAPHY_DONT_BUILD_RUST=1
+
+RUN echo ${TZ} > /etc/timezone
+
+# Make sure both files are not exactly the same
+RUN if ! [ /usr/share/zoneinfo/${TZ} -ef /etc/localtime ]; then \
+        cp /usr/share/zoneinfo/${TZ} /etc/localtime ;\
+    fi ;
+
+#RUN pip3 install -e 'git+https://github.com/DataShades/ckanext-xloader@py3#egg=ckanext-xloader'
+#RUN pip3 install -r ${APP_DIR}/src/ckanext-xloader/requirements.txt
+#RUN pip3 install -U requests[security]
+
+# Install any extensions needed by your CKAN instance
+# (Make sure to add the plugins to CKAN__PLUGINS in the .env file)
+# For instance:
+#RUN pip install -e git+https://github.com/ckan/ckanext-pages.git#egg=ckanext-pages && \
+#    pip install -e git+https://github.com/ckan/ckanext-dcat.git@v0.0.6#egg=ckanext-dcat && \
+#    pip install -r https://raw.githubusercontent.com/ckan/ckanext-dcat/v0.0.6/requirements.txt
+
+# Install the extension(s) you wrote for your own project
+# RUN pip install -e git+https://github.com/your-org/ckanext-your-extension.git@v1.0.0#egg=ckanext-your-extension
+
+# Apply any patches needed to CKAN core or any of the built extensions (not the
+# runtime mounted ones)
+# See https://github.com/okfn/docker-ckan#applying-patches
+
+#COPY patches ${APP_DIR}/patches
+
+# Copy patches and apply patches script
+COPY ./patches ${SRC_DIR}/patches
+COPY ./scripts/apply_ckan_patches.sh ${SRC_DIR}/apply_ckan_patches.sh
+# Apply patches
+#RUN ${SRC_DIR}/apply_ckan_patches.sh
+
+RUN for d in ${APP_DIR}/patches/*; do \
+        if [ -d $d ]; then \
+            for f in `ls $d/*.patch | sort -g`; do \
+                cd $SRC_DIR/`basename "$d"` && echo "$0: Applying patch $f to $SRC_DIR/`basename $d`"; patch -p1 < "$f" ; \
+            done ; \
+        fi ; \
+    done
diff --git a/Dockerfile.dev b/Dockerfile.dev
new file mode 100644
index 0000000..6086f6e
--- /dev/null
+++ b/Dockerfile.dev
@@ -0,0 +1,37 @@
+FROM ckan/ckan-dev:testing-only.2.9
+
+LABEL maintainer="brett@kowh.ai"
+
+# Set up environment variables
+ENV APP_DIR=/srv/app
+ENV TZ=UTC
+RUN echo ${TZ} > /etc/timezone
+
+# Make sure both files are not exactly the same
+RUN if ! [ /usr/share/zoneinfo/${TZ} -ef /etc/localtime ]; then \
+        cp /usr/share/zoneinfo/${TZ} /etc/localtime ;\
+    fi ;
+
+# Install any extensions needed by your CKAN instance
+# (Make sure to add the plugins to CKAN__PLUGINS in the .env file)
+# For instance:
+#RUN pip install -e git+https://github.com/ckan/ckanext-pages.git#egg=ckanext-pages && \
+#    pip install -e git+https://github.com/ckan/ckanext-dcat.git@v0.0.6#egg=ckanext-dcat && \
+#    pip install -r https://raw.githubusercontent.com/ckan/ckanext-dcat/v0.0.6/requirements.txt
+
+# Clone the extension(s) your are writing for your own project in the `src` folder
+# to get them mounted in this image at runtime
+
+# Apply any patches needed to CKAN core or any of the built extensions (not the
+# runtime mounted ones)
+# See https://github.com/okfn/docker-ckan#applying-patches
+
+COPY patches ${APP_DIR}/patches
+
+RUN for d in ${APP_DIR}/patches/*; do \
+        if [ -d $d ]; then \
+            for f in `ls $d/*.patch | sort -g`; do \
+                cd $SRC_DIR/`basename "$d"` && echo "$0: Applying patch $f to $SRC_DIR/`basename $d`"; patch -p1 < "$f" ; \
+            done ; \
+        fi ; \
+    done
diff --git a/patches/00_patch_sql_url.patch b/patches/00_patch_sql_url.patch
new file mode 100644
index 0000000..46a7135
--- /dev/null
+++ b/patches/00_patch_sql_url.patch
@@ -0,0 +1,11 @@
+--- ckan/ckan/model/__init__.py	2021-02-16 14:47:06.168327441 +0100
++++ ckan/ckan/model/__init__.py	2021-02-16 14:48:00.740780218 +0100
+@@ -266,7 +266,7 @@
+         self.reset_alembic_output()
+         alembic_config = AlembicConfig(self._alembic_ini)
+         alembic_config.set_main_option(
+-            "sqlalchemy.url", str(self.metadata.bind.url)
++            "sqlalchemy.url", str(self.metadata.bind.url).replace('%', '%%')
+         )
+         try:
+             sqlalchemy_migrate_version = self.metadata.bind.execute(
diff --git a/patches/01_patch_resource_replace_upload.patch b/patches/01_patch_resource_replace_upload.patch
new file mode 100644
index 0000000..3f11fac
--- /dev/null
+++ b/patches/01_patch_resource_replace_upload.patch
@@ -0,0 +1,11 @@
+--- ckan/ckan/logic/action/update.py	2021-02-17 16:46:55.673578728 +0100
++++ ckan/ckan/logic/action/update-edit.py	2021-02-17 16:47:28.905879170 +0100
+@@ -929,7 +929,7 @@
+ 
+     '''
+     model = context['model']
+-    session = model.Session
++    session = model.meta.create_local_session()
+     context['session'] = session
+ 
+     user = context['user']
diff --git a/patches/02_patch_postgres_username_split.patch b/patches/02_patch_postgres_username_split.patch
new file mode 100644
index 0000000..780b432
--- /dev/null
+++ b/patches/02_patch_postgres_username_split.patch
@@ -0,0 +1,11 @@
+--- ckan/ckanext/datastore/backend/postgres.py	2021-02-18 11:01:56.692267462 +0100
++++ ckan/ckanext/datastore/backend/postgres-patch.py	2021-02-18 13:45:16.033193435 +0100
+@@ -1690,7 +1690,7 @@
+         read only user.
+         '''
+         write_connection = self._get_write_engine().connect()
+-        read_connection_user = sa_url.make_url(self.read_url).username
++        read_connection_user = sa_url.make_url(self.read_url).username.split("@")[0]
+ 
+         drop_foo_sql = u'DROP TABLE IF EXISTS _foo'
+ 
diff --git a/scripts/apply_ckan_patches.sh b/scripts/apply_ckan_patches.sh
new file mode 100755
index 0000000..a7bceb9
--- /dev/null
+++ b/scripts/apply_ckan_patches.sh
@@ -0,0 +1,5 @@
+#!/bin/bash
+shopt -s nullglob
+for patch in patches/*.patch; do
+    /usr/bin/patch -p0 -i $patch
+done

From 15eaab514bc80de5bb8f8bff4818c42d7e07dfb9 Mon Sep 17 00:00:00 2001
From: Brett <brett@kowh.ai>
Date: Fri, 29 Oct 2021 13:56:23 +0200
Subject: [PATCH 03/10] commit ckan-ext directory changes

---
 Dockerfile                   | 146 ++++++++++++++++++--------
 setup/ckan-uwsgi.ini         |  15 +++
 setup/prerun.py              | 194 +++++++++++++++++++++++++++++++++++
 setup/start_ckan.sh          |  46 +++++++++
 setup/supervisor.worker.conf |  12 +++
 setup/supervisord.conf       |  23 +++++
 setup/wsgi.py                |   9 ++
 7 files changed, 401 insertions(+), 44 deletions(-)
 mode change 100644 => 100755 Dockerfile
 create mode 100644 setup/ckan-uwsgi.ini
 create mode 100755 setup/prerun.py
 create mode 100755 setup/start_ckan.sh
 create mode 100644 setup/supervisor.worker.conf
 create mode 100644 setup/supervisord.conf
 create mode 100644 setup/wsgi.py

diff --git a/Dockerfile b/Dockerfile
old mode 100644
new mode 100755
index e13b160..0aa8caa
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,55 +1,113 @@
-FROM ckan/ckan-base:testing-only.2.9
+FROM alpine:3.7
 
-LABEL maintainer="brett@kowh.ai"
-
-#RUN apk update \
-#    && apk upgrade \
-#    && apk add --no-cache libffi-dev \
-#    libmagic
-    
-
-# Set up environment variables
+# Internal environment variables
 ENV APP_DIR=/srv/app
-ENV TZ=UTC
-ENV CRYPTOGRAPHY_DONT_BUILD_RUST=1
+ENV SRC_DIR=/srv/app/src
+ENV CKAN_INI=${APP_DIR}/ckan.ini
+ENV PIP_SRC=${SRC_DIR}
+ENV CKAN_STORAGE_PATH=/var/lib/ckan
+ENV GIT_URL=https://github.com/ckan/ckan.git
+# CKAN version to build
+ENV GIT_BRANCH=ckan-2.9.3
+# Customize these on the .env file if needed
+ENV CKAN_SITE_URL=http://localhost:5000
+ENV CKAN__PLUGINS image_view text_view recline_view datastore datapusher envvars
 
-RUN echo ${TZ} > /etc/timezone
+WORKDIR ${APP_DIR}
 
-# Make sure both files are not exactly the same
-RUN if ! [ /usr/share/zoneinfo/${TZ} -ef /etc/localtime ]; then \
-        cp /usr/share/zoneinfo/${TZ} /etc/localtime ;\
-    fi ;
+# Install necessary packages to run CKAN
+RUN apk add --no-cache tzdata \
+        git \
+        gettext \
+        postgresql-client \
+        python3 \
+        apache2-utils \
+        libxml2 \
+        libxslt \
+        musl-dev \
+        uwsgi-http \
+        uwsgi-corerouter \
+        uwsgi-python3 \
+        py3-gevent \
+        uwsgi-gevent \
+        libmagic \
+        curl \
+        sudo && \
+    # Packages to build CKAN requirements and plugins
+    apk add --no-cache --virtual .build-deps \
+        postgresql-dev \
+        gcc \
+        make \
+        g++ \
+        autoconf \
+        automake \
+	    libtool \
+        python3-dev \
+        py3-virtualenv \
+        libxml2-dev \
+        libxslt-dev \
+        linux-headers && \
+        # Create SRC_DIR
+        mkdir -p ${SRC_DIR}
 
-#RUN pip3 install -e 'git+https://github.com/DataShades/ckanext-xloader@py3#egg=ckanext-xloader'
-#RUN pip3 install -r ${APP_DIR}/src/ckanext-xloader/requirements.txt
-#RUN pip3 install -U requests[security]
+# Install pip
+RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/get-pip.py && \
+    python3 ${SRC_DIR}/get-pip.py
 
-# Install any extensions needed by your CKAN instance
-# (Make sure to add the plugins to CKAN__PLUGINS in the .env file)
-# For instance:
-#RUN pip install -e git+https://github.com/ckan/ckanext-pages.git#egg=ckanext-pages && \
-#    pip install -e git+https://github.com/ckan/ckanext-dcat.git@v0.0.6#egg=ckanext-dcat && \
-#    pip install -r https://raw.githubusercontent.com/ckan/ckanext-dcat/v0.0.6/requirements.txt
+# Set up Python3 virtual environment
+RUN cd ${APP_DIR} && \
+    python3 -m venv ${APP_DIR} && \
+    source ${APP_DIR}/bin/activate
 
-# Install the extension(s) you wrote for your own project
-# RUN pip install -e git+https://github.com/your-org/ckanext-your-extension.git@v1.0.0#egg=ckanext-your-extension
+# Virtual environment binaries/scripts to be used first
+ENV PATH=${APP_DIR}/bin:${PATH}   
+    
+# Install CKAN, uwsgi plus extensions
+RUN pip3 install -e git+${GIT_URL}@${GIT_BRANCH}#egg=ckan && \ 
+    pip3 install uwsgi && \
+    cd ${SRC_DIR}/ckan && \
+    cp who.ini ${APP_DIR} && \
+    pip install --no-binary :all: -r requirements.txt && \
+    # Install CKAN envvars to support loading config from environment variables
+    pip3 install -e git+https://github.com/okfn/ckanext-envvars.git#egg=ckanext-envvars && \
+    # Install CKAN extensions
+    pip3 install -e 'git+https://github.com/DataShades/ckanext-xloader@py3#egg=ckanext-xloader' && \
+    pip3 install -r $CKAN_VENV/src/ckanext-xloader/requirements.txt && \
+    pip3 install -U requests[security] && \
+    pip3 install -e 'git+https://github.com/DataShades/ckanext-harvest.git@py3#egg=ckanext-harvest' && \
+    pip3 install -r $CKAN_VENV/src/ckanext-harvest/pip-requirements.txt && \
+    pip3 install -e 'git+https://github.com/DataShades/ckanext-syndicate@py3#egg=ckanext-syndicate' && \
+    pip3 install -r $CKAN_VENV/src/ckanext-syndicate/requirements.txt && \
+    pip3 install -e 'git+https://github.com/ckan/ckanext-scheming.git@master#egg=ckanext-scheming' && \
+    pip3 install -r $CKAN_VENV/src/ckanext-scheming/requirements.txt
 
-# Apply any patches needed to CKAN core or any of the built extensions (not the
-# runtime mounted ones)
-# See https://github.com/okfn/docker-ckan#applying-patches
+# Create and update CKAN config
+RUN ckan generate config ${CKAN_INI}
 
-#COPY patches ${APP_DIR}/patches
+# Install and configure supervisor
+RUN pip3 install supervisor && \
+mkdir /etc/supervisord.d
 
-# Copy patches and apply patches script
-COPY ./patches ${SRC_DIR}/patches
-COPY ./scripts/apply_ckan_patches.sh ${SRC_DIR}/apply_ckan_patches.sh
-# Apply patches
-#RUN ${SRC_DIR}/apply_ckan_patches.sh
+# Copy all setup files
+COPY setup ${APP_DIR}
+COPY setup/supervisor.worker.conf /etc/supervisord.d/worker.conf
+COPY setup/supervisord.conf /etc/supervisord.conf
 
-RUN for d in ${APP_DIR}/patches/*; do \
-        if [ -d $d ]; then \
-            for f in `ls $d/*.patch | sort -g`; do \
-                cd $SRC_DIR/`basename "$d"` && echo "$0: Applying patch $f to $SRC_DIR/`basename $d`"; patch -p1 < "$f" ; \
-            done ; \
-        fi ; \
-    done
+# Create a local user and group to run the app
+RUN addgroup -g 92 -S ckan && \
+    adduser -u 92 -h /srv/app -H -D -S -G ckan ckan
+
+# Create local storage folder
+RUN mkdir -p $CKAN_STORAGE_PATH && \
+    chown -R ckan:ckan $CKAN_STORAGE_PATH
+
+# Create entrypoint directory for children image scripts
+ONBUILD RUN mkdir /docker-entrypoint.d
+
+RUN chown ckan -R /srv/app
+
+EXPOSE 5000
+
+HEALTHCHECK --interval=10s --timeout=5s --retries=5 CMD curl --fail http://localhost:5000/api/3/action/status_show || exit 1
+
+CMD ["/srv/app/start_ckan.sh"] 
\ No newline at end of file
diff --git a/setup/ckan-uwsgi.ini b/setup/ckan-uwsgi.ini
new file mode 100644
index 0000000..2361f36
--- /dev/null
+++ b/setup/ckan-uwsgi.ini
@@ -0,0 +1,15 @@
+[uwsgi]
+http-socket     =  :5000
+uid             =  ckan
+guid            =  ckan
+plugins         =  python3
+wsgi-file       =  /srv/app/wsgi.py
+virtualenv      =  /srv/app
+module          =  wsgi:application
+master          =  true
+processes       =  5
+pidfile         =  /tmp/%n.pid
+harakiri        =  50
+max-requests    =  5000
+vacuum          =  true
+callable        =  application
diff --git a/setup/prerun.py b/setup/prerun.py
new file mode 100755
index 0000000..f8dd693
--- /dev/null
+++ b/setup/prerun.py
@@ -0,0 +1,194 @@
+import os
+import sys
+import subprocess
+import psycopg2
+import urllib3
+import time
+import re
+
+ckan_ini = os.environ.get('CKAN_INI', '/srv/app/ckan.ini')
+
+RETRY = 5
+
+def update_plugins():
+
+    plugins = os.environ.get('CKAN__PLUGINS', '')
+    print('[prerun] Setting the following plugins in {}:'.format(ckan_ini))
+    print(plugins)
+    cmd = ['ckan', 'config-tool', ckan_ini,
+           'ckan.plugins = {}'.format(plugins)]
+    subprocess.check_output(cmd, stderr=subprocess.STDOUT)
+    print('[prerun] Plugins set.')
+
+    
+def check_main_db_connection(retry=None):
+
+    conn_str = os.environ.get('CKAN_SQLALCHEMY_URL')
+    if not conn_str:
+        print('[prerun] CKAN_SQLALCHEMY_URL not defined, not checking db')
+    return check_db_connection(conn_str, retry)
+
+
+def check_datastore_db_connection(retry=None):
+
+    conn_str = os.environ.get('CKAN_DATASTORE_WRITE_URL')
+    if not conn_str:
+        print('[prerun] CKAN_DATASTORE_WRITE_URL not defined, not checking db')
+    return check_db_connection(conn_str, retry)
+
+
+def check_db_connection(conn_str, retry=None):
+
+    if retry is None:
+        retry = RETRY
+    elif retry == 0:
+        print('[prerun] Giving up after 5 tries...')
+        sys.exit(1)
+
+    try:
+        connection = psycopg2.connect(conn_str)
+
+    except psycopg2.Error as e:
+        print(str(e))
+        print('[prerun] Unable to connect to the database, waiting...')
+        time.sleep(10)
+        check_db_connection(conn_str, retry=retry - 1)
+    else:
+        connection.close()
+
+        
+def check_solr_connection(retry=None):
+
+    if retry is None:
+        retry = RETRY
+    elif retry == 0:
+        print('[prerun] Giving up after 5 tries...')
+        sys.exit(1)
+
+    url = os.environ.get('CKAN_SOLR_URL', '')
+    search_url = '{url}/select/?q=*&wt=json'.format(url=url)
+    http = urllib3.PoolManager()
+    try:
+        r = http.request('GET', search_url)
+    except urllib3.exceptions.ConnectionError as e:
+        print(str(e))
+        print('[prerun] Unable to connect to solr, waiting...')
+        time.sleep(10)
+        check_solr_connection(retry=retry - 1)
+    else:
+        print('[prerun] Connection Status from SOLR is ', (r.status))
+
+def init_db():
+
+    db_command = ['ckan', '-c', ckan_ini, 
+                  'db', 'init']
+    print('[prerun] Initializing or upgrading db - start')
+    try:
+        subprocess.check_output(db_command, stderr=subprocess.STDOUT)
+        print('[prerun] Initializing or upgrading db - end')
+    except subprocess.CalledProcessError as e:
+        if 'OperationalError' in e.output:
+            print(e.output)
+            print('[prerun] Database not ready, waiting a bit before exit...')
+            time.sleep(5)
+            sys.exit(1)
+        else:
+            print(e.output)
+            raise e
+
+
+def init_datastore_db():
+
+    conn_str = os.environ.get('CKAN_DATASTORE_WRITE_URL')
+    if not conn_str:
+        print('[prerun] Skipping datastore initialization')
+        return
+
+    datastore_perms_command = ['ckan', '-c', ckan_ini, 
+                               'datastore', 'set-permissions']
+
+    connection = psycopg2.connect(conn_str)
+    cursor = connection.cursor()
+
+    print('[prerun] Initializing datastore db - start')
+    try:
+        datastore_perms = subprocess.Popen(
+            datastore_perms_command,
+            stdout=subprocess.PIPE)
+
+        perms_sql = datastore_perms.stdout.read().decode('utf-8')
+        # Remove internal pg command as psycopg2 does not like it
+        perms_sql = re.sub('\\\\connect \"(.*)\"', '', perms_sql)
+        cursor.execute(perms_sql)
+        for notice in connection.notices:
+            print(notice)
+
+        connection.commit()
+
+        print('[prerun] Initializing datastore db - end')
+        print(datastore_perms.stdout.read().decode('utf-8'))
+    except psycopg2.Error as e:
+        print('[prerun] Could not initialize datastore')
+        print(str(e))
+
+    except subprocess.CalledProcessError as e:
+        if 'OperationalError' in e.output:
+            print(e.output)
+            print('[prerun] Database not ready, waiting a bit before exit...')
+            time.sleep(5)
+            sys.exit(1)
+        else:
+            print(e.output)
+            raise e
+    finally:
+        cursor.close()
+        connection.close()
+
+
+def create_sysadmin():
+
+    name = os.environ.get('CKAN_SYSADMIN_NAME')
+    password = os.environ.get('CKAN_SYSADMIN_PASSWORD')
+    email = os.environ.get('CKAN_SYSADMIN_EMAIL')
+
+    if name and password and email:
+
+        # Check if user exists
+        command = ['ckan', '-c', ckan_ini, 'user', 'show', name,]
+
+        out = subprocess.check_output(command)
+        if 'User:None' not in re.sub(r'\s', '', out.decode()):
+            print('[prerun] Sysadmin user exists, skipping creation')
+            return
+
+        # Create user
+        command = ['ckan', '-c', ckan_ini, 'user', 'add',
+                   name,
+                   'password=' + password,
+                   'email=' + email]
+
+        subprocess.call(command)
+        print('[prerun] Created user {0}'.format(name))
+
+        # Make it sysadmin
+        command = ['ckan', '-c', ckan_ini, 'sysadmin', 'add',
+                   name]
+
+        subprocess.call(command)
+        print('[prerun] Made user {0} a sysadmin'.format(name))
+
+
+if __name__ == '__main__':
+
+    maintenance = os.environ.get('MAINTENANCE_MODE', '').lower() == 'true'
+
+    if maintenance:
+        print('[prerun] Maintenance mode, skipping setup...')
+    else:
+        check_main_db_connection()
+        init_db()
+        update_plugins()
+        check_datastore_db_connection()
+        init_datastore_db()
+        check_solr_connection()
+        create_sysadmin()
diff --git a/setup/start_ckan.sh b/setup/start_ckan.sh
new file mode 100755
index 0000000..5a119e6
--- /dev/null
+++ b/setup/start_ckan.sh
@@ -0,0 +1,46 @@
+#!/bin/bash
+
+# Run the prerun script to init CKAN and create the default admin user
+sudo -u ckan -EH python3 prerun.py
+
+# Run any startup scripts provided by images extending this one
+if [[ -d "/docker-entrypoint.d" ]]
+then
+    for f in /docker-entrypoint.d/*; do
+        case "$f" in
+            *.sh)     echo "$0: Running init file $f"; . "$f" ;;
+            *.py)     echo "$0: Running init file $f"; python "$f"; echo ;;
+            *)        echo "$0: Ignoring $f (not an sh or py file)" ;;
+        esac
+        echo
+    done
+fi
+
+
+# Check whether http basic auth password protection is enabled and enable basicauth routing on uwsgi respecfully
+if [ $? -eq 0 ]
+then
+  if [ "$PASSWORD_PROTECT" = true ]
+  then
+    if [ "$HTPASSWD_USER" ] || [ "$HTPASSWD_PASSWORD" ]
+    then
+      # Generate htpasswd file for basicauth
+      htpasswd -d -b -c /srv/app/.htpasswd $HTPASSWD_USER $HTPASSWD_PASSWORD
+      # Start supervisord
+      supervisord --configuration /etc/supervisord.conf &
+      # Start uwsgi with basicauth
+      sudo -u ckan -EH uwsgi -i ckan-uwsgi.ini
+    else
+      echo "Missing HTPASSWD_USER or HTPASSWD_PASSWORD environment variables. Exiting..."
+      exit 1
+    fi
+  else
+    # Start supervisord
+    supervisord --configuration /etc/supervisord.conf &
+    # Start uwsgi
+    sudo -u ckan -EH uwsgi -i ckan-uwsgi.ini
+  fi
+else
+  echo "[prerun] failed...not starting CKAN."
+fi
+
diff --git a/setup/supervisor.worker.conf b/setup/supervisor.worker.conf
new file mode 100644
index 0000000..9d46f37
--- /dev/null
+++ b/setup/supervisor.worker.conf
@@ -0,0 +1,12 @@
+[program:ckan-worker]
+command=ckan -c /srv/app/ckan.ini jobs worker
+priority=501
+autostart=true
+autorestart=true
+redirect_stderr=true
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stdout
+stderr_logfile_maxbytes=0
+user=ckan
+environment=HOME="/srv/app",USER="ckan"
\ No newline at end of file
diff --git a/setup/supervisord.conf b/setup/supervisord.conf
new file mode 100644
index 0000000..052dbc5
--- /dev/null
+++ b/setup/supervisord.conf
@@ -0,0 +1,23 @@
+[unix_http_server]
+file = /tmp/supervisor.sock
+chmod = 0777
+chown = nobody:nogroup
+
+[supervisord]
+logfile = /tmp/supervisord.log
+logfile_maxbytes = 50MB
+logfile_backups=10
+loglevel = info
+pidfile = /tmp/supervisord.pid
+nodaemon = true
+umask = 022
+identifier = supervisor
+
+[supervisorctl]
+serverurl = unix:///tmp/supervisor.sock
+
+[rpcinterface:supervisor]
+supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
+
+[include]
+files = /etc/supervisord.d/*.conf
\ No newline at end of file
diff --git a/setup/wsgi.py b/setup/wsgi.py
new file mode 100644
index 0000000..b37d80e
--- /dev/null
+++ b/setup/wsgi.py
@@ -0,0 +1,9 @@
+import os
+from ckan.config.middleware import make_app
+from ckan.cli import CKANConfigLoader
+from logging.config import fileConfig as loggingFileConfig
+config_filepath = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'ckan.ini')
+abspath = os.path.join(os.path.dirname(os.path.abspath(__file__)))
+loggingFileConfig(config_filepath)
+config = CKANConfigLoader(config_filepath).get_config()
+application = make_app(config)

From aa0e24c3e865959ebac6685e7fd1153ab406abda Mon Sep 17 00:00:00 2001
From: Brett <brett@kowh.ai>
Date: Fri, 29 Oct 2021 13:57:04 +0200
Subject: [PATCH 04/10] commit datapusher directory changes

---
 Dockerfile                   | 135 ++++++++++++++---------------------
 setup/datapusher-uwsgi.ini   |  14 ++++
 setup/datapusher_settings.py |  37 ++++++++++
 setup/start_datapusher.sh    |   5 ++
 setup/supervisor.uwsgi.conf  |  12 ++++
 setup/supervisord.conf       |   2 +-
 6 files changed, 122 insertions(+), 83 deletions(-)
 create mode 100644 setup/datapusher-uwsgi.ini
 create mode 100644 setup/datapusher_settings.py
 create mode 100755 setup/start_datapusher.sh
 create mode 100644 setup/supervisor.uwsgi.conf

diff --git a/Dockerfile b/Dockerfile
index 0aa8caa..204b367 100755
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,113 +1,84 @@
-FROM alpine:3.7
+FROM alpine:3.13.5
 
-# Internal environment variables
 ENV APP_DIR=/srv/app
-ENV SRC_DIR=/srv/app/src
-ENV CKAN_INI=${APP_DIR}/ckan.ini
-ENV PIP_SRC=${SRC_DIR}
-ENV CKAN_STORAGE_PATH=/var/lib/ckan
-ENV GIT_URL=https://github.com/ckan/ckan.git
-# CKAN version to build
-ENV GIT_BRANCH=ckan-2.9.3
-# Customize these on the .env file if needed
-ENV CKAN_SITE_URL=http://localhost:5000
+ENV SRC_DIR=${APP_DIR}/src
+ENV GIT_URL https://github.com/kowh-ai/datapusher.git 
+#ENV GIT_URL https://github.com/keitaroinc/datapusher.git
+ENV GIT_BRANCH master
+ENV JOB_CONFIG ${APP_DIR}/datapusher_settings.py
 ENV CKAN__PLUGINS image_view text_view recline_view datastore datapusher envvars
 
 WORKDIR ${APP_DIR}
 
-# Install necessary packages to run CKAN
-RUN apk add --no-cache tzdata \
-        git \
-        gettext \
-        postgresql-client \
+RUN apk upgrade && \
+    apk add --no-cache \
         python3 \
-        apache2-utils \
-        libxml2 \
-        libxslt \
-        musl-dev \
-        uwsgi-http \
-        uwsgi-corerouter \
-        uwsgi-python3 \
-        py3-gevent \
-        uwsgi-gevent \
-        libmagic \
         curl \
-        sudo && \
-    # Packages to build CKAN requirements and plugins
-    apk add --no-cache --virtual .build-deps \
-        postgresql-dev \
         gcc \
         make \
         g++ \
         autoconf \
         automake \
-	    libtool \
+        libtool \
+        git \
+        musl-dev \
         python3-dev \
-        py3-virtualenv \
-        libxml2-dev \
+        libffi-dev \
+        openssl-dev \
+        libxml2-dev \   
         libxslt-dev \
-        linux-headers && \
-        # Create SRC_DIR
-        mkdir -p ${SRC_DIR}
+        rust  \
+        cargo
+
+RUN apk add --no-cache \
+        uwsgi \
+        uwsgi-http \
+        uwsgi-corerouter \
+        uwsgi-python
+
+# Create the src directory
+RUN mkdir -p ${SRC_DIR}
 
 # Install pip
 RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/get-pip.py && \
     python3 ${SRC_DIR}/get-pip.py
 
-# Set up Python3 virtual environment
-RUN cd ${APP_DIR} && \
-    python3 -m venv ${APP_DIR} && \
-    source ${APP_DIR}/bin/activate
+# Install datapusher
+RUN cd ${SRC_DIR} && \
+    git clone -b ${GIT_BRANCH} --depth=1 --single-branch ${GIT_URL} && \
+    cd datapusher && \
+    python3 setup.py install && \
+    pip3 install --no-cache-dir -r requirements.txt
 
-# Virtual environment binaries/scripts to be used first
-ENV PATH=${APP_DIR}/bin:${PATH}   
-    
-# Install CKAN, uwsgi plus extensions
-RUN pip3 install -e git+${GIT_URL}@${GIT_BRANCH}#egg=ckan && \ 
-    pip3 install uwsgi && \
-    cd ${SRC_DIR}/ckan && \
-    cp who.ini ${APP_DIR} && \
-    pip install --no-binary :all: -r requirements.txt && \
-    # Install CKAN envvars to support loading config from environment variables
-    pip3 install -e git+https://github.com/okfn/ckanext-envvars.git#egg=ckanext-envvars && \
-    # Install CKAN extensions
-    pip3 install -e 'git+https://github.com/DataShades/ckanext-xloader@py3#egg=ckanext-xloader' && \
-    pip3 install -r $CKAN_VENV/src/ckanext-xloader/requirements.txt && \
-    pip3 install -U requests[security] && \
-    pip3 install -e 'git+https://github.com/DataShades/ckanext-harvest.git@py3#egg=ckanext-harvest' && \
-    pip3 install -r $CKAN_VENV/src/ckanext-harvest/pip-requirements.txt && \
-    pip3 install -e 'git+https://github.com/DataShades/ckanext-syndicate@py3#egg=ckanext-syndicate' && \
-    pip3 install -r $CKAN_VENV/src/ckanext-syndicate/requirements.txt && \
-    pip3 install -e 'git+https://github.com/ckan/ckanext-scheming.git@master#egg=ckanext-scheming' && \
-    pip3 install -r $CKAN_VENV/src/ckanext-scheming/requirements.txt
-
-# Create and update CKAN config
-RUN ckan generate config ${CKAN_INI}
+RUN cp ${APP_DIR}/src/datapusher/deployment/*.* ${APP_DIR} && \
+    # Remove default values in ini file
+    sed -i '/http/d' ${APP_DIR}/datapusher-uwsgi.ini && \
+    sed -i '/wsgi-file/d' ${APP_DIR}/datapusher-uwsgi.ini && \
+    sed -i '/virtualenv/d' ${APP_DIR}/datapusher-uwsgi.ini
+    # Remove src files
+    #rm -rf ${APP_DIR}/src
 
 # Install and configure supervisor
 RUN pip3 install supervisor && \
 mkdir /etc/supervisord.d
 
 # Copy all setup files
-COPY setup ${APP_DIR}
-COPY setup/supervisor.worker.conf /etc/supervisord.d/worker.conf
+COPY setup/start_datapusher.sh ${APP_DIR}
+COPY setup/datapusher-uwsgi.ini ${APP_DIR}
+COPY setup/datapusher_settings.py ${APP_DIR}
+COPY setup/supervisor.uwsgi.conf /etc/supervisord.d/datapusher.uwsgi.conf
 COPY setup/supervisord.conf /etc/supervisord.conf
 
-# Create a local user and group to run the app
-RUN addgroup -g 92 -S ckan && \
-    adduser -u 92 -h /srv/app -H -D -S -G ckan ckan
+# Create a 'ckan' local user and group to run the app
+RUN addgroup -g 92 -S www-data && \
+    adduser -u 92 -h /srv/app -H -D -S -G www-data www-data
 
-# Create local storage folder
-RUN mkdir -p $CKAN_STORAGE_PATH && \
-    chown -R ckan:ckan $CKAN_STORAGE_PATH
+# Set timezone
+RUN echo "UTC" >  /etc/timezone && \
+    # Change ownership to app user
+    chown -R www-data:www-data /srv/app
 
-# Create entrypoint directory for children image scripts
-ONBUILD RUN mkdir /docker-entrypoint.d
-
-RUN chown ckan -R /srv/app
-
-EXPOSE 5000
-
-HEALTHCHECK --interval=10s --timeout=5s --retries=5 CMD curl --fail http://localhost:5000/api/3/action/status_show || exit 1
-
-CMD ["/srv/app/start_ckan.sh"] 
\ No newline at end of file
+EXPOSE 8800
+CMD ["/srv/app/start_datapusher.sh"] 
+#CMD ["sh", "-c", \
+#    "uwsgi --plugins=http,python --http=0.0.0.0:8800 --socket=/tmp/uwsgi.sock --ini=`echo ${APP_DIR}`/datapusher-uwsgi.ini --wsgi-file=`echo ${APP_DIR}`/datapusher.wsgi"]
diff --git a/setup/datapusher-uwsgi.ini b/setup/datapusher-uwsgi.ini
new file mode 100644
index 0000000..4c8274a
--- /dev/null
+++ b/setup/datapusher-uwsgi.ini
@@ -0,0 +1,14 @@
+[uwsgi]
+http            =  0.0.0.0:8800
+uid             =  www-data
+guid            =  www-data
+wsgi-file       =  /srv/app/datapusher.wsgi
+master          =  true
+socket          =  /tmp/uwsgi.sock
+plugins         =  http,python
+pidfile         =  /tmp/%n.pid
+harakiri        =  50
+max-requests    =  5000
+vacuum          =  true
+callable        =  application
+buffer-size     =  32768
\ No newline at end of file
diff --git a/setup/datapusher_settings.py b/setup/datapusher_settings.py
new file mode 100644
index 0000000..d6ee97a
--- /dev/null
+++ b/setup/datapusher_settings.py
@@ -0,0 +1,37 @@
+import os
+import uuid
+
+DEBUG = False
+TESTING = False
+SECRET_KEY = str(uuid.uuid4())
+USERNAME = str(uuid.uuid4())
+PASSWORD = str(uuid.uuid4())
+
+NAME = 'datapusher'
+
+# Webserver host and port
+
+HOST = os.environ.get('DATAPUSHER_HOST', '0.0.0.0')
+PORT = os.environ.get('DATAPUSHER_PORT', 8800)
+
+# Database
+
+SQLALCHEMY_DATABASE_URI = os.environ.get('DATAPUSHER_SQLALCHEMY_DATABASE_URI', 'sqlite:////tmp/job_store.db')
+
+# Download and streaming settings
+
+MAX_CONTENT_LENGTH = int(os.environ.get('DATAPUSHER_MAX_CONTENT_LENGTH', '1024000'))
+CHUNK_SIZE = int(os.environ.get('DATAPUSHER_CHUNK_SIZE', '16384'))
+CHUNK_INSERT_ROWS = int(os.environ.get('DATAPUSHER_CHUNK_INSERT_ROWS', '250'))
+DOWNLOAD_TIMEOUT = int(os.environ.get('DATAPUSHER_DOWNLOAD_TIMEOUT', '30'))
+
+# Verify SSL
+SSL_VERIFY = os.environ.get('DATAPUSHER_SSL_VERIFY', True)
+
+# logging
+#LOG_FILE = '/tmp/ckan_service.log'
+STDERR = True
+
+# Rewrite resource URL's when ckan callback url base is used
+REWRITE_RESOURCES = os.environ.get('DATAPUSHER_REWRITE_RESOURCES', True)
+REWRITE_URL = os.environ.get('DATAPUSHER_REWRITE_URL', 'http://ckan:5000/')
diff --git a/setup/start_datapusher.sh b/setup/start_datapusher.sh
new file mode 100755
index 0000000..3b78d19
--- /dev/null
+++ b/setup/start_datapusher.sh
@@ -0,0 +1,5 @@
+#!/bin/sh
+
+echo "[start_datapusher.sh] Starting supervisord."
+# Start supervisord
+supervisord --configuration /etc/supervisord.conf
diff --git a/setup/supervisor.uwsgi.conf b/setup/supervisor.uwsgi.conf
new file mode 100644
index 0000000..67a6c37
--- /dev/null
+++ b/setup/supervisor.uwsgi.conf
@@ -0,0 +1,12 @@
+[program:datapusher-uwsgi]
+command=/usr/sbin/uwsgi -i /srv/app/datapusher-uwsgi.ini
+priority=501
+autostart=true
+autorestart=true
+redirect_stderr=true
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stdout
+stderr_logfile_maxbytes=0
+user=www-data
+environment=HOME="/srv/app",USER="www-data"
\ No newline at end of file
diff --git a/setup/supervisord.conf b/setup/supervisord.conf
index 052dbc5..f21c89c 100644
--- a/setup/supervisord.conf
+++ b/setup/supervisord.conf
@@ -7,7 +7,7 @@ chown = nobody:nogroup
 logfile = /tmp/supervisord.log
 logfile_maxbytes = 50MB
 logfile_backups=10
-loglevel = info
+loglevel = trace
 pidfile = /tmp/supervisord.pid
 nodaemon = true
 umask = 022

From bfb4c475e6ffb1f826c80807db99411a8ffc4cfe Mon Sep 17 00:00:00 2001
From: Brett <brett@kowh.ai>
Date: Fri, 29 Oct 2021 13:57:44 +0200
Subject: [PATCH 05/10] commit nginx directory changes

---
 Dockerfile                      | 87 ++++-----------------------------
 index.html                      | 22 +++++++++
 setup/nginx.conf                | 34 +++++++++++++
 setup/sites-available/ckan.conf | 18 +++++++
 4 files changed, 83 insertions(+), 78 deletions(-)
 mode change 100755 => 100644 Dockerfile
 create mode 100644 index.html
 create mode 100644 setup/nginx.conf
 create mode 100644 setup/sites-available/ckan.conf

diff --git a/Dockerfile b/Dockerfile
old mode 100755
new mode 100644
index 204b367..e3b446a
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,84 +1,15 @@
-FROM alpine:3.13.5
+FROM nginx:1.19.8-alpine
 
-ENV APP_DIR=/srv/app
-ENV SRC_DIR=${APP_DIR}/src
-ENV GIT_URL https://github.com/kowh-ai/datapusher.git 
-#ENV GIT_URL https://github.com/keitaroinc/datapusher.git
-ENV GIT_BRANCH master
-ENV JOB_CONFIG ${APP_DIR}/datapusher_settings.py
-ENV CKAN__PLUGINS image_view text_view recline_view datastore datapusher envvars
+ENV NGINX_DIR=/etc/nginx
 
-WORKDIR ${APP_DIR}
+COPY index.html /usr/share/nginx/html/index.html
 
-RUN apk upgrade && \
-    apk add --no-cache \
-        python3 \
-        curl \
-        gcc \
-        make \
-        g++ \
-        autoconf \
-        automake \
-        libtool \
-        git \
-        musl-dev \
-        python3-dev \
-        libffi-dev \
-        openssl-dev \
-        libxml2-dev \   
-        libxslt-dev \
-        rust  \
-        cargo
+RUN mkdir -p ${NGINX_DIR}/sites-available
+RUN mkdir -p ${NGINX_DIR}/sites-enabled
 
-RUN apk add --no-cache \
-        uwsgi \
-        uwsgi-http \
-        uwsgi-corerouter \
-        uwsgi-python
+COPY setup/nginx.conf ${NGINX_DIR}
+COPY setup/sites-available/* ${NGINX_DIR}/sites-available
 
-# Create the src directory
-RUN mkdir -p ${SRC_DIR}
+RUN ln -s ${NGINX_DIR}/sites-available/ckan.conf ${NGINX_DIR}/sites-enabled/ckan.conf
 
-# Install pip
-RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/get-pip.py && \
-    python3 ${SRC_DIR}/get-pip.py
-
-# Install datapusher
-RUN cd ${SRC_DIR} && \
-    git clone -b ${GIT_BRANCH} --depth=1 --single-branch ${GIT_URL} && \
-    cd datapusher && \
-    python3 setup.py install && \
-    pip3 install --no-cache-dir -r requirements.txt
-
-RUN cp ${APP_DIR}/src/datapusher/deployment/*.* ${APP_DIR} && \
-    # Remove default values in ini file
-    sed -i '/http/d' ${APP_DIR}/datapusher-uwsgi.ini && \
-    sed -i '/wsgi-file/d' ${APP_DIR}/datapusher-uwsgi.ini && \
-    sed -i '/virtualenv/d' ${APP_DIR}/datapusher-uwsgi.ini
-    # Remove src files
-    #rm -rf ${APP_DIR}/src
-
-# Install and configure supervisor
-RUN pip3 install supervisor && \
-mkdir /etc/supervisord.d
-
-# Copy all setup files
-COPY setup/start_datapusher.sh ${APP_DIR}
-COPY setup/datapusher-uwsgi.ini ${APP_DIR}
-COPY setup/datapusher_settings.py ${APP_DIR}
-COPY setup/supervisor.uwsgi.conf /etc/supervisord.d/datapusher.uwsgi.conf
-COPY setup/supervisord.conf /etc/supervisord.conf
-
-# Create a 'ckan' local user and group to run the app
-RUN addgroup -g 92 -S www-data && \
-    adduser -u 92 -h /srv/app -H -D -S -G www-data www-data
-
-# Set timezone
-RUN echo "UTC" >  /etc/timezone && \
-    # Change ownership to app user
-    chown -R www-data:www-data /srv/app
-
-EXPOSE 8800
-CMD ["/srv/app/start_datapusher.sh"] 
-#CMD ["sh", "-c", \
-#    "uwsgi --plugins=http,python --http=0.0.0.0:8800 --socket=/tmp/uwsgi.sock --ini=`echo ${APP_DIR}`/datapusher-uwsgi.ini --wsgi-file=`echo ${APP_DIR}`/datapusher.wsgi"]
+EXPOSE 80
\ No newline at end of file
diff --git a/index.html b/index.html
new file mode 100644
index 0000000..561b63b
--- /dev/null
+++ b/index.html
@@ -0,0 +1,22 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <meta http-equiv="X-UA-Compatible" content="ie=edge">
+    <title>CKAN Docker NGINX landing page</title>
+    <style>
+        h1{
+            font-weight:lighter;
+            font-family: Arial, Helvetica, sans-serif;
+        }
+    </style>
+</head>
+<body>
+    
+    <h2>
+        CKAN Docker NGINX landing page
+    </h2>
+
+</body>
+</html>
\ No newline at end of file
diff --git a/setup/nginx.conf b/setup/nginx.conf
new file mode 100644
index 0000000..7f06cea
--- /dev/null
+++ b/setup/nginx.conf
@@ -0,0 +1,34 @@
+
+user  nginx;
+worker_processes  auto;
+
+error_log  /var/log/nginx/error.log warn;
+pid        /var/run/nginx.pid;
+
+
+events {
+    worker_connections  1024;
+}
+
+
+http {
+    include       /etc/nginx/mime.types;
+    default_type  application/octet-stream;
+
+    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                      '$status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_x_forwarded_for"';
+
+    access_log  /var/log/nginx/access.log  main;
+
+    sendfile        on;
+    #tcp_nopush     on;
+
+    keepalive_timeout  65;
+
+    #gzip  on;
+
+    include /etc/nginx/sites-enabled/*.conf;
+}
+
+
diff --git a/setup/sites-available/ckan.conf b/setup/sites-available/ckan.conf
new file mode 100644
index 0000000..7ffc6bb
--- /dev/null
+++ b/setup/sites-available/ckan.conf
@@ -0,0 +1,18 @@
+proxy_cache_path /tmp/nginx_cache levels=1:2 keys_zone=cache:30m max_size=250m;
+proxy_temp_path /tmp/nginx_proxy 1 2;
+
+server {
+    client_max_body_size 100M;
+    location / {
+        proxy_pass http://ckan:5000/;
+        proxy_set_header X-Forwarded-For $remote_addr;
+        proxy_set_header Host $host;
+        proxy_cache cache;
+        proxy_cache_bypass $cookie_auth_tkt;
+        proxy_no_cache $cookie_auth_tkt;
+        proxy_cache_valid 30m;
+        proxy_cache_key $host$scheme$proxy_host$request_uri;
+        # In emergency comment out line to force caching
+        # proxy_ignore_headers X-Accel-Expires Expires Cache-Control;
+    }
+}
\ No newline at end of file

From 0d7e4d8f09541dc1af0ca80de2a76be355b07745 Mon Sep 17 00:00:00 2001
From: Brett <brett@kowh.ai>
Date: Fri, 29 Oct 2021 13:58:24 +0200
Subject: [PATCH 06/10] commit postgresql directory changes

---
 Dockerfile                                    | 22 +++++++++----------
 .../10_create_datastore.sql                   |  4 ++++
 .../20_setup_test_databases.sql               |  2 ++
 3 files changed, 16 insertions(+), 12 deletions(-)
 create mode 100755 docker-entrypoint-initdb.d/10_create_datastore.sql
 create mode 100755 docker-entrypoint-initdb.d/20_setup_test_databases.sql

diff --git a/Dockerfile b/Dockerfile
index e3b446a..4a74f83 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,15 +1,13 @@
-FROM nginx:1.19.8-alpine
+FROM postgres:12-alpine
 
-ENV NGINX_DIR=/etc/nginx
+# Allow connections; we don't map out any ports so only linked docker containers can connect
+RUN echo "host all  all    0.0.0.0/0  md5" >> /var/lib/postgresql/data/pg_hba.conf
 
-COPY index.html /usr/share/nginx/html/index.html
+# Customize default user/pass/db
+ENV POSTGRES_DB ckan
+ENV POSTGRES_USER ckan
+ARG POSTGRES_PASSWORD
+ARG DATASTORE_READONLY_PASSWORD
 
-RUN mkdir -p ${NGINX_DIR}/sites-available
-RUN mkdir -p ${NGINX_DIR}/sites-enabled
-
-COPY setup/nginx.conf ${NGINX_DIR}
-COPY setup/sites-available/* ${NGINX_DIR}/sites-available
-
-RUN ln -s ${NGINX_DIR}/sites-available/ckan.conf ${NGINX_DIR}/sites-enabled/ckan.conf
-
-EXPOSE 80
\ No newline at end of file
+# Include datastore setup scripts
+COPY ./postgresql/docker-entrypoint-initdb.d /docker-entrypoint-initdb.d
\ No newline at end of file
diff --git a/docker-entrypoint-initdb.d/10_create_datastore.sql b/docker-entrypoint-initdb.d/10_create_datastore.sql
new file mode 100755
index 0000000..87529ce
--- /dev/null
+++ b/docker-entrypoint-initdb.d/10_create_datastore.sql
@@ -0,0 +1,4 @@
+\set datastore_ro_password '\'' `echo $DATASTORE_READONLY_PASSWORD` '\''
+
+CREATE ROLE datastore_ro NOSUPERUSER NOCREATEDB NOCREATEROLE LOGIN PASSWORD :datastore_ro_password;
+CREATE DATABASE datastore OWNER ckan ENCODING 'utf-8';
\ No newline at end of file
diff --git a/docker-entrypoint-initdb.d/20_setup_test_databases.sql b/docker-entrypoint-initdb.d/20_setup_test_databases.sql
new file mode 100755
index 0000000..61366d2
--- /dev/null
+++ b/docker-entrypoint-initdb.d/20_setup_test_databases.sql
@@ -0,0 +1,2 @@
+CREATE DATABASE ckan_test OWNER ckan ENCODING 'utf-8';
+CREATE DATABASE datastore_test OWNER ckan ENCODING 'utf-8';
\ No newline at end of file

From dc347958c38d160328365d67e22185461e512789 Mon Sep 17 00:00:00 2001
From: Brett <brett@kowh.ai>
Date: Fri, 29 Oct 2021 13:58:53 +0200
Subject: [PATCH 07/10] commit solr directory changes

---
 Dockerfile           |  42 ++++--
 solrconfig-2.9.3.xml | 345 +++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 377 insertions(+), 10 deletions(-)
 create mode 100644 solrconfig-2.9.3.xml

diff --git a/Dockerfile b/Dockerfile
index 4a74f83..a29437d 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,13 +1,35 @@
-FROM postgres:12-alpine
+FROM solr:6.6.6
 
-# Allow connections; we don't map out any ports so only linked docker containers can connect
-RUN echo "host all  all    0.0.0.0/0  md5" >> /var/lib/postgresql/data/pg_hba.conf
+# Enviroment variables
+ENV SOLR_CORE ckan
 
-# Customize default user/pass/db
-ENV POSTGRES_DB ckan
-ENV POSTGRES_USER ckan
-ARG POSTGRES_PASSWORD
-ARG DATASTORE_READONLY_PASSWORD
+# Build Arguments
+ARG CKAN_VERSION
+ARG SOLR_VERSION
 
-# Include datastore setup scripts
-COPY ./postgresql/docker-entrypoint-initdb.d /docker-entrypoint-initdb.d
\ No newline at end of file
+# root user for initial config
+USER root
+
+# Create directories
+RUN mkdir -p /opt/solr/server/solr/${SOLR_CORE}/conf && \
+    mkdir -p /opt/solr/server/solr/${SOLR_CORE}/data && \
+    mkdir -p /opt/solr/server/solr/${SOLR_CORE}/data/index
+
+# Add files
+COPY ./solr/solrconfig-${CKAN_VERSION}.xml /opt/solr/server/solr/${SOLR_CORE}/conf/solrconfig.xml
+ADD https://raw.githubusercontent.com/ckan/ckan/ckan-${CKAN_VERSION}/ckan/config/solr/schema.xml \
+https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/${SOLR_VERSION}/solr/server/solr/configsets/basic_configs/conf/currency.xml \
+https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/${SOLR_VERSION}/solr/server/solr/configsets/basic_configs/conf/synonyms.txt \
+https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/${SOLR_VERSION}/solr/server/solr/configsets/basic_configs/conf/stopwords.txt \
+https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/${SOLR_VERSION}/solr/server/solr/configsets/basic_configs/conf/protwords.txt \
+https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/${SOLR_VERSION}/solr/server/solr/configsets/data_driven_schema_configs/conf/elevate.xml \
+/opt/solr/server/solr/${SOLR_CORE}/conf/
+
+# Create core.properties
+RUN echo name=${SOLR_CORE} > /opt/solr/server/solr/${SOLR_CORE}/core.properties
+
+# Giving ownership to Solr
+RUN chown -R ${SOLR_USER}:${SOLR_USER} /opt/solr/server/solr/${SOLR_CORE}
+
+# non-root user for runtime
+USER ${SOLR_USER}:${SOLR_USER}
\ No newline at end of file
diff --git a/solrconfig-2.9.3.xml b/solrconfig-2.9.3.xml
new file mode 100644
index 0000000..5ddc4c4
--- /dev/null
+++ b/solrconfig-2.9.3.xml
@@ -0,0 +1,345 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--solrconfig.xml documentation [https://wiki.apache.org/solr/SolrConfigXml]-->
+<config>
+ 
+	<luceneMatchVersion>6.0.0</luceneMatchVersion>
+	
+	<lib dir="${solr.install.dir:../../../..}/contrib/extraction/lib" regex=".*\.jar" />
+	<lib dir="${solr.install.dir:../../../..}/dist/" regex="solr-cell-\d.*\.jar" />
+	<lib dir="${solr.install.dir:../../../..}/contrib/clustering/lib/" regex=".*\.jar" />
+	<lib dir="${solr.install.dir:../../../..}/dist/" regex="solr-clustering-\d.*\.jar" />
+	<lib dir="${solr.install.dir:../../../..}/contrib/langid/lib/" regex=".*\.jar" />
+	<lib dir="${solr.install.dir:../../../..}/dist/" regex="solr-langid-\d.*\.jar" />
+	<lib dir="${solr.install.dir:../../../..}/contrib/velocity/lib" regex=".*\.jar" />
+	<lib dir="${solr.install.dir:../../../..}/dist/" regex="solr-velocity-\d.*\.jar" />
+	
+	<dataDir>${solr.data.dir:}</dataDir>
+	
+	<directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.NRTCachingDirectoryFactory}" />
+	
+	<codecFactory class="solr.SchemaCodecFactory" />
+	
+	<indexConfig>
+		<lockType>${solr.lock.type:native}</lockType>
+	</indexConfig>
+	
+	<jmx />
+	
+	<updateHandler class="solr.DirectUpdateHandler2">
+	 
+		<updateLog>
+			<str name="dir">${solr.ulog.dir:}</str>
+			<int name="numVersionBuckets">${solr.ulog.numVersionBuckets:65536}</int>
+		</updateLog>
+		
+		<autoCommit>
+			<maxTime>${solr.autoCommit.maxTime:15000}</maxTime>
+			<openSearcher>false</openSearcher>
+		</autoCommit>
+		
+		<autoSoftCommit>
+			<maxTime>${solr.autoSoftCommit.maxTime:-1}</maxTime>
+		</autoSoftCommit>
+		
+	</updateHandler>
+	
+	<query>
+	 
+		<maxBooleanClauses>1024</maxBooleanClauses>
+		<filterCache class="solr.FastLRUCache" size="512" initialSize="512" autowarmCount="0" />
+		<queryResultCache class="solr.LRUCache" size="512" initialSize="512" autowarmCount="0" />
+		<documentCache class="solr.LRUCache" size="512" initialSize="512" autowarmCount="0" />
+		<cache name="perSegFilter" class="solr.search.LRUCache" size="10" initialSize="0" autowarmCount="10" regenerator="solr.NoOpRegenerator" />
+		<enableLazyFieldLoading>true</enableLazyFieldLoading>
+		<queryResultWindowSize>20</queryResultWindowSize>
+		<queryResultMaxDocsCached>200</queryResultMaxDocsCached>
+		
+		<listener event="newSearcher" class="solr.QuerySenderListener">
+			<arr name="queries" />
+		</listener>
+		
+		<listener event="firstSearcher" class="solr.QuerySenderListener">
+			<arr name="queries" />
+		</listener>
+		
+		<useColdSearcher>false</useColdSearcher>
+		<maxWarmingSearchers>2</maxWarmingSearchers>
+		
+	</query>
+	
+	<requestDispatcher handleSelect="false">
+	 
+		<requestParsers enableRemoteStreaming="true" multipartUploadLimitInKB="2048000" formdataUploadLimitInKB="2048" addHttpRequestToContext="false" />
+		<httpCaching never304="true" />
+		
+	</requestDispatcher>
+	
+	<requestHandler name="/select" class="solr.SearchHandler">
+	 
+		<lst name="defaults">
+			<str name="echoParams">explicit</str>
+			<int name="rows">10</int>
+		</lst>
+		
+	</requestHandler>
+	
+	<requestHandler name="/query" class="solr.SearchHandler">
+	 
+		<lst name="defaults">
+			<str name="echoParams">explicit</str>
+			<str name="wt">json</str>
+			<str name="indent">true</str>
+		</lst>
+		
+	</requestHandler>
+	
+	<requestHandler name="/browse" class="solr.SearchHandler" useParams="query,facets,velocity,browse">
+	 
+		<lst name="defaults">
+			<str name="echoParams">explicit</str>
+		</lst>
+		
+	</requestHandler>
+	
+	<initParams path="/update/**,/query,/select,/tvrh,/elevate,/spell,/browse">
+	 
+		<lst name="defaults">
+			<str name="df">_text_</str>
+		</lst>
+		
+	</initParams>
+	
+	<initParams path="/update/**">
+	 
+		<lst name="defaults">
+			<str name="update.chain">add-unknown-fields-to-the-schema</str>
+		</lst>
+		
+	</initParams>
+	
+	<requestHandler name="/update/extract" startup="lazy" class="solr.extraction.ExtractingRequestHandler">
+	 
+		<lst name="defaults">
+			<str name="lowernames">true</str>
+			<str name="fmap.meta">ignored_</str>
+			<str name="fmap.content">_text_</str>
+		</lst>
+		
+	</requestHandler>
+	
+	<requestHandler name="/analysis/field" startup="lazy" class="solr.FieldAnalysisRequestHandler" />
+	<requestHandler name="/analysis/document" class="solr.DocumentAnalysisRequestHandler" startup="lazy" />
+	
+	<requestHandler name="/debug/dump" class="solr.DumpRequestHandler">
+	 
+		<lst name="defaults">
+			<str name="echoParams">explicit</str>
+			<str name="echoHandler">true</str>
+		</lst>
+		
+	</requestHandler>
+	
+	<searchComponent name="spellcheck" class="solr.SpellCheckComponent">
+	 
+		<str name="queryAnalyzerFieldType">text_general</str>
+		
+		<lst name="spellchecker">
+			<str name="name">default</str>
+			<str name="field">_text_</str>
+			<str name="classname">solr.DirectSolrSpellChecker</str>
+			<str name="distanceMeasure">internal</str>
+			<float name="accuracy">0.5</float>
+			<int name="maxEdits">2</int>
+			<int name="minPrefix">1</int>
+			<int name="maxInspections">5</int>
+			<int name="minQueryLength">4</int>
+			<float name="maxQueryFrequency">0.01</float>
+		</lst>
+		
+	</searchComponent>
+	
+	<requestHandler name="/spell" class="solr.SearchHandler" startup="lazy">
+	 
+		<lst name="defaults">
+			<str name="spellcheck.dictionary">default</str>
+			<str name="spellcheck">on</str>
+			<str name="spellcheck.extendedResults">true</str>
+			<str name="spellcheck.count">10</str>
+			<str name="spellcheck.alternativeTermCount">5</str>
+			<str name="spellcheck.maxResultsForSuggest">5</str>
+			<str name="spellcheck.collate">true</str>
+			<str name="spellcheck.collateExtendedResults">true</str>
+			<str name="spellcheck.maxCollationTries">10</str>
+			<str name="spellcheck.maxCollations">5</str>
+		</lst>
+		
+		<arr name="last-components">
+			<str>spellcheck</str>
+		</arr>
+		
+	</requestHandler>
+	
+	<searchComponent name="tvComponent" class="solr.TermVectorComponent" />
+	
+	<requestHandler name="/tvrh" class="solr.SearchHandler" startup="lazy">
+	 
+		<lst name="defaults">
+			<bool name="tv">true</bool>
+		</lst>
+		<arr name="last-components">
+			<str>tvComponent</str>
+		</arr>
+		
+	</requestHandler>
+	
+	<searchComponent name="terms" class="solr.TermsComponent" />
+	
+	<requestHandler name="/terms" class="solr.SearchHandler" startup="lazy">
+	 
+		<lst name="defaults">
+			<bool name="terms">true</bool>
+			<bool name="distrib">false</bool>
+		</lst>
+		
+		<arr name="components">
+			<str>terms</str>
+		</arr>
+		
+	</requestHandler>
+	
+	<searchComponent name="elevator" class="solr.QueryElevationComponent">
+	 
+		<str name="queryFieldType">string</str>
+		<str name="config-file">elevate.xml</str>
+		
+	</searchComponent>
+	
+	<requestHandler name="/elevate" class="solr.SearchHandler" startup="lazy">
+	 
+		<lst name="defaults">
+			<str name="echoParams">explicit</str>
+		</lst>
+		<arr name="last-components">
+			<str>elevator</str>
+		</arr>
+		
+	</requestHandler>
+	
+	<searchComponent class="solr.HighlightComponent" name="highlight">
+	 
+		<highlighting>
+		 
+			<fragmenter name="gap" default="true" class="solr.highlight.GapFragmenter">
+				<lst name="defaults">
+					<int name="hl.fragsize">100</int>
+				</lst>
+			</fragmenter>
+			
+			<fragmenter name="regex" class="solr.highlight.RegexFragmenter">
+				<lst name="defaults">
+					<int name="hl.fragsize">70</int>
+					<float name="hl.regex.slop">0.5</float>
+					<str name="hl.regex.pattern">[-\w ,/\n\"']{20,200}</str>
+				</lst>
+			</fragmenter>
+			
+			<formatter name="html" default="true" class="solr.highlight.HtmlFormatter">
+				<lst name="defaults">
+					<str name="hl.simple.pre"><![CDATA[<em>]]></str>
+					<str name="hl.simple.post"><![CDATA[</em>]]></str>
+				</lst>
+			</formatter>
+			
+			<encoder name="html" class="solr.highlight.HtmlEncoder" />
+			<fragListBuilder name="simple" class="solr.highlight.SimpleFragListBuilder" />
+			<fragListBuilder name="single" class="solr.highlight.SingleFragListBuilder" />
+			<fragListBuilder name="weighted" default="true" class="solr.highlight.WeightedFragListBuilder" />
+			<fragmentsBuilder name="default" default="true" class="solr.highlight.ScoreOrderFragmentsBuilder" />
+			<fragmentsBuilder name="colored" class="solr.highlight.ScoreOrderFragmentsBuilder">
+			 
+				<lst name="defaults">
+					<str name="hl.tag.pre"><![CDATA[<b style="background:yellow">,<b style="background:lawgreen">,
+               <b style="background:aquamarine">,<b style="background:magenta">,
+               <b style="background:palegreen">,<b style="background:coral">,
+               <b style="background:wheat">,<b style="background:khaki">,
+               <b style="background:lime">,<b style="background:deepskyblue">]]></str>
+					<str name="hl.tag.post"><![CDATA[</b>]]></str>
+				</lst>
+				
+			</fragmentsBuilder>
+			
+			<boundaryScanner name="default" default="true" class="solr.highlight.SimpleBoundaryScanner">
+				<lst name="defaults">
+					<str name="hl.bs.maxScan">10</str>
+					<str name="hl.bs.chars">.,!?</str>
+				</lst>
+			</boundaryScanner>
+			
+			<boundaryScanner name="breakIterator" class="solr.highlight.BreakIteratorBoundaryScanner">
+				<lst name="defaults">
+					<str name="hl.bs.type">WORD</str>
+					<str name="hl.bs.language">en</str>
+					<str name="hl.bs.country">US</str>
+				</lst>
+			</boundaryScanner>
+			
+		</highlighting>
+		
+	</searchComponent>
+	
+	<schemaFactory class="ManagedIndexSchemaFactory">
+		<bool name="mutable">true</bool>
+	</schemaFactory>
+	
+	<updateRequestProcessorChain name="add-unknown-fields-to-the-schema">
+		<processor class="solr.UUIDUpdateProcessorFactory" />
+		<processor class="solr.LogUpdateProcessorFactory" />
+		<processor class="solr.DistributedUpdateProcessorFactory" />
+		<processor class="solr.RemoveBlankFieldUpdateProcessorFactory" />
+		<processor class="solr.FieldNameMutatingUpdateProcessorFactory">
+			<str name="pattern">[^\w-\.]</str>
+			<str name="replacement">_</str>
+		</processor>
+		<processor class="solr.ParseBooleanFieldUpdateProcessorFactory" />
+		<processor class="solr.ParseLongFieldUpdateProcessorFactory" />
+		<processor class="solr.ParseDoubleFieldUpdateProcessorFactory" />
+		<processor class="solr.ParseDateFieldUpdateProcessorFactory">
+			<arr name="format">
+				<str>yyyy-MM-dd'T'HH:mm:ss.SSSZ</str>
+				<str>yyyy-MM-dd'T'HH:mm:ss,SSSZ</str>
+				<str>yyyy-MM-dd'T'HH:mm:ss.SSS</str>
+				<str>yyyy-MM-dd'T'HH:mm:ss,SSS</str>
+				<str>yyyy-MM-dd'T'HH:mm:ssZ</str>
+				<str>yyyy-MM-dd'T'HH:mm:ss</str>
+				<str>yyyy-MM-dd'T'HH:mmZ</str>
+				<str>yyyy-MM-dd'T'HH:mm</str>
+				<str>yyyy-MM-dd HH:mm:ss.SSSZ</str>
+				<str>yyyy-MM-dd HH:mm:ss,SSSZ</str>
+				<str>yyyy-MM-dd HH:mm:ss.SSS</str>
+				<str>yyyy-MM-dd HH:mm:ss,SSS</str>
+				<str>yyyy-MM-dd HH:mm:ssZ</str>
+				<str>yyyy-MM-dd HH:mm:ss</str>
+				<str>yyyy-MM-dd HH:mmZ</str>
+				<str>yyyy-MM-dd HH:mm</str>
+				<str>yyyy-MM-dd</str>
+			</arr>
+		</processor>
+		<processor class="solr.RunUpdateProcessorFactory" />
+	</updateRequestProcessorChain>
+	
+	<queryResponseWriter name="json" class="solr.JSONResponseWriter">
+		<str name="content-type">text/plain; charset=UTF-8</str>
+	</queryResponseWriter>
+	
+	<queryResponseWriter name="velocity" class="solr.VelocityResponseWriter" startup="lazy">
+		<str name="template.base.dir">${velocity.template.base.dir:}</str>
+		<str name="solr.resource.loader.enabled">${velocity.solr.resource.loader.enabled:true}</str>
+		<str name="params.resource.loader.enabled">${velocity.params.resource.loader.enabled:false}</str>
+	</queryResponseWriter>
+	
+	<queryResponseWriter name="xslt" class="solr.XSLTResponseWriter">
+		<int name="xsltCacheLifetimeSeconds">5</int>
+	</queryResponseWriter>
+	
+</config>
\ No newline at end of file

From 00d69a40fce63174089bbe56d5cfffd19c21c91a Mon Sep 17 00:00:00 2001
From: Brett <brett@kowh.ai>
Date: Fri, 29 Oct 2021 13:59:38 +0200
Subject: [PATCH 08/10] Update docker-compose.yml

---
 docker-compose.yml | 48 +++++++++++++++++++++++++++++-----------------
 1 file changed, 30 insertions(+), 18 deletions(-)

diff --git a/docker-compose.yml b/docker-compose.yml
index b2f5aa1..c99e3d4 100755
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -1,5 +1,10 @@
 version: "3"
 
+volumes:
+  pg_data:
+  solr_data:
+  ckan_storage:
+
 services:
 
   nginx:
@@ -11,7 +16,7 @@ services:
       - ckan
     ports:
       - "0.0.0.0:80:80"
-    
+
   ckan:
     container_name: ckan
     build:
@@ -29,18 +34,13 @@ services:
     ports:
       - "0.0.0.0:${CKAN_PORT}:5000"
     volumes:
-      - ckan_storage:/var/lib/ckan
-    
-  datapusher:
-    container_name: datapusher
-    image: kowhai/datapusher:0.0.17
-    ports:
-      - "8800:8800"
+      - ckan_storage:/var/lib/ckan 
 
   db:
     container_name: db
     build:
-      context: postgresql/
+      context: .
+      dockerfile: postgresql/Dockerfile
       args:
         - DATASTORE_READONLY_PASSWORD=${DATASTORE_READONLY_PASSWORD}
         - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
@@ -50,19 +50,31 @@ services:
       - PGDATA=/var/lib/postgresql/data/db
     volumes:
       - pg_data:/var/lib/postgresql/data
-     
+    healthcheck:
+      test: ["CMD", "pg_isready", "-U", "ckan"]
+
+  datapusher:
+    container_name: datapusher
+    build:
+      context: datapusher
+    ports:
+      - "8800:8800"
+    environment:
+      - DATAPUSHER_REWRITE_RESOURCES=${DATAPUSHER_REWRITE_RESOURCES}
+      - DATAPUSHER_REWRITE_URL=${DATAPUSHER_REWRITE_URL}
+      - CKAN__DATAPUSHER__CALLBACK_URL_BASE=${CKAN__DATAPUSHER__CALLBACK_URL_BASE} 
+      
   solr:
     container_name: solr
     build:
-      context: solr/
+      context: .
+      dockerfile: solr/Dockerfile
+      args:
+        - SOLR_VERSION=${SOLR_VERSION}
+        - CKAN_VERSION=${CKAN_VERSION}        
     volumes:
-      - solr_data:/opt/solr/server/solr/ckan/data/index
+      - solr_data:/opt/solr/server/solr/ckan/data
 
   redis:
     container_name: redis
-    image: redis:alpine
-
-volumes:
-  ckan_storage:
-  pg_data:
-  solr_data:
+    image: redis:${REDIS_VERSION}

From 69be7da09f63f5f7d831bd0a9838c3d720a76399 Mon Sep 17 00:00:00 2001
From: Brett <brett@kowh.ai>
Date: Fri, 29 Oct 2021 14:01:58 +0200
Subject: [PATCH 09/10] Text file changes

---
 .env      | 54 +++++++++++++++++--------------------------
 README    | 20 ++++++++--------
 notes.txt | 69 +++++++++++++++++++++++++++++++++++++++++++++++++++++++
 todo.txt  | 12 ++++++++++
 4 files changed, 113 insertions(+), 42 deletions(-)
 create mode 100644 notes.txt
 create mode 100644 todo.txt

diff --git a/.env b/.env
index a3d9300..211d393 100644
--- a/.env
+++ b/.env
@@ -1,4 +1,10 @@
-# CKAN databases
+# CKAN
+CKAN_VERSION=2.9.3
+CKAN_SITE_URL=http://localhost:5000
+CKAN_SITE_ID=default
+CKAN_PORT=5000
+
+# Database
 POSTGRES_USER=ckan
 POSTGRES_PASSWORD=ckan
 DATASTORE_READONLY_USER=datastore_ro
@@ -8,39 +14,21 @@ CKAN_SQLALCHEMY_URL=postgresql://ckan:ckan@db/ckan
 CKAN_DATASTORE_WRITE_URL=postgresql://ckan:ckan@db/datastore
 CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:datastore@db/datastore
 
-# Test database connections
-TEST_CKAN_SQLALCHEMY_URL=postgres://ckan:ckan@db/ckan_test
-TEST_CKAN_DATASTORE_WRITE_URL=postgresql://ckan:ckan@db/datastore_test
-TEST_CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:datastore@db/datastore_test
-
-# CKAN core
-CKAN_SITE_ID=default
-CKAN_SITE_URL=http://ckan:5000
-CKAN_PORT=5000
-CKAN_SYSADMIN_NAME=ckan_admin
-CKAN_SYSADMIN_PASSWORD=test1234
-CKAN_SYSADMIN_EMAIL=your_email@example.com
-CKAN_STORAGE_PATH=/var/lib/ckan
-CKAN_SMTP_SERVER=smtp.corporateict.domain:25
-CKAN_SMTP_STARTTLS=True
-CKAN_SMTP_USER=user
-CKAN_SMTP_PASSWORD=pass
-CKAN_SMTP_MAIL_FROM=ckan@localhost
-TZ=UTC
-
-# Other services
+# solr
+SOLR_VERSION=6.6.6
 CKAN_SOLR_URL=http://solr:8983/solr/ckan
-CKAN_REDIS_URL=redis://redis:6379/1
-CKAN_DATAPUSHER_URL=http://datapusher:8801
+
+# Datapusher
+DATAPUSHER_VERSION=0.0.17
+CKAN_DATAPUSHER_URL=http://datapusher:8800
 CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://ckan:5000
+DATAPUSHER_REWRITE_RESOURCES=True
+DATAPUSHER_REWRITE_URL=http://ckan:5000  
 
-# test connections
-TEST_CKAN_SOLR_URL=http://solr:8983/solr/ckan
-TEST_CKAN_REDIS_URL=redis://redis:6379/1
+# Redis
+REDIS_VERSION=6.0.7
+CKAN_REDIS_URL=redis://redis:6379/1
 
-# Extensions
-CKAN__PLUGINS=envvars image_view text_view recline_view datastore datapusher
-CKAN__HARVEST__MQ__TYPE=redis
-CKAN__HARVEST__MQ__HOSTNAME=redis
-CKAN__HARVEST__MQ__PORT=6379
-CKAN__HARVEST__MQ__REDIS_DB=1
\ No newline at end of file
+CKAN_SYSADMIN_NAME=admin
+CKAN_SYSADMIN_PASSWORD=test1234
+CKAN_SYSADMIN_EMAIL=admin@ckan.org
diff --git a/README b/README
index 703c891..9c85779 100755
--- a/README
+++ b/README
@@ -1,10 +1,12 @@
-Build the images from this directory using:
+Build ckan-base image (2.9)
+cd /path/to/ckan-docker/ckan/ckan-base
+docker build -t ckan/ckan-base:testing-only.2.9 -f 2.9/Dockerfile .
+ 
+Login to Dockerhub
+docker login --username=kowhai
+(will then ask for a password)
 
-    cd $REPO_HOME
-    cd images/ckan-base
-    docker build -t ckan/ckan-base:testing-only-2.9 -f 2.9/Dockerfile .
-    docker push ckan/ckan-base:testing-only-2.9
-    docker build -t ckan/ckan-base:testing-only-2.8 -f 2.8/Dockerfile .
-    docker push ckan/ckan-base:testing-only-2.8
-    docker build -t ckan/ckan-base:testing-only-2.7 -f 2.7/Dockerfile .
-    docker push ckan/ckan-base:testing-only-2.7
+Push image to (ckan) Dockerhub
+docker push ckan/ckan-base:testing-only.2.9
+
+Do the same with 2.8 and 2.7
diff --git a/notes.txt b/notes.txt
new file mode 100644
index 0000000..dccc9b1
--- /dev/null
+++ b/notes.txt
@@ -0,0 +1,69 @@
+
+Environment variables are in the .env file and loaded into the docker-compose.yml file at runtime.
+The Dockerfile will need an ARG <value> line also
+
+List all env variables and decide where they go
+.env
+env-ckan
+ARG
+ENV
+
+NB:     .env variables can be referenced in the docker-compose.yml file (environment: part of the service)
+        The docker-compose.yml file can pass these variables to the Dockerfile as ARG parameters
+--------------------------- from OKFN ---------------------------
+# DB image settings
+POSTGRES_PASSWORD=ckan
+DATASTORE_READONLY_PASSWORD=datastore
+
+# Basic
+CKAN_SITE_ID=default
+CKAN_SITE_URL=http://ckan:5000
+CKAN_PORT=5000
+CKAN_SYSADMIN_NAME=ckan_admin
+CKAN_SYSADMIN_PASSWORD=test1234
+CKAN_SYSADMIN_EMAIL=your_email@example.com
+TZ=UTC
+
+# Database connections (TODO: avoid duplication)
+CKAN_SQLALCHEMY_URL=postgresql://ckan:ckan@db/ckan
+CKAN_DATASTORE_WRITE_URL=postgresql://ckan:ckan@db/datastore
+CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:datastore@db/datastore
+
+# Test database connections
+TEST_CKAN_SQLALCHEMY_URL=postgres://ckan:ckan@db/ckan_test
+TEST_CKAN_DATASTORE_WRITE_URL=postgresql://ckan:ckan@db/datastore_test
+TEST_CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:datastore@db/datastore_test
+
+# Other services connections
+CKAN_SOLR_URL=http://solr:8983/solr/ckan
+CKAN_REDIS_URL=redis://redis:6379/1
+CKAN_DATAPUSHER_URL=http://datapusher:8800
+CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://ckan:5000
+
+TEST_CKAN_SOLR_URL=http://solr:8983/solr/ckan
+TEST_CKAN_REDIS_URL=redis://redis:6379/1
+
+# Core settings
+CKAN__STORAGE_PATH=/var/lib/ckan
+
+CKAN_SMTP_SERVER=smtp.corporateict.domain:25
+CKAN_SMTP_STARTTLS=True
+CKAN_SMTP_USER=user
+CKAN_SMTP_PASSWORD=pass
+CKAN_SMTP_MAIL_FROM=ckan@localhost
+
+# Extensions
+
+CKAN__PLUGINS=envvars image_view text_view recline_view datastore datapusher
+CKAN__HARVEST__MQ__TYPE=redis
+CKAN__HARVEST__MQ__HOSTNAME=redis
+CKAN__HARVEST__MQ__PORT=6379
+CKAN__HARVEST__MQ__REDIS_DB=1
+
+
+docker-compose.yml
+DATASTORE_READONLY_PASSWORD=${DATASTORE_READONLY_PASSWORD}
+POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
+DATASTORE_READONLY_PASSWORD=${DATASTORE_READONLY_PASSWORD}
+POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
+PGDATA=/var/lib/postgresql/data/db
\ No newline at end of file
diff --git a/todo.txt b/todo.txt
new file mode 100644
index 0000000..d5d6960
--- /dev/null
+++ b/todo.txt
@@ -0,0 +1,12 @@
+Get the NGINX container running                                     <done>
+Add the redis container                                             <done>
+Add the solr container                                              <done>
+Add the db container                                                <done>
+Get a datapusher container working with supervisord                 <done>
+Test the datapusher container works
+Take out ARG's from docker-compose.yml is not needed
+Test with new versions of CKAN (ie: if problems with Python 3.9)
+Create (and push) the datapusher image
+Create the ckan container
+Create the ckanext container
+ 

From c47478502608ce66e9a6a99ebd1cd2285bc6b986 Mon Sep 17 00:00:00 2001
From: Brett <brett@kowh.ai>
Date: Fri, 29 Oct 2021 14:19:27 +0200
Subject: [PATCH 10/10] More solr changes

---
 solr/Dockerfile                               |  24 +-
 .../solrconfig-2.9.3.xml                      |   0
 solr/solrconfig.xml                           | 345 ------------------
 3 files changed, 13 insertions(+), 356 deletions(-)
 rename solrconfig-2.9.3.xml => solr/solrconfig-2.9.3.xml (100%)
 delete mode 100755 solr/solrconfig.xml

diff --git a/solr/Dockerfile b/solr/Dockerfile
index 8922342..1e0c650 100755
--- a/solr/Dockerfile
+++ b/solr/Dockerfile
@@ -2,9 +2,10 @@ FROM solr:6.6.6
 
 # Enviroment variables
 ENV SOLR_CORE ckan
-ENV SOLR_VERSION 6.6.6
-ENV CKAN_VERSION 2.9.1
-###TODO!!! CKAN_VERSION to be passed in as an ARG
+
+# Build Arguments
+ARG CKAN_VERSION
+ARG SOLR_VERSION
 
 # root user for initial config
 USER root
@@ -15,14 +16,14 @@ RUN mkdir -p /opt/solr/server/solr/${SOLR_CORE}/conf && \
     mkdir -p /opt/solr/server/solr/${SOLR_CORE}/data/index
 
 # Add files
-ADD solrconfig.xml \
-https://raw.githubusercontent.com/ckan/ckan/ckan-${CKAN_VERSION}/ckan/config/solr/schema.xml \
-https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/$SOLR_VERSION/solr/server/solr/configsets/basic_configs/conf/currency.xml \
-https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/$SOLR_VERSION/solr/server/solr/configsets/basic_configs/conf/synonyms.txt \
-https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/$SOLR_VERSION/solr/server/solr/configsets/basic_configs/conf/stopwords.txt \
-https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/$SOLR_VERSION/solr/server/solr/configsets/basic_configs/conf/protwords.txt \
-https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/$SOLR_VERSION/solr/server/solr/configsets/data_driven_schema_configs/conf/elevate.xml \
-/opt/solr/server/solr/$SOLR_CORE/conf/
+COPY ./solr/solrconfig-${CKAN_VERSION}.xml /opt/solr/server/solr/${SOLR_CORE}/conf/solrconfig.xml
+ADD https://raw.githubusercontent.com/ckan/ckan/ckan-${CKAN_VERSION}/ckan/config/solr/schema.xml \
+https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/${SOLR_VERSION}/solr/server/solr/configsets/basic_configs/conf/currency.xml \
+https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/${SOLR_VERSION}/solr/server/solr/configsets/basic_configs/conf/synonyms.txt \
+https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/${SOLR_VERSION}/solr/server/solr/configsets/basic_configs/conf/stopwords.txt \
+https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/${SOLR_VERSION}/solr/server/solr/configsets/basic_configs/conf/protwords.txt \
+https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/${SOLR_VERSION}/solr/server/solr/configsets/data_driven_schema_configs/conf/elevate.xml \
+/opt/solr/server/solr/${SOLR_CORE}/conf/
 
 # Create core.properties
 RUN echo name=${SOLR_CORE} > /opt/solr/server/solr/${SOLR_CORE}/core.properties
@@ -32,3 +33,4 @@ RUN chown -R ${SOLR_USER}:${SOLR_USER} /opt/solr/server/solr/${SOLR_CORE}
 
 # non-root user for runtime
 USER ${SOLR_USER}:${SOLR_USER}
+
diff --git a/solrconfig-2.9.3.xml b/solr/solrconfig-2.9.3.xml
similarity index 100%
rename from solrconfig-2.9.3.xml
rename to solr/solrconfig-2.9.3.xml
diff --git a/solr/solrconfig.xml b/solr/solrconfig.xml
deleted file mode 100755
index 8a3eade..0000000
--- a/solr/solrconfig.xml
+++ /dev/null
@@ -1,345 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-
-<!--solrconfig.xml documentation [https://wiki.apache.org/solr/SolrConfigXml]-->
-<config>
- 
-	<luceneMatchVersion>6.0.0</luceneMatchVersion>
-	
-	<lib dir="${solr.install.dir:../../../..}/contrib/extraction/lib" regex=".*\.jar" />
-	<lib dir="${solr.install.dir:../../../..}/dist/" regex="solr-cell-\d.*\.jar" />
-	<lib dir="${solr.install.dir:../../../..}/contrib/clustering/lib/" regex=".*\.jar" />
-	<lib dir="${solr.install.dir:../../../..}/dist/" regex="solr-clustering-\d.*\.jar" />
-	<lib dir="${solr.install.dir:../../../..}/contrib/langid/lib/" regex=".*\.jar" />
-	<lib dir="${solr.install.dir:../../../..}/dist/" regex="solr-langid-\d.*\.jar" />
-	<lib dir="${solr.install.dir:../../../..}/contrib/velocity/lib" regex=".*\.jar" />
-	<lib dir="${solr.install.dir:../../../..}/dist/" regex="solr-velocity-\d.*\.jar" />
-	
-	<dataDir>${solr.data.dir:}</dataDir>
-	
-	<directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.NRTCachingDirectoryFactory}" />
-	
-	<codecFactory class="solr.SchemaCodecFactory" />
-	
-	<indexConfig>
-		<lockType>${solr.lock.type:native}</lockType>
-	</indexConfig>
-	
-	<jmx />
-	
-	<updateHandler class="solr.DirectUpdateHandler2">
-	 
-		<updateLog>
-			<str name="dir">${solr.ulog.dir:}</str>
-			<int name="numVersionBuckets">${solr.ulog.numVersionBuckets:65536}</int>
-		</updateLog>
-		
-		<autoCommit>
-			<maxTime>${solr.autoCommit.maxTime:15000}</maxTime>
-			<openSearcher>false</openSearcher>
-		</autoCommit>
-		
-		<autoSoftCommit>
-			<maxTime>${solr.autoSoftCommit.maxTime:-1}</maxTime>
-		</autoSoftCommit>
-		
-	</updateHandler>
-	
-	<query>
-	 
-		<maxBooleanClauses>1024</maxBooleanClauses>
-		<filterCache class="solr.FastLRUCache" size="512" initialSize="512" autowarmCount="0" />
-		<queryResultCache class="solr.LRUCache" size="512" initialSize="512" autowarmCount="0" />
-		<documentCache class="solr.LRUCache" size="512" initialSize="512" autowarmCount="0" />
-		<cache name="perSegFilter" class="solr.search.LRUCache" size="10" initialSize="0" autowarmCount="10" regenerator="solr.NoOpRegenerator" />
-		<enableLazyFieldLoading>true</enableLazyFieldLoading>
-		<queryResultWindowSize>20</queryResultWindowSize>
-		<queryResultMaxDocsCached>200</queryResultMaxDocsCached>
-		
-		<listener event="newSearcher" class="solr.QuerySenderListener">
-			<arr name="queries" />
-		</listener>
-		
-		<listener event="firstSearcher" class="solr.QuerySenderListener">
-			<arr name="queries" />
-		</listener>
-		
-		<useColdSearcher>false</useColdSearcher>
-		<maxWarmingSearchers>2</maxWarmingSearchers>
-		
-	</query>
-	
-	<requestDispatcher handleSelect="false">
-	 
-		<requestParsers enableRemoteStreaming="true" multipartUploadLimitInKB="2048000" formdataUploadLimitInKB="2048" addHttpRequestToContext="false" />
-		<httpCaching never304="true" />
-		
-	</requestDispatcher>
-	
-	<requestHandler name="/select" class="solr.SearchHandler">
-	 
-		<lst name="defaults">
-			<str name="echoParams">explicit</str>
-			<int name="rows">10</int>
-		</lst>
-		
-	</requestHandler>
-	
-	<requestHandler name="/query" class="solr.SearchHandler">
-	 
-		<lst name="defaults">
-			<str name="echoParams">explicit</str>
-			<str name="wt">json</str>
-			<str name="indent">true</str>
-		</lst>
-		
-	</requestHandler>
-	
-	<requestHandler name="/browse" class="solr.SearchHandler" useParams="query,facets,velocity,browse">
-	 
-		<lst name="defaults">
-			<str name="echoParams">explicit</str>
-		</lst>
-		
-	</requestHandler>
-	
-	<initParams path="/update/**,/query,/select,/tvrh,/elevate,/spell,/browse">
-	 
-		<lst name="defaults">
-			<str name="df">_text_</str>
-		</lst>
-		
-	</initParams>
-	
-	<initParams path="/update/**">
-	 
-		<lst name="defaults">
-			<str name="update.chain">add-unknown-fields-to-the-schema</str>
-		</lst>
-		
-	</initParams>
-	
-	<requestHandler name="/update/extract" startup="lazy" class="solr.extraction.ExtractingRequestHandler">
-	 
-		<lst name="defaults">
-			<str name="lowernames">true</str>
-			<str name="fmap.meta">ignored_</str>
-			<str name="fmap.content">_text_</str>
-		</lst>
-		
-	</requestHandler>
-	
-	<requestHandler name="/analysis/field" startup="lazy" class="solr.FieldAnalysisRequestHandler" />
-	<requestHandler name="/analysis/document" class="solr.DocumentAnalysisRequestHandler" startup="lazy" />
-	
-	<requestHandler name="/debug/dump" class="solr.DumpRequestHandler">
-	 
-		<lst name="defaults">
-			<str name="echoParams">explicit</str>
-			<str name="echoHandler">true</str>
-		</lst>
-		
-	</requestHandler>
-	
-	<searchComponent name="spellcheck" class="solr.SpellCheckComponent">
-	 
-		<str name="queryAnalyzerFieldType">text_general</str>
-		
-		<lst name="spellchecker">
-			<str name="name">default</str>
-			<str name="field">_text_</str>
-			<str name="classname">solr.DirectSolrSpellChecker</str>
-			<str name="distanceMeasure">internal</str>
-			<float name="accuracy">0.5</float>
-			<int name="maxEdits">2</int>
-			<int name="minPrefix">1</int>
-			<int name="maxInspections">5</int>
-			<int name="minQueryLength">4</int>
-			<float name="maxQueryFrequency">0.01</float>
-		</lst>
-		
-	</searchComponent>
-	
-	<requestHandler name="/spell" class="solr.SearchHandler" startup="lazy">
-	 
-		<lst name="defaults">
-			<str name="spellcheck.dictionary">default</str>
-			<str name="spellcheck">on</str>
-			<str name="spellcheck.extendedResults">true</str>
-			<str name="spellcheck.count">10</str>
-			<str name="spellcheck.alternativeTermCount">5</str>
-			<str name="spellcheck.maxResultsForSuggest">5</str>
-			<str name="spellcheck.collate">true</str>
-			<str name="spellcheck.collateExtendedResults">true</str>
-			<str name="spellcheck.maxCollationTries">10</str>
-			<str name="spellcheck.maxCollations">5</str>
-		</lst>
-		
-		<arr name="last-components">
-			<str>spellcheck</str>
-		</arr>
-		
-	</requestHandler>
-	
-	<searchComponent name="tvComponent" class="solr.TermVectorComponent" />
-	
-	<requestHandler name="/tvrh" class="solr.SearchHandler" startup="lazy">
-	 
-		<lst name="defaults">
-			<bool name="tv">true</bool>
-		</lst>
-		<arr name="last-components">
-			<str>tvComponent</str>
-		</arr>
-		
-	</requestHandler>
-	
-	<searchComponent name="terms" class="solr.TermsComponent" />
-	
-	<requestHandler name="/terms" class="solr.SearchHandler" startup="lazy">
-	 
-		<lst name="defaults">
-			<bool name="terms">true</bool>
-			<bool name="distrib">false</bool>
-		</lst>
-		
-		<arr name="components">
-			<str>terms</str>
-		</arr>
-		
-	</requestHandler>
-	
-	<searchComponent name="elevator" class="solr.QueryElevationComponent">
-	 
-		<str name="queryFieldType">string</str>
-		<str name="config-file">elevate.xml</str>
-		
-	</searchComponent>
-	
-	<requestHandler name="/elevate" class="solr.SearchHandler" startup="lazy">
-	 
-		<lst name="defaults">
-			<str name="echoParams">explicit</str>
-		</lst>
-		<arr name="last-components">
-			<str>elevator</str>
-		</arr>
-		
-	</requestHandler>
-	
-	<searchComponent class="solr.HighlightComponent" name="highlight">
-	 
-		<highlighting>
-		 
-			<fragmenter name="gap" default="true" class="solr.highlight.GapFragmenter">
-				<lst name="defaults">
-					<int name="hl.fragsize">100</int>
-				</lst>
-			</fragmenter>
-			
-			<fragmenter name="regex" class="solr.highlight.RegexFragmenter">
-				<lst name="defaults">
-					<int name="hl.fragsize">70</int>
-					<float name="hl.regex.slop">0.5</float>
-					<str name="hl.regex.pattern">[-\w ,/\n\"']{20,200}</str>
-				</lst>
-			</fragmenter>
-			
-			<formatter name="html" default="true" class="solr.highlight.HtmlFormatter">
-				<lst name="defaults">
-					<str name="hl.simple.pre"><![CDATA[<em>]]></str>
-					<str name="hl.simple.post"><![CDATA[</em>]]></str>
-				</lst>
-			</formatter>
-			
-			<encoder name="html" class="solr.highlight.HtmlEncoder" />
-			<fragListBuilder name="simple" class="solr.highlight.SimpleFragListBuilder" />
-			<fragListBuilder name="single" class="solr.highlight.SingleFragListBuilder" />
-			<fragListBuilder name="weighted" default="true" class="solr.highlight.WeightedFragListBuilder" />
-			<fragmentsBuilder name="default" default="true" class="solr.highlight.ScoreOrderFragmentsBuilder" />
-			<fragmentsBuilder name="colored" class="solr.highlight.ScoreOrderFragmentsBuilder">
-			 
-				<lst name="defaults">
-					<str name="hl.tag.pre"><![CDATA[<b style="background:yellow">,<b style="background:lawgreen">,
-               <b style="background:aquamarine">,<b style="background:magenta">,
-               <b style="background:palegreen">,<b style="background:coral">,
-               <b style="background:wheat">,<b style="background:khaki">,
-               <b style="background:lime">,<b style="background:deepskyblue">]]></str>
-					<str name="hl.tag.post"><![CDATA[</b>]]></str>
-				</lst>
-				
-			</fragmentsBuilder>
-			
-			<boundaryScanner name="default" default="true" class="solr.highlight.SimpleBoundaryScanner">
-				<lst name="defaults">
-					<str name="hl.bs.maxScan">10</str>
-					<str name="hl.bs.chars">.,!?</str>
-				</lst>
-			</boundaryScanner>
-			
-			<boundaryScanner name="breakIterator" class="solr.highlight.BreakIteratorBoundaryScanner">
-				<lst name="defaults">
-					<str name="hl.bs.type">WORD</str>
-					<str name="hl.bs.language">en</str>
-					<str name="hl.bs.country">US</str>
-				</lst>
-			</boundaryScanner>
-			
-		</highlighting>
-		
-	</searchComponent>
-	
-	<schemaFactory class="ManagedIndexSchemaFactory">
-		<bool name="mutable">true</bool>
-	</schemaFactory>
-	
-	<updateRequestProcessorChain name="add-unknown-fields-to-the-schema">
-		<processor class="solr.UUIDUpdateProcessorFactory" />
-		<processor class="solr.LogUpdateProcessorFactory" />
-		<processor class="solr.DistributedUpdateProcessorFactory" />
-		<processor class="solr.RemoveBlankFieldUpdateProcessorFactory" />
-		<processor class="solr.FieldNameMutatingUpdateProcessorFactory">
-			<str name="pattern">[^\w-\.]</str>
-			<str name="replacement">_</str>
-		</processor>
-		<processor class="solr.ParseBooleanFieldUpdateProcessorFactory" />
-		<processor class="solr.ParseLongFieldUpdateProcessorFactory" />
-		<processor class="solr.ParseDoubleFieldUpdateProcessorFactory" />
-		<processor class="solr.ParseDateFieldUpdateProcessorFactory">
-			<arr name="format">
-				<str>yyyy-MM-dd'T'HH:mm:ss.SSSZ</str>
-				<str>yyyy-MM-dd'T'HH:mm:ss,SSSZ</str>
-				<str>yyyy-MM-dd'T'HH:mm:ss.SSS</str>
-				<str>yyyy-MM-dd'T'HH:mm:ss,SSS</str>
-				<str>yyyy-MM-dd'T'HH:mm:ssZ</str>
-				<str>yyyy-MM-dd'T'HH:mm:ss</str>
-				<str>yyyy-MM-dd'T'HH:mmZ</str>
-				<str>yyyy-MM-dd'T'HH:mm</str>
-				<str>yyyy-MM-dd HH:mm:ss.SSSZ</str>
-				<str>yyyy-MM-dd HH:mm:ss,SSSZ</str>
-				<str>yyyy-MM-dd HH:mm:ss.SSS</str>
-				<str>yyyy-MM-dd HH:mm:ss,SSS</str>
-				<str>yyyy-MM-dd HH:mm:ssZ</str>
-				<str>yyyy-MM-dd HH:mm:ss</str>
-				<str>yyyy-MM-dd HH:mmZ</str>
-				<str>yyyy-MM-dd HH:mm</str>
-				<str>yyyy-MM-dd</str>
-			</arr>
-		</processor>
-		<processor class="solr.RunUpdateProcessorFactory" />
-	</updateRequestProcessorChain>
-	
-	<queryResponseWriter name="json" class="solr.JSONResponseWriter">
-		<str name="content-type">text/plain; charset=UTF-8</str>
-	</queryResponseWriter>
-	
-	<queryResponseWriter name="velocity" class="solr.VelocityResponseWriter" startup="lazy">
-		<str name="template.base.dir">${velocity.template.base.dir:}</str>
-		<str name="solr.resource.loader.enabled">${velocity.solr.resource.loader.enabled:true}</str>
-		<str name="params.resource.loader.enabled">${velocity.params.resource.loader.enabled:false}</str>
-	</queryResponseWriter>
-	
-	<queryResponseWriter name="xslt" class="solr.XSLTResponseWriter">
-		<int name="xsltCacheLifetimeSeconds">5</int>
-	</queryResponseWriter>
-	
-</config>