Commit new images directory

README

@@ -0,0 +1,10 @@
+Build the images from this directory using:
+
+    cd $REPO_HOME
+    cd images/ckan-base
+    docker build -t ckan/ckan-base:testing-only-2.9 -f 2.9/Dockerfile .
+    docker push ckan/ckan-base:testing-only-2.9
+    docker build -t ckan/ckan-base:testing-only-2.8 -f 2.8/Dockerfile .
+    docker push ckan/ckan-base:testing-only-2.8
+    docker build -t ckan/ckan-base:testing-only-2.7 -f 2.7/Dockerfile .
+    docker push ckan/ckan-base:testing-only-2.7

ckan-base/2.7/Dockerfile

@@ -0,0 +1,101 @@
+FROM alpine:3.13
+
+# Internals, you probably don't need to change these
+ENV APP_DIR=/srv/app
+ENV SRC_DIR=/srv/app/src
+ENV CKAN_INI=${APP_DIR}/production.ini
+ENV PIP_SRC=${SRC_DIR}
+ENV CKAN_STORAGE_PATH=/var/lib/ckan
+ENV GIT_URL=https://github.com/ckan/ckan.git
+# CKAN version to build
+ENV GIT_BRANCH=ckan-2.7.11
+# Customize these on the .env file if needed
+ENV CKAN_SITE_URL=http://localhost:5000
+ENV CKAN__PLUGINS image_view text_view recline_view datastore datapusher envvars
+
+WORKDIR ${APP_DIR}
+
+# Install necessary packages to run CKAN
+RUN apk add --no-cache tzdata \
+        git \
+        gettext \
+        postgresql-client \
+        python \
+        apache2-utils \
+        libxml2 \
+        libxslt \
+        musl-dev \
+        uwsgi \
+        uwsgi-http \
+        uwsgi-corerouter \
+        uwsgi-python \
+        py2-gevent \
+        uwsgi-gevent \
+        libmagic \
+        curl \
+        sudo && \
+    # Packages to build CKAN requirements and plugins
+    apk add --no-cache --virtual .build-deps \
+        postgresql-dev \
+        gcc \
+        make \
+        g++ \
+        autoconf \
+        automake \
+	libtool \
+        python-dev \
+        libxml2-dev \
+        libxslt-dev \
+        linux-headers
+
+# Create SRC_DIR
+RUN mkdir -p ${SRC_DIR}
+
+# Install pip
+RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/pip/2.7/get-pip.py && \
+    python ${SRC_DIR}/get-pip.py 'pip==20.3.3'
+
+# Install pip, supervisord and uwsgi
+RUN pip install supervisor && \
+    mkdir /etc/supervisord.d && \
+    #pip wheel --wheel-dir=/wheels uwsgi gevent && \
+    rm -rf ${SRC_DIR}/get-pip.py
+
+COPY 2.7/setup/supervisord.conf /etc
+
+# Install CKAN
+RUN pip install -e git+${GIT_URL}@${GIT_BRANCH}#egg=ckan && \
+    cd ${SRC_DIR}/ckan && \
+    cp who.ini ${APP_DIR} && \
+    # Workaround to solve https://github.com/psycopg/psycopg2/issues/594 in Alpine 3.7
+	sed -i -e "s/psycopg2==2.4.5/psycopg2==2.7.3.2/g" requirements.txt && \
+    pip install --no-binary :all: -r requirements.txt && \
+    # Install CKAN envvars to support loading config from environment variables
+    pip install -e git+https://github.com/okfn/ckanext-envvars.git#egg=ckanext-envvars && \
+    # Create and update CKAN config
+    paster --plugin=ckan make-config ckan ${CKAN_INI} && \
+    paster --plugin=ckan config-tool ${CKAN_INI} "ckan.plugins = ${CKAN__PLUGINS}" && \
+    paster --plugin=ckan config-tool ${CKAN_INI} "ckan.site_url = ${CKAN__SITE_URL}"
+
+# Create a local user and group to run the app
+RUN addgroup -g 92 -S ckan && \
+    adduser -u 92 -h /srv/app -H -D -S -G ckan ckan
+
+# Create local storage folder
+RUN mkdir -p $CKAN_STORAGE_PATH && \
+    chown -R ckan:ckan $CKAN_STORAGE_PATH
+
+COPY 2.7/setup ${APP_DIR}
+COPY 2.7/setup/supervisor.worker.conf /etc/supervisord.d/worker.conf
+COPY 2.7/setup/uwsgi.conf /srv/app/uwsgi.conf
+
+# Create entrypoint directory for children image scripts
+ONBUILD RUN mkdir /docker-entrypoint.d
+
+RUN chown ckan -R /srv/app
+
+EXPOSE 5000
+
+HEALTHCHECK --interval=10s --timeout=5s --retries=5 CMD curl --fail http://localhost:5000/api/3/action/status_show || exit 1
+
+CMD ["/srv/app/start_ckan.sh"]

ckan-base/2.7/setup/prerun.py

@@ -0,0 +1,197 @@
+import os
+import sys
+import subprocess
+import psycopg2
+import urllib2
+import time
+import re
+
+ckan_ini = os.environ.get('CKAN_INI', '/srv/app/production.ini')
+
+RETRY = 5
+
+def update_plugins():
+
+    plugins = os.environ.get('CKAN__PLUGINS', '')
+    print('[prerun] Setting the following plugins in {}:'.format(ckan_ini))
+    print(plugins)
+    cmd = ['paster', '--plugin=ckan', 'config-tool',
+           ckan_ini, 'ckan.plugins = {}'.format(plugins)]
+    subprocess.check_output(cmd, stderr=subprocess.STDOUT)
+    print '[prerun] Plugins set.'
+
+    
+def check_main_db_connection(retry=None):
+
+    conn_str = os.environ.get('CKAN_SQLALCHEMY_URL')
+    if not conn_str:
+        print '[prerun] CKAN_SQLALCHEMY_URL not defined, not checking db'
+    return check_db_connection(conn_str, retry)
+
+
+def check_datastore_db_connection(retry=None):
+
+    conn_str = os.environ.get('CKAN_DATASTORE_WRITE_URL')
+    if not conn_str:
+        print '[prerun] CKAN_DATASTORE_WRITE_URL not defined, not checking db'
+    return check_db_connection(conn_str, retry)
+
+
+def check_db_connection(conn_str, retry=None):
+
+    if retry is None:
+        retry = RETRY
+    elif retry == 0:
+        print '[prerun] Giving up after 5 tries...'
+        sys.exit(1)
+
+    try:
+        connection = psycopg2.connect(conn_str)
+
+    except psycopg2.Error as e:
+        print str(e)
+        print '[prerun] Unable to connect to the database, waiting...'
+        time.sleep(10)
+        check_db_connection(conn_str, retry=retry - 1)
+    else:
+        connection.close()
+
+        
+def check_solr_connection(retry=None):
+
+    if retry is None:
+        retry = RETRY
+    elif retry == 0:
+        print '[prerun] Giving up after 5 tries...'
+        sys.exit(1)
+
+    url = os.environ.get('CKAN_SOLR_URL', '')
+    search_url = '{url}/select/?q=*&wt=json'.format(url=url)
+
+    try:
+        connection = urllib2.urlopen(search_url)
+    except urllib2.URLError as e:
+        print str(e)
+        print '[prerun] Unable to connect to solr, waiting...'
+        time.sleep(10)
+        check_solr_connection(retry=retry - 1)
+    else:
+        eval(connection.read())
+
+
+def init_db():
+
+    db_command = ['paster', '--plugin=ckan', 'db',
+                  'init', '-c', ckan_ini]
+    print '[prerun] Initializing or upgrading db - start'
+    try:
+        subprocess.check_output(db_command, stderr=subprocess.STDOUT)
+        print '[prerun] Initializing or upgrading db - end'
+    except subprocess.CalledProcessError, e:
+        if 'OperationalError' in e.output:
+            print e.output
+            print '[prerun] Database not ready, waiting a bit before exit...'
+            time.sleep(5)
+            sys.exit(1)
+        else:
+            print e.output
+            raise e
+
+
+def init_datastore_db():
+
+    conn_str = os.environ.get('CKAN_DATASTORE_WRITE_URL')
+    if not conn_str:
+        print '[prerun] Skipping datastore initialization'
+        return
+
+    datastore_perms_command = ['paster', '--plugin=ckan', 'datastore',
+                               'set-permissions', '-c', ckan_ini]
+
+    connection = psycopg2.connect(conn_str)
+    cursor = connection.cursor()
+
+    print '[prerun] Initializing datastore db - start'
+    try:
+        datastore_perms = subprocess.Popen(
+            datastore_perms_command,
+            stdout=subprocess.PIPE)
+
+        perms_sql = datastore_perms.stdout.read()
+        # Remove internal pg command as psycopg2 does not like it
+        perms_sql = re.sub('\\\\connect \"(.*)\"', '', perms_sql)
+        cursor.execute(perms_sql)
+        for notice in connection.notices:
+            print notice
+
+        connection.commit()
+
+        print '[prerun] Initializing datastore db - end'
+        print datastore_perms.stdout.read()
+    except psycopg2.Error as e:
+        print '[prerun] Could not initialize datastore'
+        print str(e)
+
+    except subprocess.CalledProcessError, e:
+        if 'OperationalError' in e.output:
+            print e.output
+            print '[prerun] Database not ready, waiting a bit before exit...'
+            time.sleep(5)
+            sys.exit(1)
+        else:
+            print e.output
+            raise e
+    finally:
+        cursor.close()
+        connection.close()
+
+
+def create_sysadmin():
+
+    name = os.environ.get('CKAN_SYSADMIN_NAME')
+    password = os.environ.get('CKAN_SYSADMIN_PASSWORD')
+    email = os.environ.get('CKAN_SYSADMIN_EMAIL')
+
+    if name and password and email:
+
+        # Check if user exists
+        command = ['paster', '--plugin=ckan', 'user', name, '-c', ckan_ini]
+
+        out = subprocess.check_output(command)
+        if 'User:None' not in re.sub(r'\s', '', out):
+            print '[prerun] Sysadmin user exists, skipping creation'
+            return
+
+        # Create user
+        command = ['paster', '--plugin=ckan', 'user', 'add',
+                   name,
+                   'password=' + password,
+                   'email=' + email,
+                   '-c', ckan_ini]
+
+        subprocess.call(command)
+        print '[prerun] Created user {0}'.format(name)
+
+        # Make it sysadmin
+        command = ['paster', '--plugin=ckan', 'sysadmin', 'add',
+                   name,
+                   '-c', ckan_ini]
+
+        subprocess.call(command)
+        print '[prerun] Made user {0} a sysadmin'.format(name)
+
+
+if __name__ == '__main__':
+
+    maintenance = os.environ.get('MAINTENANCE_MODE', '').lower() == 'true'
+
+    if maintenance:
+        print '[prerun] Maintenance mode, skipping setup...'
+    else:
+        check_main_db_connection()
+        init_db()
+        update_plugins()
+        check_datastore_db_connection()
+        init_datastore_db()
+        check_solr_connection()
+        create_sysadmin()

ckan-base/2.7/setup/start_ckan.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+
+# Run the prerun script to init CKAN and create the default admin user
+sudo -u ckan -EH python prerun.py
+
+# Run any startup scripts provided by images extending this one
+if [[ -d "/docker-entrypoint.d" ]]
+then
+    for f in /docker-entrypoint.d/*; do
+        case "$f" in
+            *.sh)     echo "$0: Running init file $f"; . "$f" ;;
+            *.py)     echo "$0: Running init file $f"; python "$f"; echo ;;
+            *)        echo "$0: Ignoring $f (not an sh or py file)" ;;
+        esac
+        echo
+    done
+fi
+
+# Set the common uwsgi options
+UWSGI_OPTS="--plugins http,python,gevent --socket /tmp/uwsgi.sock --uid 92 --gid 92 --http :5000 --master --enable-threads --paste config:/srv/app/production.ini --paste-logger --lazy-apps --gevent 2000 -p 2 -L"
+
+# Check whether http basic auth password protection is enabled and enable basicauth routing on uwsgi respecfully
+if [ $? -eq 0 ]
+then
+  if [ "$PASSWORD_PROTECT" = true ]
+  then
+    if [ "$HTPASSWD_USER" ] || [ "$HTPASSWD_PASSWORD" ]
+    then
+      # Generate htpasswd file for basicauth
+      htpasswd -d -b -c /srv/app/.htpasswd $HTPASSWD_USER $HTPASSWD_PASSWORD
+      # Start supervisord
+      supervisord --configuration /etc/supervisord.conf &
+      # Start uwsgi with basicauth
+      sudo -u ckan -EH uwsgi --ini /srv/app/uwsgi.conf --pcre-jit $UWSGI_OPTS
+    else
+      echo "Missing HTPASSWD_USER or HTPASSWD_PASSWORD environment variables. Exiting..."
+      exit 1
+    fi
+  else
+    # Start supervisord
+    supervisord --configuration /etc/supervisord.conf &
+    # Start uwsgi
+    sudo -u ckan -EH uwsgi $UWSGI_OPTS
+  fi
+else
+  echo "[prerun] failed...not starting CKAN."
+fi

ckan-base/2.7/setup/supervisor.worker.conf

@@ -0,0 +1,13 @@
+[program:ckan-worker]
+command=paster --plugin=ckan jobs worker -c /srv/app/production.ini
+priority=501
+autostart=true
+autorestart=true
+redirect_stderr=true
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stdout
+stderr_logfile_maxbytes=0
+user=ckan
+environment=HOME="/srv/app",USER="ckan"
+

ckan-base/2.7/setup/supervisord.conf

@@ -0,0 +1,23 @@
+[unix_http_server]
+file = /tmp/supervisor.sock
+chmod = 0777
+chown = nobody:nogroup
+
+[supervisord]
+logfile = /tmp/supervisord.log
+logfile_maxbytes = 50MB
+logfile_backups=10
+loglevel = info
+pidfile = /tmp/supervisord.pid
+nodaemon = true
+umask = 022
+identifier = supervisor
+
+[supervisorctl]
+serverurl = unix:///tmp/supervisor.sock
+
+[rpcinterface:supervisor]
+supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
+
+[include]
+files = /etc/supervisord.d/*.conf

ckan-base/2.7/setup/uwsgi.conf

@@ -0,0 +1,2 @@
+[uwsgi]
+route = ^(?!/api).*$ basicauth:Restricted,/srv/app/.htpasswd

ckan-base/2.8/Dockerfile

@@ -0,0 +1,99 @@
+FROM alpine:3.13
+
+# Internals, you probably don't need to change these
+ENV APP_DIR=/srv/app
+ENV SRC_DIR=/srv/app/src
+ENV CKAN_INI=${APP_DIR}/production.ini
+ENV PIP_SRC=${SRC_DIR}
+ENV CKAN_STORAGE_PATH=/var/lib/ckan
+ENV GIT_URL=https://github.com/ckan/ckan.git
+# CKAN version to build
+ENV GIT_BRANCH=ckan-2.8.8
+# Customize these on the .env file if needed
+ENV CKAN_SITE_URL=http://localhost:5000
+ENV CKAN__PLUGINS image_view text_view recline_view datastore datapusher envvars
+
+WORKDIR ${APP_DIR}
+
+# Install necessary packages to run CKAN
+RUN apk add --no-cache tzdata \
+        git \
+        gettext \
+        postgresql-client \
+        python \
+        apache2-utils \
+        libxml2 \
+        libxslt \
+        musl-dev \
+        uwsgi \
+        uwsgi-http \
+        uwsgi-corerouter \
+        uwsgi-python \
+        py2-gevent \
+        uwsgi-gevent \
+        libmagic \
+        curl \
+        sudo && \
+    # Packages to build CKAN requirements and plugins
+    apk add --no-cache --virtual .build-deps \
+        postgresql-dev \
+        gcc \
+        make \
+        g++ \
+        autoconf \
+        automake \
+	    libtool \
+        python-dev \
+        libxml2-dev \
+        libxslt-dev \
+        linux-headers
+
+# Create SRC_DIR
+RUN mkdir -p ${SRC_DIR}
+    
+# Install pip
+RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/pip/2.7/get-pip.py && \
+    python ${SRC_DIR}/get-pip.py 'pip==20.3.3'
+
+# Install supervisord and uwsgi
+RUN pip install supervisor && \
+    mkdir /etc/supervisord.d && \
+    #pip wheel --wheel-dir=/wheels uwsgi gevent && \
+    rm -rf ${SRC_DIR}/get-pip.py
+
+COPY 2.8/setup/supervisord.conf /etc
+
+# Install CKAN
+RUN pip install -e git+${GIT_URL}@${GIT_BRANCH}#egg=ckan && \
+    cd ${SRC_DIR}/ckan && \
+    cp who.ini ${APP_DIR} && \
+    pip install --no-binary :all: -r requirements.txt && \
+    # Install CKAN envvars to support loading config from environment variables
+    pip install -e git+https://github.com/okfn/ckanext-envvars.git#egg=ckanext-envvars && \
+    # Create and update CKAN config
+    paster --plugin=ckan make-config ckan ${CKAN_INI} && \
+    paster --plugin=ckan config-tool ${CKAN_INI} "ckan.plugins = ${CKAN__PLUGINS}" && \
+    paster --plugin=ckan config-tool ${CKAN_INI} "ckan.site_url = ${CKAN__SITE_URL}"
+
+# Create a local user and group to run the app
+RUN addgroup -g 92 -S ckan && \
+    adduser -u 92 -h /srv/app -H -D -S -G ckan ckan
+
+# Create local storage folder
+RUN mkdir -p $CKAN_STORAGE_PATH && \
+    chown -R ckan:ckan $CKAN_STORAGE_PATH
+
+COPY 2.8/setup ${APP_DIR}
+COPY 2.8/setup/supervisor.worker.conf /etc/supervisord.d/worker.conf
+COPY 2.8/setup/uwsgi.conf /srv/app/uwsgi.conf
+
+# Create entrypoint directory for children image scripts
+ONBUILD RUN mkdir /docker-entrypoint.d
+
+RUN chown ckan -R /srv/app
+
+EXPOSE 5000
+
+HEALTHCHECK --interval=10s --timeout=5s --retries=5 CMD curl --fail http://localhost:5000/api/3/action/status_show || exit 1
+
+CMD ["/srv/app/start_ckan.sh"]

ckan-base/2.8/setup/prerun.py

@@ -0,0 +1,197 @@
+import os
+import sys
+import subprocess
+import psycopg2
+import urllib2
+import time
+import re
+
+ckan_ini = os.environ.get('CKAN_INI', '/srv/app/production.ini')
+
+RETRY = 5
+
+def update_plugins():
+
+    plugins = os.environ.get('CKAN__PLUGINS', '')
+    print('[prerun] Setting the following plugins in {}:'.format(ckan_ini))
+    print(plugins)
+    cmd = ['paster', '--plugin=ckan', 'config-tool',
+           ckan_ini, 'ckan.plugins = {}'.format(plugins)]
+    subprocess.check_output(cmd, stderr=subprocess.STDOUT)
+    print '[prerun] Plugins set.'
+
+    
+def check_main_db_connection(retry=None):
+
+    conn_str = os.environ.get('CKAN_SQLALCHEMY_URL')
+    if not conn_str:
+        print '[prerun] CKAN_SQLALCHEMY_URL not defined, not checking db'
+    return check_db_connection(conn_str, retry)
+
+
+def check_datastore_db_connection(retry=None):
+
+    conn_str = os.environ.get('CKAN_DATASTORE_WRITE_URL')
+    if not conn_str:
+        print '[prerun] CKAN_DATASTORE_WRITE_URL not defined, not checking db'
+    return check_db_connection(conn_str, retry)
+
+
+def check_db_connection(conn_str, retry=None):
+
+    if retry is None:
+        retry = RETRY
+    elif retry == 0:
+        print '[prerun] Giving up after 5 tries...'
+        sys.exit(1)
+
+    try:
+        connection = psycopg2.connect(conn_str)
+
+    except psycopg2.Error as e:
+        print str(e)
+        print '[prerun] Unable to connect to the database, waiting...'
+        time.sleep(10)
+        check_db_connection(conn_str, retry=retry - 1)
+    else:
+        connection.close()
+
+        
+def check_solr_connection(retry=None):
+
+    if retry is None:
+        retry = RETRY
+    elif retry == 0:
+        print '[prerun] Giving up after 5 tries...'
+        sys.exit(1)
+
+    url = os.environ.get('CKAN_SOLR_URL', '')
+    search_url = '{url}/select/?q=*&wt=json'.format(url=url)
+
+    try:
+        connection = urllib2.urlopen(search_url)
+    except urllib2.URLError as e:
+        print str(e)
+        print '[prerun] Unable to connect to solr, waiting...'
+        time.sleep(10)
+        check_solr_connection(retry=retry - 1)
+    else:
+        eval(connection.read())
+
+
+def init_db():
+
+    db_command = ['paster', '--plugin=ckan', 'db',
+                  'init', '-c', ckan_ini]
+    print '[prerun] Initializing or upgrading db - start'
+    try:
+        subprocess.check_output(db_command, stderr=subprocess.STDOUT)
+        print '[prerun] Initializing or upgrading db - end'
+    except subprocess.CalledProcessError, e:
+        if 'OperationalError' in e.output:
+            print e.output
+            print '[prerun] Database not ready, waiting a bit before exit...'
+            time.sleep(5)
+            sys.exit(1)
+        else:
+            print e.output
+            raise e
+
+
+def init_datastore_db():
+
+    conn_str = os.environ.get('CKAN_DATASTORE_WRITE_URL')
+    if not conn_str:
+        print '[prerun] Skipping datastore initialization'
+        return
+
+    datastore_perms_command = ['paster', '--plugin=ckan', 'datastore',
+                               'set-permissions', '-c', ckan_ini]
+
+    connection = psycopg2.connect(conn_str)
+    cursor = connection.cursor()
+
+    print '[prerun] Initializing datastore db - start'
+    try:
+        datastore_perms = subprocess.Popen(
+            datastore_perms_command,
+            stdout=subprocess.PIPE)
+
+        perms_sql = datastore_perms.stdout.read()
+        # Remove internal pg command as psycopg2 does not like it
+        perms_sql = re.sub('\\\\connect \"(.*)\"', '', perms_sql)
+        cursor.execute(perms_sql)
+        for notice in connection.notices:
+            print notice
+
+        connection.commit()
+
+        print '[prerun] Initializing datastore db - end'
+        print datastore_perms.stdout.read()
+    except psycopg2.Error as e:
+        print '[prerun] Could not initialize datastore'
+        print str(e)
+
+    except subprocess.CalledProcessError, e:
+        if 'OperationalError' in e.output:
+            print e.output
+            print '[prerun] Database not ready, waiting a bit before exit...'
+            time.sleep(5)
+            sys.exit(1)
+        else:
+            print e.output
+            raise e
+    finally:
+        cursor.close()
+        connection.close()
+
+
+def create_sysadmin():
+
+    name = os.environ.get('CKAN_SYSADMIN_NAME')
+    password = os.environ.get('CKAN_SYSADMIN_PASSWORD')
+    email = os.environ.get('CKAN_SYSADMIN_EMAIL')
+
+    if name and password and email:
+
+        # Check if user exists
+        command = ['paster', '--plugin=ckan', 'user', name, '-c', ckan_ini]
+
+        out = subprocess.check_output(command)
+        if 'User:None' not in re.sub(r'\s', '', out):
+            print '[prerun] Sysadmin user exists, skipping creation'
+            return
+
+        # Create user
+        command = ['paster', '--plugin=ckan', 'user', 'add',
+                   name,
+                   'password=' + password,
+                   'email=' + email,
+                   '-c', ckan_ini]
+
+        subprocess.call(command)
+        print '[prerun] Created user {0}'.format(name)
+
+        # Make it sysadmin
+        command = ['paster', '--plugin=ckan', 'sysadmin', 'add',
+                   name,
+                   '-c', ckan_ini]
+
+        subprocess.call(command)
+        print '[prerun] Made user {0} a sysadmin'.format(name)
+
+
+if __name__ == '__main__':
+
+    maintenance = os.environ.get('MAINTENANCE_MODE', '').lower() == 'true'
+
+    if maintenance:
+        print '[prerun] Maintenance mode, skipping setup...'
+    else:
+        check_main_db_connection()
+        init_db()
+        update_plugins()
+        check_datastore_db_connection()
+        init_datastore_db()
+        check_solr_connection()
+        create_sysadmin()

ckan-base/2.8/setup/start_ckan.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+
+# Run the prerun script to init CKAN and create the default admin user
+sudo -u ckan -EH python prerun.py
+
+# Run any startup scripts provided by images extending this one
+if [[ -d "/docker-entrypoint.d" ]]
+then
+    for f in /docker-entrypoint.d/*; do
+        case "$f" in
+            *.sh)     echo "$0: Running init file $f"; . "$f" ;;
+            *.py)     echo "$0: Running init file $f"; python "$f"; echo ;;
+            *)        echo "$0: Ignoring $f (not an sh or py file)" ;;
+        esac
+        echo
+    done
+fi
+
+# Set the common uwsgi options
+UWSGI_OPTS="--plugins http,python,gevent --socket /tmp/uwsgi.sock --uid 92 --gid 92 --http :5000 --master --enable-threads --paste config:/srv/app/production.ini --paste-logger --lazy-apps --gevent 2000 -p 2 -L"
+
+# Check whether http basic auth password protection is enabled and enable basicauth routing on uwsgi respecfully
+if [ $? -eq 0 ]
+then
+  if [ "$PASSWORD_PROTECT" = true ]
+  then
+    if [ "$HTPASSWD_USER" ] || [ "$HTPASSWD_PASSWORD" ]
+    then
+      # Generate htpasswd file for basicauth
+      htpasswd -d -b -c /srv/app/.htpasswd $HTPASSWD_USER $HTPASSWD_PASSWORD
+      # Start supervisord
+      supervisord --configuration /etc/supervisord.conf &
+      # Start uwsgi with basicauth
+      sudo -u ckan -EH uwsgi --ini /srv/app/uwsgi.conf --pcre-jit $UWSGI_OPTS
+    else
+      echo "Missing HTPASSWD_USER or HTPASSWD_PASSWORD environment variables. Exiting..."
+      exit 1
+    fi
+  else
+    # Start supervisord
+    supervisord --configuration /etc/supervisord.conf &
+    # Start uwsgi
+    sudo -u ckan -EH uwsgi $UWSGI_OPTS
+  fi
+else
+  echo "[prerun] failed...not starting CKAN."
+fi

ckan-base/2.8/setup/supervisor.worker.conf

@@ -0,0 +1,13 @@
+[program:ckan-worker]
+command=paster --plugin=ckan jobs worker -c /srv/app/production.ini
+priority=501
+autostart=true
+autorestart=true
+redirect_stderr=true
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stdout
+stderr_logfile_maxbytes=0
+user=ckan
+environment=HOME="/srv/app",USER="ckan"
+

ckan-base/2.8/setup/supervisord.conf

@@ -0,0 +1,23 @@
+[unix_http_server]
+file = /tmp/supervisor.sock
+chmod = 0777
+chown = nobody:nogroup
+
+[supervisord]
+logfile = /tmp/supervisord.log
+logfile_maxbytes = 50MB
+logfile_backups=10
+loglevel = info
+pidfile = /tmp/supervisord.pid
+nodaemon = true
+umask = 022
+identifier = supervisor
+
+[supervisorctl]
+serverurl = unix:///tmp/supervisor.sock
+
+[rpcinterface:supervisor]
+supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
+
+[include]
+files = /etc/supervisord.d/*.conf

ckan-base/2.8/setup/uwsgi.conf

@@ -0,0 +1,2 @@
+[uwsgi]
+route = ^(?!/api).*$ basicauth:Restricted,/srv/app/.htpasswd

ckan-base/2.9/Dockerfile

@@ -0,0 +1,109 @@
+FROM alpine:3.13
+
+# Internal environment variables
+ENV APP_DIR=/srv/app
+ENV SRC_DIR=/srv/app/src
+ENV CKAN_INI=${APP_DIR}/ckan.ini
+ENV PIP_SRC=${SRC_DIR}
+ENV CKAN_STORAGE_PATH=/var/lib/ckan
+ENV GIT_URL=https://github.com/ckan/ckan.git
+# CKAN version to build
+ENV GIT_BRANCH=ckan-2.9.4
+# Customize these on the .env file if needed
+ENV CKAN_SITE_URL=http://localhost:5000
+ENV CKAN__PLUGINS image_view text_view recline_view datastore datapusher envvars
+
+WORKDIR ${APP_DIR}
+
+# Install necessary packages to run CKAN
+RUN apk add --no-cache tzdata \
+        git \
+        gettext \
+        postgresql-client \
+        python3 \
+        apache2-utils \
+        libxml2 \
+        libxslt \
+        musl-dev \
+        uwsgi-http \
+        uwsgi-corerouter \
+        uwsgi-python3 \
+        py3-gevent \
+        uwsgi-gevent \
+        libmagic \
+        curl \
+        sudo && \
+    # Packages to build CKAN requirements and plugins
+    apk add --no-cache --virtual .build-deps \
+        postgresql-dev \
+        gcc \
+        make \
+        g++ \
+        autoconf \
+        automake \
+	    libtool \
+        python3-dev \
+        py3-virtualenv \
+        libxml2-dev \
+        libxslt-dev \
+        linux-headers
+
+# Create src directory (SRC_DIR)
+RUN mkdir -p ${SRC_DIR}
+
+# Install pip
+RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/get-pip.py && \
+    python3 ${SRC_DIR}/get-pip.py
+
+# Set up Python3 virtual environment
+RUN cd ${APP_DIR} && \
+    python3 -m venv ${APP_DIR} && \
+    source ${APP_DIR}/bin/activate
+
+# Virtual environment binaries/scripts to be used first
+ENV PATH=${APP_DIR}/bin:${PATH}   
+    
+# Install CKAN code
+RUN pip3 install -e git+${GIT_URL}@${GIT_BRANCH}#egg=ckan
+
+# Install uwsgi
+RUN pip3 install uwsgi
+
+# Complete CKAN install
+RUN cd ${SRC_DIR}/ckan && \
+    cp who.ini ${APP_DIR} && \
+    pip3 install -r requirement-setuptools.txt && \
+    pip3 install --no-binary :all: -r requirements.txt && \
+    # Install CKAN envvars to support loading config from environment variables
+    pip3 install -e git+https://github.com/okfn/ckanext-envvars.git#egg=ckanext-envvars
+    
+# Create and update CKAN config
+RUN ckan generate config ${CKAN_INI}
+
+# Install and configure supervisor
+RUN pip3 install supervisor && \
+mkdir /etc/supervisord.d
+
+# Copy all setup files
+COPY 2.9/setup ${APP_DIR}
+COPY 2.9/setup/supervisor.worker.conf /etc/supervisord.d/worker.conf
+COPY 2.9/setup/supervisord.conf /etc/supervisord.conf
+
+# Create a local user and group to run the app
+RUN addgroup -g 92 -S ckan && \
+    adduser -u 92 -h /srv/app -H -D -S -G ckan ckan
+
+# Create local storage folder
+RUN mkdir -p $CKAN_STORAGE_PATH && \
+    chown -R ckan:ckan $CKAN_STORAGE_PATH
+
+# Create entrypoint directory for children image scripts
+ONBUILD RUN mkdir /docker-entrypoint.d
+
+RUN chown ckan -R /srv/app
+
+EXPOSE 5000
+
+HEALTHCHECK --interval=10s --timeout=5s --retries=5 CMD curl --fail http://localhost:5000/api/3/action/status_show || exit 1
+
+CMD ["/srv/app/start_ckan.sh"] 

ckan-base/2.9/setup/ckan-uwsgi.ini

@@ -0,0 +1,15 @@
+[uwsgi]
+http-socket     =  :5000
+uid             =  ckan
+guid            =  ckan
+plugins         =  python3
+wsgi-file       =  /srv/app/wsgi.py
+virtualenv      =  /srv/app
+module          =  wsgi:application
+master          =  true
+processes       =  5
+pidfile         =  /tmp/%n.pid
+harakiri        =  50
+max-requests    =  5000
+vacuum          =  true
+callable        =  application

ckan-base/2.9/setup/prerun.py

@@ -0,0 +1,194 @@
+import os
+import sys
+import subprocess
+import psycopg2
+import urllib3
+import time
+import re
+
+ckan_ini = os.environ.get('CKAN_INI', '/srv/app/ckan.ini')
+
+RETRY = 5
+
+def update_plugins():
+
+    plugins = os.environ.get('CKAN__PLUGINS', '')
+    print('[prerun] Setting the following plugins in {}:'.format(ckan_ini))
+    print(plugins)
+    cmd = ['ckan', 'config-tool', ckan_ini,
+           'ckan.plugins = {}'.format(plugins)]
+    subprocess.check_output(cmd, stderr=subprocess.STDOUT)
+    print('[prerun] Plugins set.')
+
+    
+def check_main_db_connection(retry=None):
+
+    conn_str = os.environ.get('CKAN_SQLALCHEMY_URL')
+    if not conn_str:
+        print('[prerun] CKAN_SQLALCHEMY_URL not defined, not checking db')
+    return check_db_connection(conn_str, retry)
+
+
+def check_datastore_db_connection(retry=None):
+
+    conn_str = os.environ.get('CKAN_DATASTORE_WRITE_URL')
+    if not conn_str:
+        print('[prerun] CKAN_DATASTORE_WRITE_URL not defined, not checking db')
+    return check_db_connection(conn_str, retry)
+
+
+def check_db_connection(conn_str, retry=None):
+
+    if retry is None:
+        retry = RETRY
+    elif retry == 0:
+        print('[prerun] Giving up after 5 tries...')
+        sys.exit(1)
+
+    try:
+        connection = psycopg2.connect(conn_str)
+
+    except psycopg2.Error as e:
+        print(str(e))
+        print('[prerun] Unable to connect to the database, waiting...')
+        time.sleep(10)
+        check_db_connection(conn_str, retry=retry - 1)
+    else:
+        connection.close()
+
+        
+def check_solr_connection(retry=None):
+
+    if retry is None:
+        retry = RETRY
+    elif retry == 0:
+        print('[prerun] Giving up after 5 tries...')
+        sys.exit(1)
+
+    url = os.environ.get('CKAN_SOLR_URL', '')
+    search_url = '{url}/select/?q=*&wt=json'.format(url=url)
+    http = urllib3.PoolManager()
+    try:
+        r = http.request('GET', search_url)
+    except urllib3.exceptions.ConnectionError as e:
+        print(str(e))
+        print('[prerun] Unable to connect to solr, waiting...')
+        time.sleep(10)
+        check_solr_connection(retry=retry - 1)
+    else:
+        print('[prerun] Connection Status from SOLR is ', (r.status))
+
+def init_db():
+
+    db_command = ['ckan', '-c', ckan_ini, 
+                  'db', 'init']
+    print('[prerun] Initializing or upgrading db - start')
+    try:
+        subprocess.check_output(db_command, stderr=subprocess.STDOUT)
+        print('[prerun] Initializing or upgrading db - end')
+    except subprocess.CalledProcessError as e:
+        if 'OperationalError' in e.output:
+            print(e.output)
+            print('[prerun] Database not ready, waiting a bit before exit...')
+            time.sleep(5)
+            sys.exit(1)
+        else:
+            print(e.output)
+            raise e
+
+
+def init_datastore_db():
+
+    conn_str = os.environ.get('CKAN_DATASTORE_WRITE_URL')
+    if not conn_str:
+        print('[prerun] Skipping datastore initialization')
+        return
+
+    datastore_perms_command = ['ckan', '-c', ckan_ini, 
+                               'datastore', 'set-permissions']
+
+    connection = psycopg2.connect(conn_str)
+    cursor = connection.cursor()
+
+    print('[prerun] Initializing datastore db - start')
+    try:
+        datastore_perms = subprocess.Popen(
+            datastore_perms_command,
+            stdout=subprocess.PIPE)
+
+        perms_sql = datastore_perms.stdout.read().decode('utf-8')
+        # Remove internal pg command as psycopg2 does not like it
+        perms_sql = re.sub('\\\\connect \"(.*)\"', '', perms_sql)
+        cursor.execute(perms_sql)
+        for notice in connection.notices:
+            print(notice)
+
+        connection.commit()
+
+        print('[prerun] Initializing datastore db - end')
+        print(datastore_perms.stdout.read().decode('utf-8'))
+    except psycopg2.Error as e:
+        print('[prerun] Could not initialize datastore')
+        print(str(e))
+
+    except subprocess.CalledProcessError as e:
+        if 'OperationalError' in e.output:
+            print(e.output)
+            print('[prerun] Database not ready, waiting a bit before exit...')
+            time.sleep(5)
+            sys.exit(1)
+        else:
+            print(e.output)
+            raise e
+    finally:
+        cursor.close()
+        connection.close()
+
+
+def create_sysadmin():
+
+    name = os.environ.get('CKAN_SYSADMIN_NAME')
+    password = os.environ.get('CKAN_SYSADMIN_PASSWORD')
+    email = os.environ.get('CKAN_SYSADMIN_EMAIL')
+
+    if name and password and email:
+
+        # Check if user exists
+        command = ['ckan', '-c', ckan_ini, 'user', 'show', name,]
+
+        out = subprocess.check_output(command)
+        if 'User:None' not in re.sub(r'\s', '', out.decode()):
+            print('[prerun] Sysadmin user exists, skipping creation')
+            return
+
+        # Create user
+        command = ['ckan', '-c', ckan_ini, 'user', 'add',
+                   name,
+                   'password=' + password,
+                   'email=' + email]
+
+        subprocess.call(command)
+        print('[prerun] Created user {0}'.format(name))
+
+        # Make it sysadmin
+        command = ['ckan', '-c', ckan_ini, 'sysadmin', 'add',
+                   name]
+
+        subprocess.call(command)
+        print('[prerun] Made user {0} a sysadmin'.format(name))
+
+
+if __name__ == '__main__':
+
+    maintenance = os.environ.get('MAINTENANCE_MODE', '').lower() == 'true'
+
+    if maintenance:
+        print('[prerun] Maintenance mode, skipping setup...')
+    else:
+        check_main_db_connection()
+        init_db()
+        update_plugins()
+        check_datastore_db_connection()
+        init_datastore_db()
+        check_solr_connection()
+        create_sysadmin()

ckan-base/2.9/setup/start_ckan.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+# Run the prerun script to init CKAN and create the default admin user
+sudo -u ckan -EH python3 prerun.py
+
+# Run any startup scripts provided by images extending this one
+if [[ -d "/docker-entrypoint.d" ]]
+then
+    for f in /docker-entrypoint.d/*; do
+        case "$f" in
+            *.sh)     echo "$0: Running init file $f"; . "$f" ;;
+            *.py)     echo "$0: Running init file $f"; python "$f"; echo ;;
+            *)        echo "$0: Ignoring $f (not an sh or py file)" ;;
+        esac
+        echo
+    done
+fi
+
+
+# Check whether http basic auth password protection is enabled and enable basicauth routing on uwsgi respecfully
+if [ $? -eq 0 ]
+then
+  if [ "$PASSWORD_PROTECT" = true ]
+  then
+    if [ "$HTPASSWD_USER" ] || [ "$HTPASSWD_PASSWORD" ]
+    then
+      # Generate htpasswd file for basicauth
+      htpasswd -d -b -c /srv/app/.htpasswd $HTPASSWD_USER $HTPASSWD_PASSWORD
+      # Start supervisord
+      supervisord --configuration /etc/supervisord.conf &
+      # Start uwsgi with basicauth
+      sudo -u ckan -EH uwsgi -i ckan-uwsgi.ini
+    else
+      echo "Missing HTPASSWD_USER or HTPASSWD_PASSWORD environment variables. Exiting..."
+      exit 1
+    fi
+  else
+    # Start supervisord
+    supervisord --configuration /etc/supervisord.conf &
+    # Start uwsgi
+    sudo -u ckan -EH uwsgi -i ckan-uwsgi.ini
+  fi
+else
+  echo "[prerun] failed...not starting CKAN."
+fi
+

ckan-base/2.9/setup/supervisor.worker.conf

@@ -0,0 +1,12 @@
+[program:ckan-worker]
+command=ckan -c /srv/app/ckan.ini jobs worker
+priority=501
+autostart=true
+autorestart=true
+redirect_stderr=true
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stdout
+stderr_logfile_maxbytes=0
+user=ckan
+environment=HOME="/srv/app",USER="ckan"

ckan-base/2.9/setup/supervisord.conf

@@ -0,0 +1,23 @@
+[unix_http_server]
+file = /tmp/supervisor.sock
+chmod = 0777
+chown = nobody:nogroup
+
+[supervisord]
+logfile = /tmp/supervisord.log
+logfile_maxbytes = 50MB
+logfile_backups=10
+loglevel = info
+pidfile = /tmp/supervisord.pid
+nodaemon = true
+umask = 022
+identifier = supervisor
+
+[supervisorctl]
+serverurl = unix:///tmp/supervisor.sock
+
+[rpcinterface:supervisor]
+supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
+
+[include]
+files = /etc/supervisord.d/*.conf

ckan-base/2.9/setup/wsgi.py

@@ -0,0 +1,9 @@
+import os
+from ckan.config.middleware import make_app
+from ckan.cli import CKANConfigLoader
+from logging.config import fileConfig as loggingFileConfig
+config_filepath = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'ckan.ini')
+abspath = os.path.join(os.path.dirname(os.path.abspath(__file__)))
+loggingFileConfig(config_filepath)
+config = CKANConfigLoader(config_filepath).get_config()
+application = make_app(config)

ckan-dev/2.7/Dockerfile

@@ -0,0 +1,20 @@
+FROM ckan/ckan-base:testing-only.2.7
+
+LABEL maintainer="brett@kowh.ai"
+
+ENV APP_DIR=/srv/app
+ENV SRC_EXTENSIONS_DIR=/srv/app/src_extensions
+
+# Install packages needed by the dev requirements
+RUN apk add --no-cache libffi-dev
+
+# Install CKAN dev requirements
+RUN pip install --no-binary :all: -r https://raw.githubusercontent.com/ckan/ckan/${GIT_BRANCH}/dev-requirements.txt
+
+# Create folder for local extensions sources
+RUN mkdir $SRC_EXTENSIONS_DIR
+
+COPY setup/start_ckan_development.sh ${APP_DIR}
+
+
+CMD ["/srv/app/start_ckan_development.sh"]

ckan-dev/2.8/Dockerfile

@@ -0,0 +1,20 @@
+FROM ckan/ckan-base:testing-only.2.8
+
+LABEL maintainer="brett@kowh.ai"
+
+ENV APP_DIR=/srv/app
+ENV SRC_EXTENSIONS_DIR=/srv/app/src_extensions
+
+# Install packages needed by the dev requirements
+RUN apk add --no-cache libffi-dev
+
+# Install CKAN dev requirements
+RUN pip install --no-binary :all: -r https://raw.githubusercontent.com/ckan/ckan/${GIT_BRANCH}/dev-requirements.txt
+
+# Create folder for local extensions sources
+RUN mkdir $SRC_EXTENSIONS_DIR
+
+COPY setup/start_ckan_development.sh ${APP_DIR}
+
+
+CMD ["/srv/app/start_ckan_development.sh"]

ckan-dev/2.9/Dockerfile

@@ -0,0 +1,29 @@
+FROM ckan/ckan-base:testing-only.2.9
+
+LABEL maintainer="brett@kowh.ai"
+
+ENV APP_DIR=/srv/app
+ENV SRC_EXTENSIONS_DIR=/srv/app/src_extensions
+
+# Install packages needed by the dev requirements
+RUN apk add --no-cache libffi-dev
+
+# Set up Python3 virtual environment
+RUN cd ${APP_DIR} && \
+    source ${APP_DIR}/bin/activate
+
+# Virtual environment binaries/scripts to be used first
+ENV PATH=${APP_DIR}/bin:${PATH} 
+
+# Install CKAN dev requirements
+# Will need to change this eventually - when CKAN 2.9 is out
+# wget https://raw.githubusercontent.com/ckan/ckan/master/dev-requirements.txt
+# RUN pip3 install --no-binary :all: -r https://raw.githubusercontent.com/ckan/ckan/master/dev-requirements.txt
+RUN pip3 install -r https://raw.githubusercontent.com/ckan/ckan/master/dev-requirements.txt
+
+# Create folder for local extensions sources
+RUN mkdir $SRC_EXTENSIONS_DIR
+
+COPY setup/start_ckan_development.sh ${APP_DIR}
+
+CMD ["/srv/app/start_ckan_development.sh"]

ckan-dev/setup/start_ckan_development.sh

@@ -0,0 +1,82 @@
+#!/bin/bash
+
+# Install any local extensions in the src_extensions volume
+echo "Looking for local extensions to install..."
+echo "Extension dir contents:"
+ls -la $SRC_EXTENSIONS_DIR
+for i in $SRC_EXTENSIONS_DIR/*
+do
+    if [ -d $i ];
+    then
+
+        if [ -f $i/pip-requirements.txt ];
+        then
+            pip install -r $i/pip-requirements.txt
+            echo "Found requirements file in $i"
+        fi
+        if [ -f $i/requirements.txt ];
+        then
+            pip install -r $i/requirements.txt
+            echo "Found requirements file in $i"
+        fi
+        if [ -f $i/dev-requirements.txt ];
+        then
+            pip install -r $i/dev-requirements.txt
+            echo "Found dev-requirements file in $i"
+        fi
+        if [ -f $i/setup.py ];
+        then
+            cd $i
+            python $i/setup.py develop
+            echo "Found setup.py file in $i"
+            cd $APP_DIR
+        fi
+
+        # Point `use` in test.ini to location of `test-core.ini`
+        if [ -f $i/test.ini ];
+        then
+            echo "Updating \`test.ini\` reference to \`test-core.ini\` for plugin $i"
+            paster --plugin=ckan config-tool $i/test.ini "use = config:../../src/ckan/test-core.ini"
+        fi
+    fi
+done
+
+# Set debug to true
+echo "Enabling debug mode"
+ckan config-tool $CKAN_INI -s DEFAULT "debug = true"
+
+# Update the plugins setting in the ini file with the values defined in the env var
+echo "Loading the following plugins: $CKAN__PLUGINS"
+ckan config-tool $CKAN_INI "ckan.plugins = $CKAN__PLUGINS"
+
+# Update test-core.ini DB, SOLR & Redis settings
+echo "Loading test settings into test-core.ini"
+ckan config-tool $SRC_DIR/ckan/test-core.ini \
+    "sqlalchemy.url = $TEST_CKAN_SQLALCHEMY_URL" \
+    "ckan.datstore.write_url = $TEST_CKAN_DATASTORE_WRITE_URL" \
+    "ckan.datstore.read_url = $TEST_CKAN_DATASTORE_READ_URL" \
+    "solr_url = $TEST_CKAN_SOLR_URL" \
+    "ckan.redis_url = $TEST_CKAN_REDIS_URL"
+
+# Run the prerun script to init CKAN and create the default admin user
+sudo -u ckan -EH python prerun.py
+
+# Run any startup scripts provided by images extending this one
+if [[ -d "/docker-entrypoint.d" ]]
+then
+    for f in /docker-entrypoint.d/*; do
+        case "$f" in
+            *.sh)     echo "$0: Running init file $f"; . "$f" ;;
+            *.py)     echo "$0: Running init file $f"; python "$f"; echo ;;
+            *)        echo "$0: Ignoring $f (not an sh or py file)" ;;
+        esac
+        echo
+    done
+fi
+
+# Start supervisord
+supervisord --configuration /etc/supervisord.conf &
+
+# Start the development server with automatic reload
+# Check the --reloader options sudo -u ckan -EH ckan -c $CKAN_INI run --reloader <TEXT>
+sudo -u ckan -EH ckan -c $CKAN_INI run