diff --git a/.env b/.env index a3d9300..211d393 100644 --- a/.env +++ b/.env @@ -1,4 +1,10 @@ -# CKAN databases +# CKAN +CKAN_VERSION=2.9.3 +CKAN_SITE_URL=http://localhost:5000 +CKAN_SITE_ID=default +CKAN_PORT=5000 + +# Database POSTGRES_USER=ckan POSTGRES_PASSWORD=ckan DATASTORE_READONLY_USER=datastore_ro @@ -8,39 +14,21 @@ CKAN_SQLALCHEMY_URL=postgresql://ckan:ckan@db/ckan CKAN_DATASTORE_WRITE_URL=postgresql://ckan:ckan@db/datastore CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:datastore@db/datastore -# Test database connections -TEST_CKAN_SQLALCHEMY_URL=postgres://ckan:ckan@db/ckan_test -TEST_CKAN_DATASTORE_WRITE_URL=postgresql://ckan:ckan@db/datastore_test -TEST_CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:datastore@db/datastore_test - -# CKAN core -CKAN_SITE_ID=default -CKAN_SITE_URL=http://ckan:5000 -CKAN_PORT=5000 -CKAN_SYSADMIN_NAME=ckan_admin -CKAN_SYSADMIN_PASSWORD=test1234 -CKAN_SYSADMIN_EMAIL=your_email@example.com -CKAN_STORAGE_PATH=/var/lib/ckan -CKAN_SMTP_SERVER=smtp.corporateict.domain:25 -CKAN_SMTP_STARTTLS=True -CKAN_SMTP_USER=user -CKAN_SMTP_PASSWORD=pass -CKAN_SMTP_MAIL_FROM=ckan@localhost -TZ=UTC - -# Other services +# solr +SOLR_VERSION=6.6.6 CKAN_SOLR_URL=http://solr:8983/solr/ckan -CKAN_REDIS_URL=redis://redis:6379/1 -CKAN_DATAPUSHER_URL=http://datapusher:8801 + +# Datapusher +DATAPUSHER_VERSION=0.0.17 +CKAN_DATAPUSHER_URL=http://datapusher:8800 CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://ckan:5000 +DATAPUSHER_REWRITE_RESOURCES=True +DATAPUSHER_REWRITE_URL=http://ckan:5000 -# test connections -TEST_CKAN_SOLR_URL=http://solr:8983/solr/ckan -TEST_CKAN_REDIS_URL=redis://redis:6379/1 +# Redis +REDIS_VERSION=6.0.7 +CKAN_REDIS_URL=redis://redis:6379/1 -# Extensions -CKAN__PLUGINS=envvars image_view text_view recline_view datastore datapusher -CKAN__HARVEST__MQ__TYPE=redis -CKAN__HARVEST__MQ__HOSTNAME=redis -CKAN__HARVEST__MQ__PORT=6379 -CKAN__HARVEST__MQ__REDIS_DB=1 \ No newline at end of file +CKAN_SYSADMIN_NAME=admin +CKAN_SYSADMIN_PASSWORD=test1234 +CKAN_SYSADMIN_EMAIL=admin@ckan.org diff --git a/.gitkeep b/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..a29437d --- /dev/null +++ b/Dockerfile @@ -0,0 +1,35 @@ +FROM solr:6.6.6 + +# Enviroment variables +ENV SOLR_CORE ckan + +# Build Arguments +ARG CKAN_VERSION +ARG SOLR_VERSION + +# root user for initial config +USER root + +# Create directories +RUN mkdir -p /opt/solr/server/solr/${SOLR_CORE}/conf && \ + mkdir -p /opt/solr/server/solr/${SOLR_CORE}/data && \ + mkdir -p /opt/solr/server/solr/${SOLR_CORE}/data/index + +# Add files +COPY ./solr/solrconfig-${CKAN_VERSION}.xml /opt/solr/server/solr/${SOLR_CORE}/conf/solrconfig.xml +ADD https://raw.githubusercontent.com/ckan/ckan/ckan-${CKAN_VERSION}/ckan/config/solr/schema.xml \ +https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/${SOLR_VERSION}/solr/server/solr/configsets/basic_configs/conf/currency.xml \ +https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/${SOLR_VERSION}/solr/server/solr/configsets/basic_configs/conf/synonyms.txt \ +https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/${SOLR_VERSION}/solr/server/solr/configsets/basic_configs/conf/stopwords.txt \ +https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/${SOLR_VERSION}/solr/server/solr/configsets/basic_configs/conf/protwords.txt \ +https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/${SOLR_VERSION}/solr/server/solr/configsets/data_driven_schema_configs/conf/elevate.xml \ +/opt/solr/server/solr/${SOLR_CORE}/conf/ + +# Create core.properties +RUN echo name=${SOLR_CORE} > /opt/solr/server/solr/${SOLR_CORE}/core.properties + +# Giving ownership to Solr +RUN chown -R ${SOLR_USER}:${SOLR_USER} /opt/solr/server/solr/${SOLR_CORE} + +# non-root user for runtime +USER ${SOLR_USER}:${SOLR_USER} \ No newline at end of file diff --git a/Dockerfile.dev b/Dockerfile.dev new file mode 100644 index 0000000..6086f6e --- /dev/null +++ b/Dockerfile.dev @@ -0,0 +1,37 @@ +FROM ckan/ckan-dev:testing-only.2.9 + +LABEL maintainer="brett@kowh.ai" + +# Set up environment variables +ENV APP_DIR=/srv/app +ENV TZ=UTC +RUN echo ${TZ} > /etc/timezone + +# Make sure both files are not exactly the same +RUN if ! [ /usr/share/zoneinfo/${TZ} -ef /etc/localtime ]; then \ + cp /usr/share/zoneinfo/${TZ} /etc/localtime ;\ + fi ; + +# Install any extensions needed by your CKAN instance +# (Make sure to add the plugins to CKAN__PLUGINS in the .env file) +# For instance: +#RUN pip install -e git+https://github.com/ckan/ckanext-pages.git#egg=ckanext-pages && \ +# pip install -e git+https://github.com/ckan/ckanext-dcat.git@v0.0.6#egg=ckanext-dcat && \ +# pip install -r https://raw.githubusercontent.com/ckan/ckanext-dcat/v0.0.6/requirements.txt + +# Clone the extension(s) your are writing for your own project in the `src` folder +# to get them mounted in this image at runtime + +# Apply any patches needed to CKAN core or any of the built extensions (not the +# runtime mounted ones) +# See https://github.com/okfn/docker-ckan#applying-patches + +COPY patches ${APP_DIR}/patches + +RUN for d in ${APP_DIR}/patches/*; do \ + if [ -d $d ]; then \ + for f in `ls $d/*.patch | sort -g`; do \ + cd $SRC_DIR/`basename "$d"` && echo "$0: Applying patch $f to $SRC_DIR/`basename $d`"; patch -p1 < "$f" ; \ + done ; \ + fi ; \ + done diff --git a/README b/README new file mode 100755 index 0000000..9c85779 --- /dev/null +++ b/README @@ -0,0 +1,12 @@ +Build ckan-base image (2.9) +cd /path/to/ckan-docker/ckan/ckan-base +docker build -t ckan/ckan-base:testing-only.2.9 -f 2.9/Dockerfile . + +Login to Dockerhub +docker login --username=kowhai +(will then ask for a password) + +Push image to (ckan) Dockerhub +docker push ckan/ckan-base:testing-only.2.9 + +Do the same with 2.8 and 2.7 diff --git a/ckan-base/2.7/Dockerfile b/ckan-base/2.7/Dockerfile new file mode 100755 index 0000000..4eb4332 --- /dev/null +++ b/ckan-base/2.7/Dockerfile @@ -0,0 +1,101 @@ +FROM alpine:3.13 + +# Internals, you probably don't need to change these +ENV APP_DIR=/srv/app +ENV SRC_DIR=/srv/app/src +ENV CKAN_INI=${APP_DIR}/production.ini +ENV PIP_SRC=${SRC_DIR} +ENV CKAN_STORAGE_PATH=/var/lib/ckan +ENV GIT_URL=https://github.com/ckan/ckan.git +# CKAN version to build +ENV GIT_BRANCH=ckan-2.7.11 +# Customize these on the .env file if needed +ENV CKAN_SITE_URL=http://localhost:5000 +ENV CKAN__PLUGINS image_view text_view recline_view datastore datapusher envvars + +WORKDIR ${APP_DIR} + +# Install necessary packages to run CKAN +RUN apk add --no-cache tzdata \ + git \ + gettext \ + postgresql-client \ + python \ + apache2-utils \ + libxml2 \ + libxslt \ + musl-dev \ + uwsgi \ + uwsgi-http \ + uwsgi-corerouter \ + uwsgi-python \ + py2-gevent \ + uwsgi-gevent \ + libmagic \ + curl \ + sudo && \ + # Packages to build CKAN requirements and plugins + apk add --no-cache --virtual .build-deps \ + postgresql-dev \ + gcc \ + make \ + g++ \ + autoconf \ + automake \ + libtool \ + python-dev \ + libxml2-dev \ + libxslt-dev \ + linux-headers + +# Create SRC_DIR +RUN mkdir -p ${SRC_DIR} + +# Install pip +RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/pip/2.7/get-pip.py && \ + python ${SRC_DIR}/get-pip.py 'pip==20.3.3' + +# Install pip, supervisord and uwsgi +RUN pip install supervisor && \ + mkdir /etc/supervisord.d && \ + #pip wheel --wheel-dir=/wheels uwsgi gevent && \ + rm -rf ${SRC_DIR}/get-pip.py + +COPY 2.7/setup/supervisord.conf /etc + +# Install CKAN +RUN pip install -e git+${GIT_URL}@${GIT_BRANCH}#egg=ckan && \ + cd ${SRC_DIR}/ckan && \ + cp who.ini ${APP_DIR} && \ + # Workaround to solve https://github.com/psycopg/psycopg2/issues/594 in Alpine 3.7 + sed -i -e "s/psycopg2==2.4.5/psycopg2==2.7.3.2/g" requirements.txt && \ + pip install --no-binary :all: -r requirements.txt && \ + # Install CKAN envvars to support loading config from environment variables + pip install -e git+https://github.com/okfn/ckanext-envvars.git#egg=ckanext-envvars && \ + # Create and update CKAN config + paster --plugin=ckan make-config ckan ${CKAN_INI} && \ + paster --plugin=ckan config-tool ${CKAN_INI} "ckan.plugins = ${CKAN__PLUGINS}" && \ + paster --plugin=ckan config-tool ${CKAN_INI} "ckan.site_url = ${CKAN__SITE_URL}" + +# Create a local user and group to run the app +RUN addgroup -g 92 -S ckan && \ + adduser -u 92 -h /srv/app -H -D -S -G ckan ckan + +# Create local storage folder +RUN mkdir -p $CKAN_STORAGE_PATH && \ + chown -R ckan:ckan $CKAN_STORAGE_PATH + +COPY 2.7/setup ${APP_DIR} +COPY 2.7/setup/supervisor.worker.conf /etc/supervisord.d/worker.conf +COPY 2.7/setup/uwsgi.conf /srv/app/uwsgi.conf + +# Create entrypoint directory for children image scripts +ONBUILD RUN mkdir /docker-entrypoint.d + +RUN chown ckan -R /srv/app + +EXPOSE 5000 + +HEALTHCHECK --interval=10s --timeout=5s --retries=5 CMD curl --fail http://localhost:5000/api/3/action/status_show || exit 1 + +CMD ["/srv/app/start_ckan.sh"] diff --git a/ckan-base/2.7/setup/prerun.py b/ckan-base/2.7/setup/prerun.py new file mode 100755 index 0000000..7ebacfa --- /dev/null +++ b/ckan-base/2.7/setup/prerun.py @@ -0,0 +1,197 @@ +import os +import sys +import subprocess +import psycopg2 +import urllib2 +import time +import re + +ckan_ini = os.environ.get('CKAN_INI', '/srv/app/production.ini') + +RETRY = 5 + +def update_plugins(): + + plugins = os.environ.get('CKAN__PLUGINS', '') + print('[prerun] Setting the following plugins in {}:'.format(ckan_ini)) + print(plugins) + cmd = ['paster', '--plugin=ckan', 'config-tool', + ckan_ini, 'ckan.plugins = {}'.format(plugins)] + subprocess.check_output(cmd, stderr=subprocess.STDOUT) + print '[prerun] Plugins set.' + + +def check_main_db_connection(retry=None): + + conn_str = os.environ.get('CKAN_SQLALCHEMY_URL') + if not conn_str: + print '[prerun] CKAN_SQLALCHEMY_URL not defined, not checking db' + return check_db_connection(conn_str, retry) + + +def check_datastore_db_connection(retry=None): + + conn_str = os.environ.get('CKAN_DATASTORE_WRITE_URL') + if not conn_str: + print '[prerun] CKAN_DATASTORE_WRITE_URL not defined, not checking db' + return check_db_connection(conn_str, retry) + + +def check_db_connection(conn_str, retry=None): + + if retry is None: + retry = RETRY + elif retry == 0: + print '[prerun] Giving up after 5 tries...' + sys.exit(1) + + try: + connection = psycopg2.connect(conn_str) + + except psycopg2.Error as e: + print str(e) + print '[prerun] Unable to connect to the database, waiting...' + time.sleep(10) + check_db_connection(conn_str, retry=retry - 1) + else: + connection.close() + + +def check_solr_connection(retry=None): + + if retry is None: + retry = RETRY + elif retry == 0: + print '[prerun] Giving up after 5 tries...' + sys.exit(1) + + url = os.environ.get('CKAN_SOLR_URL', '') + search_url = '{url}/select/?q=*&wt=json'.format(url=url) + + try: + connection = urllib2.urlopen(search_url) + except urllib2.URLError as e: + print str(e) + print '[prerun] Unable to connect to solr, waiting...' + time.sleep(10) + check_solr_connection(retry=retry - 1) + else: + eval(connection.read()) + + +def init_db(): + + db_command = ['paster', '--plugin=ckan', 'db', + 'init', '-c', ckan_ini] + print '[prerun] Initializing or upgrading db - start' + try: + subprocess.check_output(db_command, stderr=subprocess.STDOUT) + print '[prerun] Initializing or upgrading db - end' + except subprocess.CalledProcessError, e: + if 'OperationalError' in e.output: + print e.output + print '[prerun] Database not ready, waiting a bit before exit...' + time.sleep(5) + sys.exit(1) + else: + print e.output + raise e + + +def init_datastore_db(): + + conn_str = os.environ.get('CKAN_DATASTORE_WRITE_URL') + if not conn_str: + print '[prerun] Skipping datastore initialization' + return + + datastore_perms_command = ['paster', '--plugin=ckan', 'datastore', + 'set-permissions', '-c', ckan_ini] + + connection = psycopg2.connect(conn_str) + cursor = connection.cursor() + + print '[prerun] Initializing datastore db - start' + try: + datastore_perms = subprocess.Popen( + datastore_perms_command, + stdout=subprocess.PIPE) + + perms_sql = datastore_perms.stdout.read() + # Remove internal pg command as psycopg2 does not like it + perms_sql = re.sub('\\\\connect \"(.*)\"', '', perms_sql) + cursor.execute(perms_sql) + for notice in connection.notices: + print notice + + connection.commit() + + print '[prerun] Initializing datastore db - end' + print datastore_perms.stdout.read() + except psycopg2.Error as e: + print '[prerun] Could not initialize datastore' + print str(e) + + except subprocess.CalledProcessError, e: + if 'OperationalError' in e.output: + print e.output + print '[prerun] Database not ready, waiting a bit before exit...' + time.sleep(5) + sys.exit(1) + else: + print e.output + raise e + finally: + cursor.close() + connection.close() + + +def create_sysadmin(): + + name = os.environ.get('CKAN_SYSADMIN_NAME') + password = os.environ.get('CKAN_SYSADMIN_PASSWORD') + email = os.environ.get('CKAN_SYSADMIN_EMAIL') + + if name and password and email: + + # Check if user exists + command = ['paster', '--plugin=ckan', 'user', name, '-c', ckan_ini] + + out = subprocess.check_output(command) + if 'User:None' not in re.sub(r'\s', '', out): + print '[prerun] Sysadmin user exists, skipping creation' + return + + # Create user + command = ['paster', '--plugin=ckan', 'user', 'add', + name, + 'password=' + password, + 'email=' + email, + '-c', ckan_ini] + + subprocess.call(command) + print '[prerun] Created user {0}'.format(name) + + # Make it sysadmin + command = ['paster', '--plugin=ckan', 'sysadmin', 'add', + name, + '-c', ckan_ini] + + subprocess.call(command) + print '[prerun] Made user {0} a sysadmin'.format(name) + + +if __name__ == '__main__': + + maintenance = os.environ.get('MAINTENANCE_MODE', '').lower() == 'true' + + if maintenance: + print '[prerun] Maintenance mode, skipping setup...' + else: + check_main_db_connection() + init_db() + update_plugins() + check_datastore_db_connection() + init_datastore_db() + check_solr_connection() + create_sysadmin() \ No newline at end of file diff --git a/ckan-base/2.7/setup/start_ckan.sh b/ckan-base/2.7/setup/start_ckan.sh new file mode 100755 index 0000000..d1c6726 --- /dev/null +++ b/ckan-base/2.7/setup/start_ckan.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +# Run the prerun script to init CKAN and create the default admin user +sudo -u ckan -EH python prerun.py + +# Run any startup scripts provided by images extending this one +if [[ -d "/docker-entrypoint.d" ]] +then + for f in /docker-entrypoint.d/*; do + case "$f" in + *.sh) echo "$0: Running init file $f"; . "$f" ;; + *.py) echo "$0: Running init file $f"; python "$f"; echo ;; + *) echo "$0: Ignoring $f (not an sh or py file)" ;; + esac + echo + done +fi + +# Set the common uwsgi options +UWSGI_OPTS="--plugins http,python,gevent --socket /tmp/uwsgi.sock --uid 92 --gid 92 --http :5000 --master --enable-threads --paste config:/srv/app/production.ini --paste-logger --lazy-apps --gevent 2000 -p 2 -L" + +# Check whether http basic auth password protection is enabled and enable basicauth routing on uwsgi respecfully +if [ $? -eq 0 ] +then + if [ "$PASSWORD_PROTECT" = true ] + then + if [ "$HTPASSWD_USER" ] || [ "$HTPASSWD_PASSWORD" ] + then + # Generate htpasswd file for basicauth + htpasswd -d -b -c /srv/app/.htpasswd $HTPASSWD_USER $HTPASSWD_PASSWORD + # Start supervisord + supervisord --configuration /etc/supervisord.conf & + # Start uwsgi with basicauth + sudo -u ckan -EH uwsgi --ini /srv/app/uwsgi.conf --pcre-jit $UWSGI_OPTS + else + echo "Missing HTPASSWD_USER or HTPASSWD_PASSWORD environment variables. Exiting..." + exit 1 + fi + else + # Start supervisord + supervisord --configuration /etc/supervisord.conf & + # Start uwsgi + sudo -u ckan -EH uwsgi $UWSGI_OPTS + fi +else + echo "[prerun] failed...not starting CKAN." +fi diff --git a/ckan-base/2.7/setup/supervisor.worker.conf b/ckan-base/2.7/setup/supervisor.worker.conf new file mode 100755 index 0000000..0ca43ba --- /dev/null +++ b/ckan-base/2.7/setup/supervisor.worker.conf @@ -0,0 +1,13 @@ +[program:ckan-worker] +command=paster --plugin=ckan jobs worker -c /srv/app/production.ini +priority=501 +autostart=true +autorestart=true +redirect_stderr=true +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stdout +stderr_logfile_maxbytes=0 +user=ckan +environment=HOME="/srv/app",USER="ckan" + diff --git a/ckan-base/2.7/setup/supervisord.conf b/ckan-base/2.7/setup/supervisord.conf new file mode 100755 index 0000000..a3f6671 --- /dev/null +++ b/ckan-base/2.7/setup/supervisord.conf @@ -0,0 +1,23 @@ +[unix_http_server] +file = /tmp/supervisor.sock +chmod = 0777 +chown = nobody:nogroup + +[supervisord] +logfile = /tmp/supervisord.log +logfile_maxbytes = 50MB +logfile_backups=10 +loglevel = info +pidfile = /tmp/supervisord.pid +nodaemon = true +umask = 022 +identifier = supervisor + +[supervisorctl] +serverurl = unix:///tmp/supervisor.sock + +[rpcinterface:supervisor] +supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface + +[include] +files = /etc/supervisord.d/*.conf diff --git a/ckan-base/2.7/setup/uwsgi.conf b/ckan-base/2.7/setup/uwsgi.conf new file mode 100755 index 0000000..6321d6d --- /dev/null +++ b/ckan-base/2.7/setup/uwsgi.conf @@ -0,0 +1,2 @@ +[uwsgi] +route = ^(?!/api).*$ basicauth:Restricted,/srv/app/.htpasswd diff --git a/ckan-base/2.8/Dockerfile b/ckan-base/2.8/Dockerfile new file mode 100755 index 0000000..59904a3 --- /dev/null +++ b/ckan-base/2.8/Dockerfile @@ -0,0 +1,99 @@ +FROM alpine:3.13 + +# Internals, you probably don't need to change these +ENV APP_DIR=/srv/app +ENV SRC_DIR=/srv/app/src +ENV CKAN_INI=${APP_DIR}/production.ini +ENV PIP_SRC=${SRC_DIR} +ENV CKAN_STORAGE_PATH=/var/lib/ckan +ENV GIT_URL=https://github.com/ckan/ckan.git +# CKAN version to build +ENV GIT_BRANCH=ckan-2.8.8 +# Customize these on the .env file if needed +ENV CKAN_SITE_URL=http://localhost:5000 +ENV CKAN__PLUGINS image_view text_view recline_view datastore datapusher envvars + +WORKDIR ${APP_DIR} + +# Install necessary packages to run CKAN +RUN apk add --no-cache tzdata \ + git \ + gettext \ + postgresql-client \ + python \ + apache2-utils \ + libxml2 \ + libxslt \ + musl-dev \ + uwsgi \ + uwsgi-http \ + uwsgi-corerouter \ + uwsgi-python \ + py2-gevent \ + uwsgi-gevent \ + libmagic \ + curl \ + sudo && \ + # Packages to build CKAN requirements and plugins + apk add --no-cache --virtual .build-deps \ + postgresql-dev \ + gcc \ + make \ + g++ \ + autoconf \ + automake \ + libtool \ + python-dev \ + libxml2-dev \ + libxslt-dev \ + linux-headers + +# Create SRC_DIR +RUN mkdir -p ${SRC_DIR} + +# Install pip +RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/pip/2.7/get-pip.py && \ + python ${SRC_DIR}/get-pip.py 'pip==20.3.3' + +# Install supervisord and uwsgi +RUN pip install supervisor && \ + mkdir /etc/supervisord.d && \ + #pip wheel --wheel-dir=/wheels uwsgi gevent && \ + rm -rf ${SRC_DIR}/get-pip.py + +COPY 2.8/setup/supervisord.conf /etc + +# Install CKAN +RUN pip install -e git+${GIT_URL}@${GIT_BRANCH}#egg=ckan && \ + cd ${SRC_DIR}/ckan && \ + cp who.ini ${APP_DIR} && \ + pip install --no-binary :all: -r requirements.txt && \ + # Install CKAN envvars to support loading config from environment variables + pip install -e git+https://github.com/okfn/ckanext-envvars.git#egg=ckanext-envvars && \ + # Create and update CKAN config + paster --plugin=ckan make-config ckan ${CKAN_INI} && \ + paster --plugin=ckan config-tool ${CKAN_INI} "ckan.plugins = ${CKAN__PLUGINS}" && \ + paster --plugin=ckan config-tool ${CKAN_INI} "ckan.site_url = ${CKAN__SITE_URL}" + +# Create a local user and group to run the app +RUN addgroup -g 92 -S ckan && \ + adduser -u 92 -h /srv/app -H -D -S -G ckan ckan + +# Create local storage folder +RUN mkdir -p $CKAN_STORAGE_PATH && \ + chown -R ckan:ckan $CKAN_STORAGE_PATH + +COPY 2.8/setup ${APP_DIR} +COPY 2.8/setup/supervisor.worker.conf /etc/supervisord.d/worker.conf +COPY 2.8/setup/uwsgi.conf /srv/app/uwsgi.conf + +# Create entrypoint directory for children image scripts +ONBUILD RUN mkdir /docker-entrypoint.d + +RUN chown ckan -R /srv/app + +EXPOSE 5000 + +HEALTHCHECK --interval=10s --timeout=5s --retries=5 CMD curl --fail http://localhost:5000/api/3/action/status_show || exit 1 + +CMD ["/srv/app/start_ckan.sh"] diff --git a/ckan-base/2.8/setup/prerun.py b/ckan-base/2.8/setup/prerun.py new file mode 100755 index 0000000..7ebacfa --- /dev/null +++ b/ckan-base/2.8/setup/prerun.py @@ -0,0 +1,197 @@ +import os +import sys +import subprocess +import psycopg2 +import urllib2 +import time +import re + +ckan_ini = os.environ.get('CKAN_INI', '/srv/app/production.ini') + +RETRY = 5 + +def update_plugins(): + + plugins = os.environ.get('CKAN__PLUGINS', '') + print('[prerun] Setting the following plugins in {}:'.format(ckan_ini)) + print(plugins) + cmd = ['paster', '--plugin=ckan', 'config-tool', + ckan_ini, 'ckan.plugins = {}'.format(plugins)] + subprocess.check_output(cmd, stderr=subprocess.STDOUT) + print '[prerun] Plugins set.' + + +def check_main_db_connection(retry=None): + + conn_str = os.environ.get('CKAN_SQLALCHEMY_URL') + if not conn_str: + print '[prerun] CKAN_SQLALCHEMY_URL not defined, not checking db' + return check_db_connection(conn_str, retry) + + +def check_datastore_db_connection(retry=None): + + conn_str = os.environ.get('CKAN_DATASTORE_WRITE_URL') + if not conn_str: + print '[prerun] CKAN_DATASTORE_WRITE_URL not defined, not checking db' + return check_db_connection(conn_str, retry) + + +def check_db_connection(conn_str, retry=None): + + if retry is None: + retry = RETRY + elif retry == 0: + print '[prerun] Giving up after 5 tries...' + sys.exit(1) + + try: + connection = psycopg2.connect(conn_str) + + except psycopg2.Error as e: + print str(e) + print '[prerun] Unable to connect to the database, waiting...' + time.sleep(10) + check_db_connection(conn_str, retry=retry - 1) + else: + connection.close() + + +def check_solr_connection(retry=None): + + if retry is None: + retry = RETRY + elif retry == 0: + print '[prerun] Giving up after 5 tries...' + sys.exit(1) + + url = os.environ.get('CKAN_SOLR_URL', '') + search_url = '{url}/select/?q=*&wt=json'.format(url=url) + + try: + connection = urllib2.urlopen(search_url) + except urllib2.URLError as e: + print str(e) + print '[prerun] Unable to connect to solr, waiting...' + time.sleep(10) + check_solr_connection(retry=retry - 1) + else: + eval(connection.read()) + + +def init_db(): + + db_command = ['paster', '--plugin=ckan', 'db', + 'init', '-c', ckan_ini] + print '[prerun] Initializing or upgrading db - start' + try: + subprocess.check_output(db_command, stderr=subprocess.STDOUT) + print '[prerun] Initializing or upgrading db - end' + except subprocess.CalledProcessError, e: + if 'OperationalError' in e.output: + print e.output + print '[prerun] Database not ready, waiting a bit before exit...' + time.sleep(5) + sys.exit(1) + else: + print e.output + raise e + + +def init_datastore_db(): + + conn_str = os.environ.get('CKAN_DATASTORE_WRITE_URL') + if not conn_str: + print '[prerun] Skipping datastore initialization' + return + + datastore_perms_command = ['paster', '--plugin=ckan', 'datastore', + 'set-permissions', '-c', ckan_ini] + + connection = psycopg2.connect(conn_str) + cursor = connection.cursor() + + print '[prerun] Initializing datastore db - start' + try: + datastore_perms = subprocess.Popen( + datastore_perms_command, + stdout=subprocess.PIPE) + + perms_sql = datastore_perms.stdout.read() + # Remove internal pg command as psycopg2 does not like it + perms_sql = re.sub('\\\\connect \"(.*)\"', '', perms_sql) + cursor.execute(perms_sql) + for notice in connection.notices: + print notice + + connection.commit() + + print '[prerun] Initializing datastore db - end' + print datastore_perms.stdout.read() + except psycopg2.Error as e: + print '[prerun] Could not initialize datastore' + print str(e) + + except subprocess.CalledProcessError, e: + if 'OperationalError' in e.output: + print e.output + print '[prerun] Database not ready, waiting a bit before exit...' + time.sleep(5) + sys.exit(1) + else: + print e.output + raise e + finally: + cursor.close() + connection.close() + + +def create_sysadmin(): + + name = os.environ.get('CKAN_SYSADMIN_NAME') + password = os.environ.get('CKAN_SYSADMIN_PASSWORD') + email = os.environ.get('CKAN_SYSADMIN_EMAIL') + + if name and password and email: + + # Check if user exists + command = ['paster', '--plugin=ckan', 'user', name, '-c', ckan_ini] + + out = subprocess.check_output(command) + if 'User:None' not in re.sub(r'\s', '', out): + print '[prerun] Sysadmin user exists, skipping creation' + return + + # Create user + command = ['paster', '--plugin=ckan', 'user', 'add', + name, + 'password=' + password, + 'email=' + email, + '-c', ckan_ini] + + subprocess.call(command) + print '[prerun] Created user {0}'.format(name) + + # Make it sysadmin + command = ['paster', '--plugin=ckan', 'sysadmin', 'add', + name, + '-c', ckan_ini] + + subprocess.call(command) + print '[prerun] Made user {0} a sysadmin'.format(name) + + +if __name__ == '__main__': + + maintenance = os.environ.get('MAINTENANCE_MODE', '').lower() == 'true' + + if maintenance: + print '[prerun] Maintenance mode, skipping setup...' + else: + check_main_db_connection() + init_db() + update_plugins() + check_datastore_db_connection() + init_datastore_db() + check_solr_connection() + create_sysadmin() \ No newline at end of file diff --git a/ckan-base/2.8/setup/start_ckan.sh b/ckan-base/2.8/setup/start_ckan.sh new file mode 100755 index 0000000..d1c6726 --- /dev/null +++ b/ckan-base/2.8/setup/start_ckan.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +# Run the prerun script to init CKAN and create the default admin user +sudo -u ckan -EH python prerun.py + +# Run any startup scripts provided by images extending this one +if [[ -d "/docker-entrypoint.d" ]] +then + for f in /docker-entrypoint.d/*; do + case "$f" in + *.sh) echo "$0: Running init file $f"; . "$f" ;; + *.py) echo "$0: Running init file $f"; python "$f"; echo ;; + *) echo "$0: Ignoring $f (not an sh or py file)" ;; + esac + echo + done +fi + +# Set the common uwsgi options +UWSGI_OPTS="--plugins http,python,gevent --socket /tmp/uwsgi.sock --uid 92 --gid 92 --http :5000 --master --enable-threads --paste config:/srv/app/production.ini --paste-logger --lazy-apps --gevent 2000 -p 2 -L" + +# Check whether http basic auth password protection is enabled and enable basicauth routing on uwsgi respecfully +if [ $? -eq 0 ] +then + if [ "$PASSWORD_PROTECT" = true ] + then + if [ "$HTPASSWD_USER" ] || [ "$HTPASSWD_PASSWORD" ] + then + # Generate htpasswd file for basicauth + htpasswd -d -b -c /srv/app/.htpasswd $HTPASSWD_USER $HTPASSWD_PASSWORD + # Start supervisord + supervisord --configuration /etc/supervisord.conf & + # Start uwsgi with basicauth + sudo -u ckan -EH uwsgi --ini /srv/app/uwsgi.conf --pcre-jit $UWSGI_OPTS + else + echo "Missing HTPASSWD_USER or HTPASSWD_PASSWORD environment variables. Exiting..." + exit 1 + fi + else + # Start supervisord + supervisord --configuration /etc/supervisord.conf & + # Start uwsgi + sudo -u ckan -EH uwsgi $UWSGI_OPTS + fi +else + echo "[prerun] failed...not starting CKAN." +fi diff --git a/ckan-base/2.8/setup/supervisor.worker.conf b/ckan-base/2.8/setup/supervisor.worker.conf new file mode 100755 index 0000000..0ca43ba --- /dev/null +++ b/ckan-base/2.8/setup/supervisor.worker.conf @@ -0,0 +1,13 @@ +[program:ckan-worker] +command=paster --plugin=ckan jobs worker -c /srv/app/production.ini +priority=501 +autostart=true +autorestart=true +redirect_stderr=true +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stdout +stderr_logfile_maxbytes=0 +user=ckan +environment=HOME="/srv/app",USER="ckan" + diff --git a/ckan-base/2.8/setup/supervisord.conf b/ckan-base/2.8/setup/supervisord.conf new file mode 100755 index 0000000..a3f6671 --- /dev/null +++ b/ckan-base/2.8/setup/supervisord.conf @@ -0,0 +1,23 @@ +[unix_http_server] +file = /tmp/supervisor.sock +chmod = 0777 +chown = nobody:nogroup + +[supervisord] +logfile = /tmp/supervisord.log +logfile_maxbytes = 50MB +logfile_backups=10 +loglevel = info +pidfile = /tmp/supervisord.pid +nodaemon = true +umask = 022 +identifier = supervisor + +[supervisorctl] +serverurl = unix:///tmp/supervisor.sock + +[rpcinterface:supervisor] +supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface + +[include] +files = /etc/supervisord.d/*.conf diff --git a/ckan-base/2.8/setup/uwsgi.conf b/ckan-base/2.8/setup/uwsgi.conf new file mode 100755 index 0000000..6321d6d --- /dev/null +++ b/ckan-base/2.8/setup/uwsgi.conf @@ -0,0 +1,2 @@ +[uwsgi] +route = ^(?!/api).*$ basicauth:Restricted,/srv/app/.htpasswd diff --git a/ckan-base/2.9/Dockerfile b/ckan-base/2.9/Dockerfile new file mode 100755 index 0000000..45b5aae --- /dev/null +++ b/ckan-base/2.9/Dockerfile @@ -0,0 +1,109 @@ +FROM alpine:3.13 + +# Internal environment variables +ENV APP_DIR=/srv/app +ENV SRC_DIR=/srv/app/src +ENV CKAN_INI=${APP_DIR}/ckan.ini +ENV PIP_SRC=${SRC_DIR} +ENV CKAN_STORAGE_PATH=/var/lib/ckan +ENV GIT_URL=https://github.com/ckan/ckan.git +# CKAN version to build +ENV GIT_BRANCH=ckan-2.9.4 +# Customize these on the .env file if needed +ENV CKAN_SITE_URL=http://localhost:5000 +ENV CKAN__PLUGINS image_view text_view recline_view datastore datapusher envvars + +WORKDIR ${APP_DIR} + +# Install necessary packages to run CKAN +RUN apk add --no-cache tzdata \ + git \ + gettext \ + postgresql-client \ + python3 \ + apache2-utils \ + libxml2 \ + libxslt \ + musl-dev \ + uwsgi-http \ + uwsgi-corerouter \ + uwsgi-python3 \ + py3-gevent \ + uwsgi-gevent \ + libmagic \ + curl \ + sudo && \ + # Packages to build CKAN requirements and plugins + apk add --no-cache --virtual .build-deps \ + postgresql-dev \ + gcc \ + make \ + g++ \ + autoconf \ + automake \ + libtool \ + python3-dev \ + py3-virtualenv \ + libxml2-dev \ + libxslt-dev \ + linux-headers + +# Create src directory (SRC_DIR) +RUN mkdir -p ${SRC_DIR} + +# Install pip +RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/get-pip.py && \ + python3 ${SRC_DIR}/get-pip.py + +# Set up Python3 virtual environment +RUN cd ${APP_DIR} && \ + python3 -m venv ${APP_DIR} && \ + source ${APP_DIR}/bin/activate + +# Virtual environment binaries/scripts to be used first +ENV PATH=${APP_DIR}/bin:${PATH} + +# Install CKAN code +RUN pip3 install -e git+${GIT_URL}@${GIT_BRANCH}#egg=ckan + +# Install uwsgi +RUN pip3 install uwsgi + +# Complete CKAN install +RUN cd ${SRC_DIR}/ckan && \ + cp who.ini ${APP_DIR} && \ + pip3 install -r requirement-setuptools.txt && \ + pip3 install --no-binary :all: -r requirements.txt && \ + # Install CKAN envvars to support loading config from environment variables + pip3 install -e git+https://github.com/okfn/ckanext-envvars.git#egg=ckanext-envvars + +# Create and update CKAN config +RUN ckan generate config ${CKAN_INI} + +# Install and configure supervisor +RUN pip3 install supervisor && \ +mkdir /etc/supervisord.d + +# Copy all setup files +COPY 2.9/setup ${APP_DIR} +COPY 2.9/setup/supervisor.worker.conf /etc/supervisord.d/worker.conf +COPY 2.9/setup/supervisord.conf /etc/supervisord.conf + +# Create a local user and group to run the app +RUN addgroup -g 92 -S ckan && \ + adduser -u 92 -h /srv/app -H -D -S -G ckan ckan + +# Create local storage folder +RUN mkdir -p $CKAN_STORAGE_PATH && \ + chown -R ckan:ckan $CKAN_STORAGE_PATH + +# Create entrypoint directory for children image scripts +ONBUILD RUN mkdir /docker-entrypoint.d + +RUN chown ckan -R /srv/app + +EXPOSE 5000 + +HEALTHCHECK --interval=10s --timeout=5s --retries=5 CMD curl --fail http://localhost:5000/api/3/action/status_show || exit 1 + +CMD ["/srv/app/start_ckan.sh"] \ No newline at end of file diff --git a/ckan-base/2.9/setup/ckan-uwsgi.ini b/ckan-base/2.9/setup/ckan-uwsgi.ini new file mode 100644 index 0000000..2361f36 --- /dev/null +++ b/ckan-base/2.9/setup/ckan-uwsgi.ini @@ -0,0 +1,15 @@ +[uwsgi] +http-socket = :5000 +uid = ckan +guid = ckan +plugins = python3 +wsgi-file = /srv/app/wsgi.py +virtualenv = /srv/app +module = wsgi:application +master = true +processes = 5 +pidfile = /tmp/%n.pid +harakiri = 50 +max-requests = 5000 +vacuum = true +callable = application diff --git a/ckan-base/2.9/setup/prerun.py b/ckan-base/2.9/setup/prerun.py new file mode 100755 index 0000000..f8dd693 --- /dev/null +++ b/ckan-base/2.9/setup/prerun.py @@ -0,0 +1,194 @@ +import os +import sys +import subprocess +import psycopg2 +import urllib3 +import time +import re + +ckan_ini = os.environ.get('CKAN_INI', '/srv/app/ckan.ini') + +RETRY = 5 + +def update_plugins(): + + plugins = os.environ.get('CKAN__PLUGINS', '') + print('[prerun] Setting the following plugins in {}:'.format(ckan_ini)) + print(plugins) + cmd = ['ckan', 'config-tool', ckan_ini, + 'ckan.plugins = {}'.format(plugins)] + subprocess.check_output(cmd, stderr=subprocess.STDOUT) + print('[prerun] Plugins set.') + + +def check_main_db_connection(retry=None): + + conn_str = os.environ.get('CKAN_SQLALCHEMY_URL') + if not conn_str: + print('[prerun] CKAN_SQLALCHEMY_URL not defined, not checking db') + return check_db_connection(conn_str, retry) + + +def check_datastore_db_connection(retry=None): + + conn_str = os.environ.get('CKAN_DATASTORE_WRITE_URL') + if not conn_str: + print('[prerun] CKAN_DATASTORE_WRITE_URL not defined, not checking db') + return check_db_connection(conn_str, retry) + + +def check_db_connection(conn_str, retry=None): + + if retry is None: + retry = RETRY + elif retry == 0: + print('[prerun] Giving up after 5 tries...') + sys.exit(1) + + try: + connection = psycopg2.connect(conn_str) + + except psycopg2.Error as e: + print(str(e)) + print('[prerun] Unable to connect to the database, waiting...') + time.sleep(10) + check_db_connection(conn_str, retry=retry - 1) + else: + connection.close() + + +def check_solr_connection(retry=None): + + if retry is None: + retry = RETRY + elif retry == 0: + print('[prerun] Giving up after 5 tries...') + sys.exit(1) + + url = os.environ.get('CKAN_SOLR_URL', '') + search_url = '{url}/select/?q=*&wt=json'.format(url=url) + http = urllib3.PoolManager() + try: + r = http.request('GET', search_url) + except urllib3.exceptions.ConnectionError as e: + print(str(e)) + print('[prerun] Unable to connect to solr, waiting...') + time.sleep(10) + check_solr_connection(retry=retry - 1) + else: + print('[prerun] Connection Status from SOLR is ', (r.status)) + +def init_db(): + + db_command = ['ckan', '-c', ckan_ini, + 'db', 'init'] + print('[prerun] Initializing or upgrading db - start') + try: + subprocess.check_output(db_command, stderr=subprocess.STDOUT) + print('[prerun] Initializing or upgrading db - end') + except subprocess.CalledProcessError as e: + if 'OperationalError' in e.output: + print(e.output) + print('[prerun] Database not ready, waiting a bit before exit...') + time.sleep(5) + sys.exit(1) + else: + print(e.output) + raise e + + +def init_datastore_db(): + + conn_str = os.environ.get('CKAN_DATASTORE_WRITE_URL') + if not conn_str: + print('[prerun] Skipping datastore initialization') + return + + datastore_perms_command = ['ckan', '-c', ckan_ini, + 'datastore', 'set-permissions'] + + connection = psycopg2.connect(conn_str) + cursor = connection.cursor() + + print('[prerun] Initializing datastore db - start') + try: + datastore_perms = subprocess.Popen( + datastore_perms_command, + stdout=subprocess.PIPE) + + perms_sql = datastore_perms.stdout.read().decode('utf-8') + # Remove internal pg command as psycopg2 does not like it + perms_sql = re.sub('\\\\connect \"(.*)\"', '', perms_sql) + cursor.execute(perms_sql) + for notice in connection.notices: + print(notice) + + connection.commit() + + print('[prerun] Initializing datastore db - end') + print(datastore_perms.stdout.read().decode('utf-8')) + except psycopg2.Error as e: + print('[prerun] Could not initialize datastore') + print(str(e)) + + except subprocess.CalledProcessError as e: + if 'OperationalError' in e.output: + print(e.output) + print('[prerun] Database not ready, waiting a bit before exit...') + time.sleep(5) + sys.exit(1) + else: + print(e.output) + raise e + finally: + cursor.close() + connection.close() + + +def create_sysadmin(): + + name = os.environ.get('CKAN_SYSADMIN_NAME') + password = os.environ.get('CKAN_SYSADMIN_PASSWORD') + email = os.environ.get('CKAN_SYSADMIN_EMAIL') + + if name and password and email: + + # Check if user exists + command = ['ckan', '-c', ckan_ini, 'user', 'show', name,] + + out = subprocess.check_output(command) + if 'User:None' not in re.sub(r'\s', '', out.decode()): + print('[prerun] Sysadmin user exists, skipping creation') + return + + # Create user + command = ['ckan', '-c', ckan_ini, 'user', 'add', + name, + 'password=' + password, + 'email=' + email] + + subprocess.call(command) + print('[prerun] Created user {0}'.format(name)) + + # Make it sysadmin + command = ['ckan', '-c', ckan_ini, 'sysadmin', 'add', + name] + + subprocess.call(command) + print('[prerun] Made user {0} a sysadmin'.format(name)) + + +if __name__ == '__main__': + + maintenance = os.environ.get('MAINTENANCE_MODE', '').lower() == 'true' + + if maintenance: + print('[prerun] Maintenance mode, skipping setup...') + else: + check_main_db_connection() + init_db() + update_plugins() + check_datastore_db_connection() + init_datastore_db() + check_solr_connection() + create_sysadmin() diff --git a/ckan-base/2.9/setup/start_ckan.sh b/ckan-base/2.9/setup/start_ckan.sh new file mode 100755 index 0000000..5a119e6 --- /dev/null +++ b/ckan-base/2.9/setup/start_ckan.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +# Run the prerun script to init CKAN and create the default admin user +sudo -u ckan -EH python3 prerun.py + +# Run any startup scripts provided by images extending this one +if [[ -d "/docker-entrypoint.d" ]] +then + for f in /docker-entrypoint.d/*; do + case "$f" in + *.sh) echo "$0: Running init file $f"; . "$f" ;; + *.py) echo "$0: Running init file $f"; python "$f"; echo ;; + *) echo "$0: Ignoring $f (not an sh or py file)" ;; + esac + echo + done +fi + + +# Check whether http basic auth password protection is enabled and enable basicauth routing on uwsgi respecfully +if [ $? -eq 0 ] +then + if [ "$PASSWORD_PROTECT" = true ] + then + if [ "$HTPASSWD_USER" ] || [ "$HTPASSWD_PASSWORD" ] + then + # Generate htpasswd file for basicauth + htpasswd -d -b -c /srv/app/.htpasswd $HTPASSWD_USER $HTPASSWD_PASSWORD + # Start supervisord + supervisord --configuration /etc/supervisord.conf & + # Start uwsgi with basicauth + sudo -u ckan -EH uwsgi -i ckan-uwsgi.ini + else + echo "Missing HTPASSWD_USER or HTPASSWD_PASSWORD environment variables. Exiting..." + exit 1 + fi + else + # Start supervisord + supervisord --configuration /etc/supervisord.conf & + # Start uwsgi + sudo -u ckan -EH uwsgi -i ckan-uwsgi.ini + fi +else + echo "[prerun] failed...not starting CKAN." +fi + diff --git a/ckan-base/2.9/setup/supervisor.worker.conf b/ckan-base/2.9/setup/supervisor.worker.conf new file mode 100644 index 0000000..9d46f37 --- /dev/null +++ b/ckan-base/2.9/setup/supervisor.worker.conf @@ -0,0 +1,12 @@ +[program:ckan-worker] +command=ckan -c /srv/app/ckan.ini jobs worker +priority=501 +autostart=true +autorestart=true +redirect_stderr=true +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stdout +stderr_logfile_maxbytes=0 +user=ckan +environment=HOME="/srv/app",USER="ckan" \ No newline at end of file diff --git a/ckan-base/2.9/setup/supervisord.conf b/ckan-base/2.9/setup/supervisord.conf new file mode 100644 index 0000000..052dbc5 --- /dev/null +++ b/ckan-base/2.9/setup/supervisord.conf @@ -0,0 +1,23 @@ +[unix_http_server] +file = /tmp/supervisor.sock +chmod = 0777 +chown = nobody:nogroup + +[supervisord] +logfile = /tmp/supervisord.log +logfile_maxbytes = 50MB +logfile_backups=10 +loglevel = info +pidfile = /tmp/supervisord.pid +nodaemon = true +umask = 022 +identifier = supervisor + +[supervisorctl] +serverurl = unix:///tmp/supervisor.sock + +[rpcinterface:supervisor] +supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface + +[include] +files = /etc/supervisord.d/*.conf \ No newline at end of file diff --git a/ckan-base/2.9/setup/wsgi.py b/ckan-base/2.9/setup/wsgi.py new file mode 100644 index 0000000..b37d80e --- /dev/null +++ b/ckan-base/2.9/setup/wsgi.py @@ -0,0 +1,9 @@ +import os +from ckan.config.middleware import make_app +from ckan.cli import CKANConfigLoader +from logging.config import fileConfig as loggingFileConfig +config_filepath = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'ckan.ini') +abspath = os.path.join(os.path.dirname(os.path.abspath(__file__))) +loggingFileConfig(config_filepath) +config = CKANConfigLoader(config_filepath).get_config() +application = make_app(config) diff --git a/ckan-dev/2.7/Dockerfile b/ckan-dev/2.7/Dockerfile new file mode 100755 index 0000000..a421a59 --- /dev/null +++ b/ckan-dev/2.7/Dockerfile @@ -0,0 +1,20 @@ +FROM ckan/ckan-base:testing-only.2.7 + +LABEL maintainer="brett@kowh.ai" + +ENV APP_DIR=/srv/app +ENV SRC_EXTENSIONS_DIR=/srv/app/src_extensions + +# Install packages needed by the dev requirements +RUN apk add --no-cache libffi-dev + +# Install CKAN dev requirements +RUN pip install --no-binary :all: -r https://raw.githubusercontent.com/ckan/ckan/${GIT_BRANCH}/dev-requirements.txt + +# Create folder for local extensions sources +RUN mkdir $SRC_EXTENSIONS_DIR + +COPY setup/start_ckan_development.sh ${APP_DIR} + + +CMD ["/srv/app/start_ckan_development.sh"] diff --git a/ckan-dev/2.8/Dockerfile b/ckan-dev/2.8/Dockerfile new file mode 100755 index 0000000..6142a8d --- /dev/null +++ b/ckan-dev/2.8/Dockerfile @@ -0,0 +1,20 @@ +FROM ckan/ckan-base:testing-only.2.8 + +LABEL maintainer="brett@kowh.ai" + +ENV APP_DIR=/srv/app +ENV SRC_EXTENSIONS_DIR=/srv/app/src_extensions + +# Install packages needed by the dev requirements +RUN apk add --no-cache libffi-dev + +# Install CKAN dev requirements +RUN pip install --no-binary :all: -r https://raw.githubusercontent.com/ckan/ckan/${GIT_BRANCH}/dev-requirements.txt + +# Create folder for local extensions sources +RUN mkdir $SRC_EXTENSIONS_DIR + +COPY setup/start_ckan_development.sh ${APP_DIR} + + +CMD ["/srv/app/start_ckan_development.sh"] diff --git a/ckan-dev/2.9/Dockerfile b/ckan-dev/2.9/Dockerfile new file mode 100755 index 0000000..fc170ee --- /dev/null +++ b/ckan-dev/2.9/Dockerfile @@ -0,0 +1,29 @@ +FROM ckan/ckan-base:testing-only.2.9 + +LABEL maintainer="brett@kowh.ai" + +ENV APP_DIR=/srv/app +ENV SRC_EXTENSIONS_DIR=/srv/app/src_extensions + +# Install packages needed by the dev requirements +RUN apk add --no-cache libffi-dev + +# Set up Python3 virtual environment +RUN cd ${APP_DIR} && \ + source ${APP_DIR}/bin/activate + +# Virtual environment binaries/scripts to be used first +ENV PATH=${APP_DIR}/bin:${PATH} + +# Install CKAN dev requirements +# Will need to change this eventually - when CKAN 2.9 is out +# wget https://raw.githubusercontent.com/ckan/ckan/master/dev-requirements.txt +# RUN pip3 install --no-binary :all: -r https://raw.githubusercontent.com/ckan/ckan/master/dev-requirements.txt +RUN pip3 install -r https://raw.githubusercontent.com/ckan/ckan/master/dev-requirements.txt + +# Create folder for local extensions sources +RUN mkdir $SRC_EXTENSIONS_DIR + +COPY setup/start_ckan_development.sh ${APP_DIR} + +CMD ["/srv/app/start_ckan_development.sh"] diff --git a/ckan-dev/setup/start_ckan_development.sh b/ckan-dev/setup/start_ckan_development.sh new file mode 100755 index 0000000..331266c --- /dev/null +++ b/ckan-dev/setup/start_ckan_development.sh @@ -0,0 +1,82 @@ +#!/bin/bash + +# Install any local extensions in the src_extensions volume +echo "Looking for local extensions to install..." +echo "Extension dir contents:" +ls -la $SRC_EXTENSIONS_DIR +for i in $SRC_EXTENSIONS_DIR/* +do + if [ -d $i ]; + then + + if [ -f $i/pip-requirements.txt ]; + then + pip install -r $i/pip-requirements.txt + echo "Found requirements file in $i" + fi + if [ -f $i/requirements.txt ]; + then + pip install -r $i/requirements.txt + echo "Found requirements file in $i" + fi + if [ -f $i/dev-requirements.txt ]; + then + pip install -r $i/dev-requirements.txt + echo "Found dev-requirements file in $i" + fi + if [ -f $i/setup.py ]; + then + cd $i + python $i/setup.py develop + echo "Found setup.py file in $i" + cd $APP_DIR + fi + + # Point `use` in test.ini to location of `test-core.ini` + if [ -f $i/test.ini ]; + then + echo "Updating \`test.ini\` reference to \`test-core.ini\` for plugin $i" + paster --plugin=ckan config-tool $i/test.ini "use = config:../../src/ckan/test-core.ini" + fi + fi +done + +# Set debug to true +echo "Enabling debug mode" +ckan config-tool $CKAN_INI -s DEFAULT "debug = true" + +# Update the plugins setting in the ini file with the values defined in the env var +echo "Loading the following plugins: $CKAN__PLUGINS" +ckan config-tool $CKAN_INI "ckan.plugins = $CKAN__PLUGINS" + +# Update test-core.ini DB, SOLR & Redis settings +echo "Loading test settings into test-core.ini" +ckan config-tool $SRC_DIR/ckan/test-core.ini \ + "sqlalchemy.url = $TEST_CKAN_SQLALCHEMY_URL" \ + "ckan.datstore.write_url = $TEST_CKAN_DATASTORE_WRITE_URL" \ + "ckan.datstore.read_url = $TEST_CKAN_DATASTORE_READ_URL" \ + "solr_url = $TEST_CKAN_SOLR_URL" \ + "ckan.redis_url = $TEST_CKAN_REDIS_URL" + +# Run the prerun script to init CKAN and create the default admin user +sudo -u ckan -EH python prerun.py + +# Run any startup scripts provided by images extending this one +if [[ -d "/docker-entrypoint.d" ]] +then + for f in /docker-entrypoint.d/*; do + case "$f" in + *.sh) echo "$0: Running init file $f"; . "$f" ;; + *.py) echo "$0: Running init file $f"; python "$f"; echo ;; + *) echo "$0: Ignoring $f (not an sh or py file)" ;; + esac + echo + done +fi + +# Start supervisord +supervisord --configuration /etc/supervisord.conf & + +# Start the development server with automatic reload +# Check the --reloader options sudo -u ckan -EH ckan -c $CKAN_INI run --reloader +sudo -u ckan -EH ckan -c $CKAN_INI run diff --git a/docker-compose.yml b/docker-compose.yml index b2f5aa1..c99e3d4 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,10 @@ version: "3" +volumes: + pg_data: + solr_data: + ckan_storage: + services: nginx: @@ -11,7 +16,7 @@ services: - ckan ports: - "0.0.0.0:80:80" - + ckan: container_name: ckan build: @@ -29,18 +34,13 @@ services: ports: - "0.0.0.0:${CKAN_PORT}:5000" volumes: - - ckan_storage:/var/lib/ckan - - datapusher: - container_name: datapusher - image: kowhai/datapusher:0.0.17 - ports: - - "8800:8800" + - ckan_storage:/var/lib/ckan db: container_name: db build: - context: postgresql/ + context: . + dockerfile: postgresql/Dockerfile args: - DATASTORE_READONLY_PASSWORD=${DATASTORE_READONLY_PASSWORD} - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} @@ -50,19 +50,31 @@ services: - PGDATA=/var/lib/postgresql/data/db volumes: - pg_data:/var/lib/postgresql/data - + healthcheck: + test: ["CMD", "pg_isready", "-U", "ckan"] + + datapusher: + container_name: datapusher + build: + context: datapusher + ports: + - "8800:8800" + environment: + - DATAPUSHER_REWRITE_RESOURCES=${DATAPUSHER_REWRITE_RESOURCES} + - DATAPUSHER_REWRITE_URL=${DATAPUSHER_REWRITE_URL} + - CKAN__DATAPUSHER__CALLBACK_URL_BASE=${CKAN__DATAPUSHER__CALLBACK_URL_BASE} + solr: container_name: solr build: - context: solr/ + context: . + dockerfile: solr/Dockerfile + args: + - SOLR_VERSION=${SOLR_VERSION} + - CKAN_VERSION=${CKAN_VERSION} volumes: - - solr_data:/opt/solr/server/solr/ckan/data/index + - solr_data:/opt/solr/server/solr/ckan/data redis: container_name: redis - image: redis:alpine - -volumes: - ckan_storage: - pg_data: - solr_data: + image: redis:${REDIS_VERSION} diff --git a/docker-entrypoint-initdb.d/10_create_datastore.sql b/docker-entrypoint-initdb.d/10_create_datastore.sql new file mode 100755 index 0000000..87529ce --- /dev/null +++ b/docker-entrypoint-initdb.d/10_create_datastore.sql @@ -0,0 +1,4 @@ +\set datastore_ro_password '\'' `echo $DATASTORE_READONLY_PASSWORD` '\'' + +CREATE ROLE datastore_ro NOSUPERUSER NOCREATEDB NOCREATEROLE LOGIN PASSWORD :datastore_ro_password; +CREATE DATABASE datastore OWNER ckan ENCODING 'utf-8'; \ No newline at end of file diff --git a/docker-entrypoint-initdb.d/20_setup_test_databases.sql b/docker-entrypoint-initdb.d/20_setup_test_databases.sql new file mode 100755 index 0000000..61366d2 --- /dev/null +++ b/docker-entrypoint-initdb.d/20_setup_test_databases.sql @@ -0,0 +1,2 @@ +CREATE DATABASE ckan_test OWNER ckan ENCODING 'utf-8'; +CREATE DATABASE datastore_test OWNER ckan ENCODING 'utf-8'; \ No newline at end of file diff --git a/index.html b/index.html new file mode 100644 index 0000000..561b63b --- /dev/null +++ b/index.html @@ -0,0 +1,22 @@ + + + + + + + CKAN Docker NGINX landing page + + + + +

+ CKAN Docker NGINX landing page +

+ + + \ No newline at end of file diff --git a/notes.txt b/notes.txt new file mode 100644 index 0000000..dccc9b1 --- /dev/null +++ b/notes.txt @@ -0,0 +1,69 @@ + +Environment variables are in the .env file and loaded into the docker-compose.yml file at runtime. +The Dockerfile will need an ARG line also + +List all env variables and decide where they go +.env +env-ckan +ARG +ENV + +NB: .env variables can be referenced in the docker-compose.yml file (environment: part of the service) + The docker-compose.yml file can pass these variables to the Dockerfile as ARG parameters +--------------------------- from OKFN --------------------------- +# DB image settings +POSTGRES_PASSWORD=ckan +DATASTORE_READONLY_PASSWORD=datastore + +# Basic +CKAN_SITE_ID=default +CKAN_SITE_URL=http://ckan:5000 +CKAN_PORT=5000 +CKAN_SYSADMIN_NAME=ckan_admin +CKAN_SYSADMIN_PASSWORD=test1234 +CKAN_SYSADMIN_EMAIL=your_email@example.com +TZ=UTC + +# Database connections (TODO: avoid duplication) +CKAN_SQLALCHEMY_URL=postgresql://ckan:ckan@db/ckan +CKAN_DATASTORE_WRITE_URL=postgresql://ckan:ckan@db/datastore +CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:datastore@db/datastore + +# Test database connections +TEST_CKAN_SQLALCHEMY_URL=postgres://ckan:ckan@db/ckan_test +TEST_CKAN_DATASTORE_WRITE_URL=postgresql://ckan:ckan@db/datastore_test +TEST_CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:datastore@db/datastore_test + +# Other services connections +CKAN_SOLR_URL=http://solr:8983/solr/ckan +CKAN_REDIS_URL=redis://redis:6379/1 +CKAN_DATAPUSHER_URL=http://datapusher:8800 +CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://ckan:5000 + +TEST_CKAN_SOLR_URL=http://solr:8983/solr/ckan +TEST_CKAN_REDIS_URL=redis://redis:6379/1 + +# Core settings +CKAN__STORAGE_PATH=/var/lib/ckan + +CKAN_SMTP_SERVER=smtp.corporateict.domain:25 +CKAN_SMTP_STARTTLS=True +CKAN_SMTP_USER=user +CKAN_SMTP_PASSWORD=pass +CKAN_SMTP_MAIL_FROM=ckan@localhost + +# Extensions + +CKAN__PLUGINS=envvars image_view text_view recline_view datastore datapusher +CKAN__HARVEST__MQ__TYPE=redis +CKAN__HARVEST__MQ__HOSTNAME=redis +CKAN__HARVEST__MQ__PORT=6379 +CKAN__HARVEST__MQ__REDIS_DB=1 + + +docker-compose.yml +DATASTORE_READONLY_PASSWORD=${DATASTORE_READONLY_PASSWORD} +POSTGRES_PASSWORD=${POSTGRES_PASSWORD} +DATASTORE_READONLY_PASSWORD=${DATASTORE_READONLY_PASSWORD} +POSTGRES_PASSWORD=${POSTGRES_PASSWORD} +PGDATA=/var/lib/postgresql/data/db \ No newline at end of file diff --git a/patches/00_patch_sql_url.patch b/patches/00_patch_sql_url.patch new file mode 100644 index 0000000..46a7135 --- /dev/null +++ b/patches/00_patch_sql_url.patch @@ -0,0 +1,11 @@ +--- ckan/ckan/model/__init__.py 2021-02-16 14:47:06.168327441 +0100 ++++ ckan/ckan/model/__init__.py 2021-02-16 14:48:00.740780218 +0100 +@@ -266,7 +266,7 @@ + self.reset_alembic_output() + alembic_config = AlembicConfig(self._alembic_ini) + alembic_config.set_main_option( +- "sqlalchemy.url", str(self.metadata.bind.url) ++ "sqlalchemy.url", str(self.metadata.bind.url).replace('%', '%%') + ) + try: + sqlalchemy_migrate_version = self.metadata.bind.execute( diff --git a/patches/01_patch_resource_replace_upload.patch b/patches/01_patch_resource_replace_upload.patch new file mode 100644 index 0000000..3f11fac --- /dev/null +++ b/patches/01_patch_resource_replace_upload.patch @@ -0,0 +1,11 @@ +--- ckan/ckan/logic/action/update.py 2021-02-17 16:46:55.673578728 +0100 ++++ ckan/ckan/logic/action/update-edit.py 2021-02-17 16:47:28.905879170 +0100 +@@ -929,7 +929,7 @@ + + ''' + model = context['model'] +- session = model.Session ++ session = model.meta.create_local_session() + context['session'] = session + + user = context['user'] diff --git a/patches/02_patch_postgres_username_split.patch b/patches/02_patch_postgres_username_split.patch new file mode 100644 index 0000000..780b432 --- /dev/null +++ b/patches/02_patch_postgres_username_split.patch @@ -0,0 +1,11 @@ +--- ckan/ckanext/datastore/backend/postgres.py 2021-02-18 11:01:56.692267462 +0100 ++++ ckan/ckanext/datastore/backend/postgres-patch.py 2021-02-18 13:45:16.033193435 +0100 +@@ -1690,7 +1690,7 @@ + read only user. + ''' + write_connection = self._get_write_engine().connect() +- read_connection_user = sa_url.make_url(self.read_url).username ++ read_connection_user = sa_url.make_url(self.read_url).username.split("@")[0] + + drop_foo_sql = u'DROP TABLE IF EXISTS _foo' + diff --git a/scripts/apply_ckan_patches.sh b/scripts/apply_ckan_patches.sh new file mode 100755 index 0000000..a7bceb9 --- /dev/null +++ b/scripts/apply_ckan_patches.sh @@ -0,0 +1,5 @@ +#!/bin/bash +shopt -s nullglob +for patch in patches/*.patch; do + /usr/bin/patch -p0 -i $patch +done diff --git a/setup/ckan-uwsgi.ini b/setup/ckan-uwsgi.ini new file mode 100644 index 0000000..2361f36 --- /dev/null +++ b/setup/ckan-uwsgi.ini @@ -0,0 +1,15 @@ +[uwsgi] +http-socket = :5000 +uid = ckan +guid = ckan +plugins = python3 +wsgi-file = /srv/app/wsgi.py +virtualenv = /srv/app +module = wsgi:application +master = true +processes = 5 +pidfile = /tmp/%n.pid +harakiri = 50 +max-requests = 5000 +vacuum = true +callable = application diff --git a/setup/datapusher-uwsgi.ini b/setup/datapusher-uwsgi.ini new file mode 100644 index 0000000..4c8274a --- /dev/null +++ b/setup/datapusher-uwsgi.ini @@ -0,0 +1,14 @@ +[uwsgi] +http = 0.0.0.0:8800 +uid = www-data +guid = www-data +wsgi-file = /srv/app/datapusher.wsgi +master = true +socket = /tmp/uwsgi.sock +plugins = http,python +pidfile = /tmp/%n.pid +harakiri = 50 +max-requests = 5000 +vacuum = true +callable = application +buffer-size = 32768 \ No newline at end of file diff --git a/setup/datapusher_settings.py b/setup/datapusher_settings.py new file mode 100644 index 0000000..d6ee97a --- /dev/null +++ b/setup/datapusher_settings.py @@ -0,0 +1,37 @@ +import os +import uuid + +DEBUG = False +TESTING = False +SECRET_KEY = str(uuid.uuid4()) +USERNAME = str(uuid.uuid4()) +PASSWORD = str(uuid.uuid4()) + +NAME = 'datapusher' + +# Webserver host and port + +HOST = os.environ.get('DATAPUSHER_HOST', '0.0.0.0') +PORT = os.environ.get('DATAPUSHER_PORT', 8800) + +# Database + +SQLALCHEMY_DATABASE_URI = os.environ.get('DATAPUSHER_SQLALCHEMY_DATABASE_URI', 'sqlite:////tmp/job_store.db') + +# Download and streaming settings + +MAX_CONTENT_LENGTH = int(os.environ.get('DATAPUSHER_MAX_CONTENT_LENGTH', '1024000')) +CHUNK_SIZE = int(os.environ.get('DATAPUSHER_CHUNK_SIZE', '16384')) +CHUNK_INSERT_ROWS = int(os.environ.get('DATAPUSHER_CHUNK_INSERT_ROWS', '250')) +DOWNLOAD_TIMEOUT = int(os.environ.get('DATAPUSHER_DOWNLOAD_TIMEOUT', '30')) + +# Verify SSL +SSL_VERIFY = os.environ.get('DATAPUSHER_SSL_VERIFY', True) + +# logging +#LOG_FILE = '/tmp/ckan_service.log' +STDERR = True + +# Rewrite resource URL's when ckan callback url base is used +REWRITE_RESOURCES = os.environ.get('DATAPUSHER_REWRITE_RESOURCES', True) +REWRITE_URL = os.environ.get('DATAPUSHER_REWRITE_URL', 'http://ckan:5000/') diff --git a/setup/nginx.conf b/setup/nginx.conf new file mode 100644 index 0000000..7f06cea --- /dev/null +++ b/setup/nginx.conf @@ -0,0 +1,34 @@ + +user nginx; +worker_processes auto; + +error_log /var/log/nginx/error.log warn; +pid /var/run/nginx.pid; + + +events { + worker_connections 1024; +} + + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + #tcp_nopush on; + + keepalive_timeout 65; + + #gzip on; + + include /etc/nginx/sites-enabled/*.conf; +} + + diff --git a/setup/prerun.py b/setup/prerun.py new file mode 100755 index 0000000..f8dd693 --- /dev/null +++ b/setup/prerun.py @@ -0,0 +1,194 @@ +import os +import sys +import subprocess +import psycopg2 +import urllib3 +import time +import re + +ckan_ini = os.environ.get('CKAN_INI', '/srv/app/ckan.ini') + +RETRY = 5 + +def update_plugins(): + + plugins = os.environ.get('CKAN__PLUGINS', '') + print('[prerun] Setting the following plugins in {}:'.format(ckan_ini)) + print(plugins) + cmd = ['ckan', 'config-tool', ckan_ini, + 'ckan.plugins = {}'.format(plugins)] + subprocess.check_output(cmd, stderr=subprocess.STDOUT) + print('[prerun] Plugins set.') + + +def check_main_db_connection(retry=None): + + conn_str = os.environ.get('CKAN_SQLALCHEMY_URL') + if not conn_str: + print('[prerun] CKAN_SQLALCHEMY_URL not defined, not checking db') + return check_db_connection(conn_str, retry) + + +def check_datastore_db_connection(retry=None): + + conn_str = os.environ.get('CKAN_DATASTORE_WRITE_URL') + if not conn_str: + print('[prerun] CKAN_DATASTORE_WRITE_URL not defined, not checking db') + return check_db_connection(conn_str, retry) + + +def check_db_connection(conn_str, retry=None): + + if retry is None: + retry = RETRY + elif retry == 0: + print('[prerun] Giving up after 5 tries...') + sys.exit(1) + + try: + connection = psycopg2.connect(conn_str) + + except psycopg2.Error as e: + print(str(e)) + print('[prerun] Unable to connect to the database, waiting...') + time.sleep(10) + check_db_connection(conn_str, retry=retry - 1) + else: + connection.close() + + +def check_solr_connection(retry=None): + + if retry is None: + retry = RETRY + elif retry == 0: + print('[prerun] Giving up after 5 tries...') + sys.exit(1) + + url = os.environ.get('CKAN_SOLR_URL', '') + search_url = '{url}/select/?q=*&wt=json'.format(url=url) + http = urllib3.PoolManager() + try: + r = http.request('GET', search_url) + except urllib3.exceptions.ConnectionError as e: + print(str(e)) + print('[prerun] Unable to connect to solr, waiting...') + time.sleep(10) + check_solr_connection(retry=retry - 1) + else: + print('[prerun] Connection Status from SOLR is ', (r.status)) + +def init_db(): + + db_command = ['ckan', '-c', ckan_ini, + 'db', 'init'] + print('[prerun] Initializing or upgrading db - start') + try: + subprocess.check_output(db_command, stderr=subprocess.STDOUT) + print('[prerun] Initializing or upgrading db - end') + except subprocess.CalledProcessError as e: + if 'OperationalError' in e.output: + print(e.output) + print('[prerun] Database not ready, waiting a bit before exit...') + time.sleep(5) + sys.exit(1) + else: + print(e.output) + raise e + + +def init_datastore_db(): + + conn_str = os.environ.get('CKAN_DATASTORE_WRITE_URL') + if not conn_str: + print('[prerun] Skipping datastore initialization') + return + + datastore_perms_command = ['ckan', '-c', ckan_ini, + 'datastore', 'set-permissions'] + + connection = psycopg2.connect(conn_str) + cursor = connection.cursor() + + print('[prerun] Initializing datastore db - start') + try: + datastore_perms = subprocess.Popen( + datastore_perms_command, + stdout=subprocess.PIPE) + + perms_sql = datastore_perms.stdout.read().decode('utf-8') + # Remove internal pg command as psycopg2 does not like it + perms_sql = re.sub('\\\\connect \"(.*)\"', '', perms_sql) + cursor.execute(perms_sql) + for notice in connection.notices: + print(notice) + + connection.commit() + + print('[prerun] Initializing datastore db - end') + print(datastore_perms.stdout.read().decode('utf-8')) + except psycopg2.Error as e: + print('[prerun] Could not initialize datastore') + print(str(e)) + + except subprocess.CalledProcessError as e: + if 'OperationalError' in e.output: + print(e.output) + print('[prerun] Database not ready, waiting a bit before exit...') + time.sleep(5) + sys.exit(1) + else: + print(e.output) + raise e + finally: + cursor.close() + connection.close() + + +def create_sysadmin(): + + name = os.environ.get('CKAN_SYSADMIN_NAME') + password = os.environ.get('CKAN_SYSADMIN_PASSWORD') + email = os.environ.get('CKAN_SYSADMIN_EMAIL') + + if name and password and email: + + # Check if user exists + command = ['ckan', '-c', ckan_ini, 'user', 'show', name,] + + out = subprocess.check_output(command) + if 'User:None' not in re.sub(r'\s', '', out.decode()): + print('[prerun] Sysadmin user exists, skipping creation') + return + + # Create user + command = ['ckan', '-c', ckan_ini, 'user', 'add', + name, + 'password=' + password, + 'email=' + email] + + subprocess.call(command) + print('[prerun] Created user {0}'.format(name)) + + # Make it sysadmin + command = ['ckan', '-c', ckan_ini, 'sysadmin', 'add', + name] + + subprocess.call(command) + print('[prerun] Made user {0} a sysadmin'.format(name)) + + +if __name__ == '__main__': + + maintenance = os.environ.get('MAINTENANCE_MODE', '').lower() == 'true' + + if maintenance: + print('[prerun] Maintenance mode, skipping setup...') + else: + check_main_db_connection() + init_db() + update_plugins() + check_datastore_db_connection() + init_datastore_db() + check_solr_connection() + create_sysadmin() diff --git a/setup/sites-available/ckan.conf b/setup/sites-available/ckan.conf new file mode 100644 index 0000000..7ffc6bb --- /dev/null +++ b/setup/sites-available/ckan.conf @@ -0,0 +1,18 @@ +proxy_cache_path /tmp/nginx_cache levels=1:2 keys_zone=cache:30m max_size=250m; +proxy_temp_path /tmp/nginx_proxy 1 2; + +server { + client_max_body_size 100M; + location / { + proxy_pass http://ckan:5000/; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header Host $host; + proxy_cache cache; + proxy_cache_bypass $cookie_auth_tkt; + proxy_no_cache $cookie_auth_tkt; + proxy_cache_valid 30m; + proxy_cache_key $host$scheme$proxy_host$request_uri; + # In emergency comment out line to force caching + # proxy_ignore_headers X-Accel-Expires Expires Cache-Control; + } +} \ No newline at end of file diff --git a/setup/start_ckan.sh b/setup/start_ckan.sh new file mode 100755 index 0000000..5a119e6 --- /dev/null +++ b/setup/start_ckan.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +# Run the prerun script to init CKAN and create the default admin user +sudo -u ckan -EH python3 prerun.py + +# Run any startup scripts provided by images extending this one +if [[ -d "/docker-entrypoint.d" ]] +then + for f in /docker-entrypoint.d/*; do + case "$f" in + *.sh) echo "$0: Running init file $f"; . "$f" ;; + *.py) echo "$0: Running init file $f"; python "$f"; echo ;; + *) echo "$0: Ignoring $f (not an sh or py file)" ;; + esac + echo + done +fi + + +# Check whether http basic auth password protection is enabled and enable basicauth routing on uwsgi respecfully +if [ $? -eq 0 ] +then + if [ "$PASSWORD_PROTECT" = true ] + then + if [ "$HTPASSWD_USER" ] || [ "$HTPASSWD_PASSWORD" ] + then + # Generate htpasswd file for basicauth + htpasswd -d -b -c /srv/app/.htpasswd $HTPASSWD_USER $HTPASSWD_PASSWORD + # Start supervisord + supervisord --configuration /etc/supervisord.conf & + # Start uwsgi with basicauth + sudo -u ckan -EH uwsgi -i ckan-uwsgi.ini + else + echo "Missing HTPASSWD_USER or HTPASSWD_PASSWORD environment variables. Exiting..." + exit 1 + fi + else + # Start supervisord + supervisord --configuration /etc/supervisord.conf & + # Start uwsgi + sudo -u ckan -EH uwsgi -i ckan-uwsgi.ini + fi +else + echo "[prerun] failed...not starting CKAN." +fi + diff --git a/setup/start_datapusher.sh b/setup/start_datapusher.sh new file mode 100755 index 0000000..3b78d19 --- /dev/null +++ b/setup/start_datapusher.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +echo "[start_datapusher.sh] Starting supervisord." +# Start supervisord +supervisord --configuration /etc/supervisord.conf diff --git a/setup/supervisor.uwsgi.conf b/setup/supervisor.uwsgi.conf new file mode 100644 index 0000000..67a6c37 --- /dev/null +++ b/setup/supervisor.uwsgi.conf @@ -0,0 +1,12 @@ +[program:datapusher-uwsgi] +command=/usr/sbin/uwsgi -i /srv/app/datapusher-uwsgi.ini +priority=501 +autostart=true +autorestart=true +redirect_stderr=true +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stdout +stderr_logfile_maxbytes=0 +user=www-data +environment=HOME="/srv/app",USER="www-data" \ No newline at end of file diff --git a/setup/supervisor.worker.conf b/setup/supervisor.worker.conf new file mode 100644 index 0000000..9d46f37 --- /dev/null +++ b/setup/supervisor.worker.conf @@ -0,0 +1,12 @@ +[program:ckan-worker] +command=ckan -c /srv/app/ckan.ini jobs worker +priority=501 +autostart=true +autorestart=true +redirect_stderr=true +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stdout +stderr_logfile_maxbytes=0 +user=ckan +environment=HOME="/srv/app",USER="ckan" \ No newline at end of file diff --git a/setup/supervisord.conf b/setup/supervisord.conf new file mode 100644 index 0000000..f21c89c --- /dev/null +++ b/setup/supervisord.conf @@ -0,0 +1,23 @@ +[unix_http_server] +file = /tmp/supervisor.sock +chmod = 0777 +chown = nobody:nogroup + +[supervisord] +logfile = /tmp/supervisord.log +logfile_maxbytes = 50MB +logfile_backups=10 +loglevel = trace +pidfile = /tmp/supervisord.pid +nodaemon = true +umask = 022 +identifier = supervisor + +[supervisorctl] +serverurl = unix:///tmp/supervisor.sock + +[rpcinterface:supervisor] +supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface + +[include] +files = /etc/supervisord.d/*.conf \ No newline at end of file diff --git a/setup/wsgi.py b/setup/wsgi.py new file mode 100644 index 0000000..b37d80e --- /dev/null +++ b/setup/wsgi.py @@ -0,0 +1,9 @@ +import os +from ckan.config.middleware import make_app +from ckan.cli import CKANConfigLoader +from logging.config import fileConfig as loggingFileConfig +config_filepath = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'ckan.ini') +abspath = os.path.join(os.path.dirname(os.path.abspath(__file__))) +loggingFileConfig(config_filepath) +config = CKANConfigLoader(config_filepath).get_config() +application = make_app(config) diff --git a/solr/Dockerfile b/solr/Dockerfile index 8922342..1e0c650 100755 --- a/solr/Dockerfile +++ b/solr/Dockerfile @@ -2,9 +2,10 @@ FROM solr:6.6.6 # Enviroment variables ENV SOLR_CORE ckan -ENV SOLR_VERSION 6.6.6 -ENV CKAN_VERSION 2.9.1 -###TODO!!! CKAN_VERSION to be passed in as an ARG + +# Build Arguments +ARG CKAN_VERSION +ARG SOLR_VERSION # root user for initial config USER root @@ -15,14 +16,14 @@ RUN mkdir -p /opt/solr/server/solr/${SOLR_CORE}/conf && \ mkdir -p /opt/solr/server/solr/${SOLR_CORE}/data/index # Add files -ADD solrconfig.xml \ -https://raw.githubusercontent.com/ckan/ckan/ckan-${CKAN_VERSION}/ckan/config/solr/schema.xml \ -https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/$SOLR_VERSION/solr/server/solr/configsets/basic_configs/conf/currency.xml \ -https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/$SOLR_VERSION/solr/server/solr/configsets/basic_configs/conf/synonyms.txt \ -https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/$SOLR_VERSION/solr/server/solr/configsets/basic_configs/conf/stopwords.txt \ -https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/$SOLR_VERSION/solr/server/solr/configsets/basic_configs/conf/protwords.txt \ -https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/$SOLR_VERSION/solr/server/solr/configsets/data_driven_schema_configs/conf/elevate.xml \ -/opt/solr/server/solr/$SOLR_CORE/conf/ +COPY ./solr/solrconfig-${CKAN_VERSION}.xml /opt/solr/server/solr/${SOLR_CORE}/conf/solrconfig.xml +ADD https://raw.githubusercontent.com/ckan/ckan/ckan-${CKAN_VERSION}/ckan/config/solr/schema.xml \ +https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/${SOLR_VERSION}/solr/server/solr/configsets/basic_configs/conf/currency.xml \ +https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/${SOLR_VERSION}/solr/server/solr/configsets/basic_configs/conf/synonyms.txt \ +https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/${SOLR_VERSION}/solr/server/solr/configsets/basic_configs/conf/stopwords.txt \ +https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/${SOLR_VERSION}/solr/server/solr/configsets/basic_configs/conf/protwords.txt \ +https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/${SOLR_VERSION}/solr/server/solr/configsets/data_driven_schema_configs/conf/elevate.xml \ +/opt/solr/server/solr/${SOLR_CORE}/conf/ # Create core.properties RUN echo name=${SOLR_CORE} > /opt/solr/server/solr/${SOLR_CORE}/core.properties @@ -32,3 +33,4 @@ RUN chown -R ${SOLR_USER}:${SOLR_USER} /opt/solr/server/solr/${SOLR_CORE} # non-root user for runtime USER ${SOLR_USER}:${SOLR_USER} + diff --git a/solr/solrconfig.xml b/solr/solrconfig-2.9.3.xml old mode 100755 new mode 100644 similarity index 99% rename from solr/solrconfig.xml rename to solr/solrconfig-2.9.3.xml index 8a3eade..5ddc4c4 --- a/solr/solrconfig.xml +++ b/solr/solrconfig-2.9.3.xml @@ -342,4 +342,4 @@ 5 - + \ No newline at end of file diff --git a/todo.txt b/todo.txt new file mode 100644 index 0000000..d5d6960 --- /dev/null +++ b/todo.txt @@ -0,0 +1,12 @@ +Get the NGINX container running +Add the redis container +Add the solr container +Add the db container +Get a datapusher container working with supervisord +Test the datapusher container works +Take out ARG's from docker-compose.yml is not needed +Test with new versions of CKAN (ie: if problems with Python 3.9) +Create (and push) the datapusher image +Create the ckan container +Create the ckanext container +