gitea/ckan
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #26 from ckan/new-changes-(29-Oct-2021)

Browse Source 
New changes (29 Oct 2021)
This commit is contained in:
Brett 2021-10-29 16:26:32 +02:00 committed by GitHub
commit 0882a6ed7e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
52 changed files with 2038 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
.env
View File

@ -1,4 +1,10 @@
# CKAN databases # CKAN
CKAN_VERSION=2.9.3
CKAN_SITE_URL=http://localhost:5000
CKAN_SITE_ID=default
CKAN_PORT=5000
# Database
POSTGRES_USER=ckan POSTGRES_USER=ckan
POSTGRES_PASSWORD=ckan POSTGRES_PASSWORD=ckan
DATASTORE_READONLY_USER=datastore_ro DATASTORE_READONLY_USER=datastore_ro
@ -8,39 +14,21 @@ CKAN_SQLALCHEMY_URL=postgresql://ckan:ckan@db/ckan
CKAN_DATASTORE_WRITE_URL=postgresql://ckan:ckan@db/datastore CKAN_DATASTORE_WRITE_URL=postgresql://ckan:ckan@db/datastore
CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:datastore@db/datastore CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:datastore@db/datastore
# Test database connections # solr
TEST_CKAN_SQLALCHEMY_URL=postgres://ckan:ckan@db/ckan_test SOLR_VERSION=6.6.6
TEST_CKAN_DATASTORE_WRITE_URL=postgresql://ckan:ckan@db/datastore_test
TEST_CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:datastore@db/datastore_test
# CKAN core
CKAN_SITE_ID=default
CKAN_SITE_URL=http://ckan:5000
CKAN_PORT=5000
CKAN_SYSADMIN_NAME=ckan_admin
CKAN_SYSADMIN_PASSWORD=test1234
CKAN_SYSADMIN_EMAIL=your_email@example.com
CKAN_STORAGE_PATH=/var/lib/ckan
CKAN_SMTP_SERVER=smtp.corporateict.domain:25
CKAN_SMTP_STARTTLS=True
CKAN_SMTP_USER=user
CKAN_SMTP_PASSWORD=pass
CKAN_SMTP_MAIL_FROM=ckan@localhost
TZ=UTC
# Other services
CKAN_SOLR_URL=http://solr:8983/solr/ckan CKAN_SOLR_URL=http://solr:8983/solr/ckan
CKAN_REDIS_URL=redis://redis:6379/1
CKAN_DATAPUSHER_URL=http://datapusher:8801 # Datapusher
DATAPUSHER_VERSION=0.0.17
CKAN_DATAPUSHER_URL=http://datapusher:8800
CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://ckan:5000 CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://ckan:5000
DATAPUSHER_REWRITE_RESOURCES=True
DATAPUSHER_REWRITE_URL=http://ckan:5000
# test connections # Redis
TEST_CKAN_SOLR_URL=http://solr:8983/solr/ckan REDIS_VERSION=6.0.7
TEST_CKAN_REDIS_URL=redis://redis:6379/1 CKAN_REDIS_URL=redis://redis:6379/1
# Extensions CKAN_SYSADMIN_NAME=admin
CKAN__PLUGINS=envvars image_view text_view recline_view datastore datapusher CKAN_SYSADMIN_PASSWORD=test1234
CKAN__HARVEST__MQ__TYPE=redis CKAN_SYSADMIN_EMAIL=admin@ckan.org
CKAN__HARVEST__MQ__HOSTNAME=redis
CKAN__HARVEST__MQ__PORT=6379
CKAN__HARVEST__MQ__REDIS_DB=1

0
.gitkeep Normal file
View File

35
Dockerfile Normal file
View File

@ -0,0 +1,35 @@
FROM solr:6.6.6
# Enviroment variables
ENV SOLR_CORE ckan
# Build Arguments
ARG CKAN_VERSION
ARG SOLR_VERSION
# root user for initial config
USER root
# Create directories
RUN mkdir -p /opt/solr/server/solr/${SOLR_CORE}/conf && \
mkdir -p /opt/solr/server/solr/${SOLR_CORE}/data && \
mkdir -p /opt/solr/server/solr/${SOLR_CORE}/data/index
# Add files
COPY ./solr/solrconfig-${CKAN_VERSION}.xml /opt/solr/server/solr/${SOLR_CORE}/conf/solrconfig.xml
ADD https://raw.githubusercontent.com/ckan/ckan/ckan-${CKAN_VERSION}/ckan/config/solr/schema.xml \
https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/${SOLR_VERSION}/solr/server/solr/configsets/basic_configs/conf/currency.xml \
https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/${SOLR_VERSION}/solr/server/solr/configsets/basic_configs/conf/synonyms.txt \
https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/${SOLR_VERSION}/solr/server/solr/configsets/basic_configs/conf/stopwords.txt \
https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/${SOLR_VERSION}/solr/server/solr/configsets/basic_configs/conf/protwords.txt \
https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/${SOLR_VERSION}/solr/server/solr/configsets/data_driven_schema_configs/conf/elevate.xml \
/opt/solr/server/solr/${SOLR_CORE}/conf/
# Create core.properties
RUN echo name=${SOLR_CORE} > /opt/solr/server/solr/${SOLR_CORE}/core.properties
# Giving ownership to Solr
RUN chown -R ${SOLR_USER}:${SOLR_USER} /opt/solr/server/solr/${SOLR_CORE}
# non-root user for runtime
USER ${SOLR_USER}:${SOLR_USER}

37
Dockerfile.dev Normal file
View File

@ -0,0 +1,37 @@
FROM ckan/ckan-dev:testing-only.2.9
LABEL maintainer="brett@kowh.ai"
# Set up environment variables
ENV APP_DIR=/srv/app
ENV TZ=UTC
RUN echo ${TZ} > /etc/timezone
# Make sure both files are not exactly the same
RUN if ! [ /usr/share/zoneinfo/${TZ} -ef /etc/localtime ]; then \
cp /usr/share/zoneinfo/${TZ} /etc/localtime ;\
fi ;
# Install any extensions needed by your CKAN instance
# (Make sure to add the plugins to CKAN__PLUGINS in the .env file)
# For instance:
#RUN pip install -e git+https://github.com/ckan/ckanext-pages.git#egg=ckanext-pages && \
# pip install -e git+https://github.com/ckan/ckanext-dcat.git@v0.0.6#egg=ckanext-dcat && \
# pip install -r https://raw.githubusercontent.com/ckan/ckanext-dcat/v0.0.6/requirements.txt
# Clone the extension(s) your are writing for your own project in the `src` folder
# to get them mounted in this image at runtime
# Apply any patches needed to CKAN core or any of the built extensions (not the
# runtime mounted ones)
# See https://github.com/okfn/docker-ckan#applying-patches
COPY patches ${APP_DIR}/patches
RUN for d in ${APP_DIR}/patches/*; do \
if [ -d $d ]; then \
for f in `ls $d/*.patch | sort -g`; do \
cd $SRC_DIR/`basename "$d"` && echo "$0: Applying patch $f to $SRC_DIR/`basename $d`"; patch -p1 < "$f" ; \
done ; \
fi ; \
done

12
README Executable file
View File

@ -0,0 +1,12 @@
Build ckan-base image (2.9)
cd /path/to/ckan-docker/ckan/ckan-base
docker build -t ckan/ckan-base:testing-only.2.9 -f 2.9/Dockerfile .
Login to Dockerhub
docker login --username=kowhai
(will then ask for a password)
Push image to (ckan) Dockerhub
docker push ckan/ckan-base:testing-only.2.9
Do the same with 2.8 and 2.7

101
ckan-base/2.7/Dockerfile Executable file
View File

@ -0,0 +1,101 @@
FROM alpine:3.13
# Internals, you probably don't need to change these
ENV APP_DIR=/srv/app
ENV SRC_DIR=/srv/app/src
ENV CKAN_INI=${APP_DIR}/production.ini
ENV PIP_SRC=${SRC_DIR}
ENV CKAN_STORAGE_PATH=/var/lib/ckan
ENV GIT_URL=https://github.com/ckan/ckan.git
# CKAN version to build
ENV GIT_BRANCH=ckan-2.7.11
# Customize these on the .env file if needed
ENV CKAN_SITE_URL=http://localhost:5000
ENV CKAN__PLUGINS image_view text_view recline_view datastore datapusher envvars
WORKDIR ${APP_DIR}
# Install necessary packages to run CKAN
RUN apk add --no-cache tzdata \
git \
gettext \
postgresql-client \
python \
apache2-utils \
libxml2 \
libxslt \
musl-dev \
uwsgi \
uwsgi-http \
uwsgi-corerouter \
uwsgi-python \
py2-gevent \
uwsgi-gevent \
libmagic \
curl \
sudo && \
# Packages to build CKAN requirements and plugins
apk add --no-cache --virtual .build-deps \
postgresql-dev \
gcc \
make \
g++ \
autoconf \
automake \
libtool \
python-dev \
libxml2-dev \
libxslt-dev \
linux-headers
# Create SRC_DIR
RUN mkdir -p ${SRC_DIR}
# Install pip
RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/pip/2.7/get-pip.py && \
python ${SRC_DIR}/get-pip.py 'pip==20.3.3'
# Install pip, supervisord and uwsgi
RUN pip install supervisor && \
mkdir /etc/supervisord.d && \
#pip wheel --wheel-dir=/wheels uwsgi gevent && \
rm -rf ${SRC_DIR}/get-pip.py
COPY 2.7/setup/supervisord.conf /etc
# Install CKAN
RUN pip install -e git+${GIT_URL}@${GIT_BRANCH}#egg=ckan && \
cd ${SRC_DIR}/ckan && \
cp who.ini ${APP_DIR} && \
# Workaround to solve https://github.com/psycopg/psycopg2/issues/594 in Alpine 3.7
sed -i -e "s/psycopg2==2.4.5/psycopg2==2.7.3.2/g" requirements.txt && \
pip install --no-binary :all: -r requirements.txt && \
# Install CKAN envvars to support loading config from environment variables
pip install -e git+https://github.com/okfn/ckanext-envvars.git#egg=ckanext-envvars && \
# Create and update CKAN config
paster --plugin=ckan make-config ckan ${CKAN_INI} && \
paster --plugin=ckan config-tool ${CKAN_INI} "ckan.plugins = ${CKAN__PLUGINS}" && \
paster --plugin=ckan config-tool ${CKAN_INI} "ckan.site_url = ${CKAN__SITE_URL}"
# Create a local user and group to run the app
RUN addgroup -g 92 -S ckan && \
adduser -u 92 -h /srv/app -H -D -S -G ckan ckan
# Create local storage folder
RUN mkdir -p $CKAN_STORAGE_PATH && \
chown -R ckan:ckan $CKAN_STORAGE_PATH
COPY 2.7/setup ${APP_DIR}
COPY 2.7/setup/supervisor.worker.conf /etc/supervisord.d/worker.conf
COPY 2.7/setup/uwsgi.conf /srv/app/uwsgi.conf
# Create entrypoint directory for children image scripts
ONBUILD RUN mkdir /docker-entrypoint.d
RUN chown ckan -R /srv/app
EXPOSE 5000
HEALTHCHECK --interval=10s --timeout=5s --retries=5 CMD curl --fail http://localhost:5000/api/3/action/status_show || exit 1
CMD ["/srv/app/start_ckan.sh"]

197
ckan-base/2.7/setup/prerun.py Executable file
View File

@ -0,0 +1,197 @@
import os
import sys
import subprocess
import psycopg2
import urllib2
import time
import re
ckan_ini = os.environ.get('CKAN_INI', '/srv/app/production.ini')
RETRY = 5
def update_plugins():
plugins = os.environ.get('CKAN__PLUGINS', '')
print('[prerun] Setting the following plugins in {}:'.format(ckan_ini))
print(plugins)
cmd = ['paster', '--plugin=ckan', 'config-tool',
ckan_ini, 'ckan.plugins = {}'.format(plugins)]
subprocess.check_output(cmd, stderr=subprocess.STDOUT)
print '[prerun] Plugins set.'
def check_main_db_connection(retry=None):
conn_str = os.environ.get('CKAN_SQLALCHEMY_URL')
if not conn_str:
print '[prerun] CKAN_SQLALCHEMY_URL not defined, not checking db'
return check_db_connection(conn_str, retry)
def check_datastore_db_connection(retry=None):
conn_str = os.environ.get('CKAN_DATASTORE_WRITE_URL')
if not conn_str:
print '[prerun] CKAN_DATASTORE_WRITE_URL not defined, not checking db'
return check_db_connection(conn_str, retry)
def check_db_connection(conn_str, retry=None):
if retry is None:
retry = RETRY
elif retry == 0:
print '[prerun] Giving up after 5 tries...'
sys.exit(1)
try:
connection = psycopg2.connect(conn_str)
except psycopg2.Error as e:
print str(e)
print '[prerun] Unable to connect to the database, waiting...'
time.sleep(10)
check_db_connection(conn_str, retry=retry - 1)
else:
connection.close()
def check_solr_connection(retry=None):
if retry is None:
retry = RETRY
elif retry == 0:
print '[prerun] Giving up after 5 tries...'
sys.exit(1)
url = os.environ.get('CKAN_SOLR_URL', '')
search_url = '{url}/select/?q=*&wt=json'.format(url=url)
try:
connection = urllib2.urlopen(search_url)
except urllib2.URLError as e:
print str(e)
print '[prerun] Unable to connect to solr, waiting...'
time.sleep(10)
check_solr_connection(retry=retry - 1)
else:
eval(connection.read())
def init_db():
db_command = ['paster', '--plugin=ckan', 'db',
'init', '-c', ckan_ini]
print '[prerun] Initializing or upgrading db - start'
try:
subprocess.check_output(db_command, stderr=subprocess.STDOUT)
print '[prerun] Initializing or upgrading db - end'
except subprocess.CalledProcessError, e:
if 'OperationalError' in e.output:
print e.output
print '[prerun] Database not ready, waiting a bit before exit...'
time.sleep(5)
sys.exit(1)
else:
print e.output
raise e
def init_datastore_db():
conn_str = os.environ.get('CKAN_DATASTORE_WRITE_URL')
if not conn_str:
print '[prerun] Skipping datastore initialization'
return
datastore_perms_command = ['paster', '--plugin=ckan', 'datastore',
'set-permissions', '-c', ckan_ini]
connection = psycopg2.connect(conn_str)
cursor = connection.cursor()
print '[prerun] Initializing datastore db - start'
try:
datastore_perms = subprocess.Popen(
datastore_perms_command,
stdout=subprocess.PIPE)
perms_sql = datastore_perms.stdout.read()
# Remove internal pg command as psycopg2 does not like it
perms_sql = re.sub('\\\\connect \"(.*)\"', '', perms_sql)
cursor.execute(perms_sql)
for notice in connection.notices:
print notice
connection.commit()
print '[prerun] Initializing datastore db - end'
print datastore_perms.stdout.read()
except psycopg2.Error as e:
print '[prerun] Could not initialize datastore'
print str(e)
except subprocess.CalledProcessError, e:
if 'OperationalError' in e.output:
print e.output
print '[prerun] Database not ready, waiting a bit before exit...'
time.sleep(5)
sys.exit(1)
else:
print e.output
raise e
finally:
cursor.close()
connection.close()
def create_sysadmin():
name = os.environ.get('CKAN_SYSADMIN_NAME')
password = os.environ.get('CKAN_SYSADMIN_PASSWORD')
email = os.environ.get('CKAN_SYSADMIN_EMAIL')
if name and password and email:
# Check if user exists
command = ['paster', '--plugin=ckan', 'user', name, '-c', ckan_ini]
out = subprocess.check_output(command)
if 'User:None' not in re.sub(r'\s', '', out):
print '[prerun] Sysadmin user exists, skipping creation'
return
# Create user
command = ['paster', '--plugin=ckan', 'user', 'add',
name,
'password=' + password,
'email=' + email,
'-c', ckan_ini]
subprocess.call(command)
print '[prerun] Created user {0}'.format(name)
# Make it sysadmin
command = ['paster', '--plugin=ckan', 'sysadmin', 'add',
name,
'-c', ckan_ini]
subprocess.call(command)
print '[prerun] Made user {0} a sysadmin'.format(name)
if __name__ == '__main__':
maintenance = os.environ.get('MAINTENANCE_MODE', '').lower() == 'true'
if maintenance:
print '[prerun] Maintenance mode, skipping setup...'
else:
check_main_db_connection()
init_db()
update_plugins()
check_datastore_db_connection()
init_datastore_db()
check_solr_connection()
create_sysadmin()

47
ckan-base/2.7/setup/start_ckan.sh Executable file
View File

@ -0,0 +1,47 @@
#!/bin/bash
# Run the prerun script to init CKAN and create the default admin user
sudo -u ckan -EH python prerun.py
# Run any startup scripts provided by images extending this one
if [[ -d "/docker-entrypoint.d" ]]
then
for f in /docker-entrypoint.d/*; do
case "$f" in
*.sh) echo "$0: Running init file $f"; . "$f" ;;
*.py) echo "$0: Running init file $f"; python "$f"; echo ;;
*) echo "$0: Ignoring $f (not an sh or py file)" ;;
esac
echo
done
fi
# Set the common uwsgi options
UWSGI_OPTS="--plugins http,python,gevent --socket /tmp/uwsgi.sock --uid 92 --gid 92 --http :5000 --master --enable-threads --paste config:/srv/app/production.ini --paste-logger --lazy-apps --gevent 2000 -p 2 -L"
# Check whether http basic auth password protection is enabled and enable basicauth routing on uwsgi respecfully
if [ $? -eq 0 ]
then
if [ "$PASSWORD_PROTECT" = true ]
then
if [ "$HTPASSWD_USER" ] || [ "$HTPASSWD_PASSWORD" ]
then
# Generate htpasswd file for basicauth
htpasswd -d -b -c /srv/app/.htpasswd $HTPASSWD_USER $HTPASSWD_PASSWORD
# Start supervisord
supervisord --configuration /etc/supervisord.conf &
# Start uwsgi with basicauth
sudo -u ckan -EH uwsgi --ini /srv/app/uwsgi.conf --pcre-jit $UWSGI_OPTS
else
echo "Missing HTPASSWD_USER or HTPASSWD_PASSWORD environment variables. Exiting..."
exit 1
fi
else
# Start supervisord
supervisord --configuration /etc/supervisord.conf &
# Start uwsgi
sudo -u ckan -EH uwsgi $UWSGI_OPTS
fi
else
echo "[prerun] failed...not starting CKAN."
fi

13
ckan-base/2.7/setup/supervisor.worker.conf Executable file
View File

@ -0,0 +1,13 @@
[program:ckan-worker]
command=paster --plugin=ckan jobs worker -c /srv/app/production.ini
priority=501
autostart=true
autorestart=true
redirect_stderr=true
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stdout
stderr_logfile_maxbytes=0
user=ckan
environment=HOME="/srv/app",USER="ckan"

23
ckan-base/2.7/setup/supervisord.conf Executable file
View File

@ -0,0 +1,23 @@
[unix_http_server]
file = /tmp/supervisor.sock
chmod = 0777
chown = nobody:nogroup
[supervisord]
logfile = /tmp/supervisord.log
logfile_maxbytes = 50MB
logfile_backups=10
loglevel = info
pidfile = /tmp/supervisord.pid
nodaemon = true
umask = 022
identifier = supervisor
[supervisorctl]
serverurl = unix:///tmp/supervisor.sock
[rpcinterface:supervisor]
supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
[include]
files = /etc/supervisord.d/*.conf

2
ckan-base/2.7/setup/uwsgi.conf Executable file
View File

@ -0,0 +1,2 @@
[uwsgi]
route = ^(?!/api).*$ basicauth:Restricted,/srv/app/.htpasswd

99
ckan-base/2.8/Dockerfile Executable file
View File

@ -0,0 +1,99 @@
FROM alpine:3.13
# Internals, you probably don't need to change these
ENV APP_DIR=/srv/app
ENV SRC_DIR=/srv/app/src
ENV CKAN_INI=${APP_DIR}/production.ini
ENV PIP_SRC=${SRC_DIR}
ENV CKAN_STORAGE_PATH=/var/lib/ckan
ENV GIT_URL=https://github.com/ckan/ckan.git
# CKAN version to build
ENV GIT_BRANCH=ckan-2.8.8
# Customize these on the .env file if needed
ENV CKAN_SITE_URL=http://localhost:5000
ENV CKAN__PLUGINS image_view text_view recline_view datastore datapusher envvars
WORKDIR ${APP_DIR}
# Install necessary packages to run CKAN
RUN apk add --no-cache tzdata \
git \
gettext \
postgresql-client \
python \
apache2-utils \
libxml2 \
libxslt \
musl-dev \
uwsgi \
uwsgi-http \
uwsgi-corerouter \
uwsgi-python \
py2-gevent \
uwsgi-gevent \
libmagic \
curl \
sudo && \
# Packages to build CKAN requirements and plugins
apk add --no-cache --virtual .build-deps \
postgresql-dev \
gcc \
make \
g++ \
autoconf \
automake \
libtool \
python-dev \
libxml2-dev \
libxslt-dev \
linux-headers
# Create SRC_DIR
RUN mkdir -p ${SRC_DIR}
# Install pip
RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/pip/2.7/get-pip.py && \
python ${SRC_DIR}/get-pip.py 'pip==20.3.3'
# Install supervisord and uwsgi
RUN pip install supervisor && \
mkdir /etc/supervisord.d && \
#pip wheel --wheel-dir=/wheels uwsgi gevent && \
rm -rf ${SRC_DIR}/get-pip.py
COPY 2.8/setup/supervisord.conf /etc
# Install CKAN
RUN pip install -e git+${GIT_URL}@${GIT_BRANCH}#egg=ckan && \
cd ${SRC_DIR}/ckan && \
cp who.ini ${APP_DIR} && \
pip install --no-binary :all: -r requirements.txt && \
# Install CKAN envvars to support loading config from environment variables
pip install -e git+https://github.com/okfn/ckanext-envvars.git#egg=ckanext-envvars && \
# Create and update CKAN config
paster --plugin=ckan make-config ckan ${CKAN_INI} && \
paster --plugin=ckan config-tool ${CKAN_INI} "ckan.plugins = ${CKAN__PLUGINS}" && \
paster --plugin=ckan config-tool ${CKAN_INI} "ckan.site_url = ${CKAN__SITE_URL}"
# Create a local user and group to run the app
RUN addgroup -g 92 -S ckan && \
adduser -u 92 -h /srv/app -H -D -S -G ckan ckan
# Create local storage folder
RUN mkdir -p $CKAN_STORAGE_PATH && \
chown -R ckan:ckan $CKAN_STORAGE_PATH
COPY 2.8/setup ${APP_DIR}
COPY 2.8/setup/supervisor.worker.conf /etc/supervisord.d/worker.conf
COPY 2.8/setup/uwsgi.conf /srv/app/uwsgi.conf
# Create entrypoint directory for children image scripts
ONBUILD RUN mkdir /docker-entrypoint.d
RUN chown ckan -R /srv/app
EXPOSE 5000
HEALTHCHECK --interval=10s --timeout=5s --retries=5 CMD curl --fail http://localhost:5000/api/3/action/status_show || exit 1
CMD ["/srv/app/start_ckan.sh"]

197
ckan-base/2.8/setup/prerun.py Executable file
View File

@ -0,0 +1,197 @@
import os
import sys
import subprocess
import psycopg2
import urllib2
import time
import re
ckan_ini = os.environ.get('CKAN_INI', '/srv/app/production.ini')
RETRY = 5
def update_plugins():
plugins = os.environ.get('CKAN__PLUGINS', '')
print('[prerun] Setting the following plugins in {}:'.format(ckan_ini))
print(plugins)
cmd = ['paster', '--plugin=ckan', 'config-tool',
ckan_ini, 'ckan.plugins = {}'.format(plugins)]
subprocess.check_output(cmd, stderr=subprocess.STDOUT)
print '[prerun] Plugins set.'
def check_main_db_connection(retry=None):
conn_str = os.environ.get('CKAN_SQLALCHEMY_URL')
if not conn_str:
print '[prerun] CKAN_SQLALCHEMY_URL not defined, not checking db'
return check_db_connection(conn_str, retry)
def check_datastore_db_connection(retry=None):
conn_str = os.environ.get('CKAN_DATASTORE_WRITE_URL')
if not conn_str:
print '[prerun] CKAN_DATASTORE_WRITE_URL not defined, not checking db'
return check_db_connection(conn_str, retry)
def check_db_connection(conn_str, retry=None):
if retry is None:
retry = RETRY
elif retry == 0:
print '[prerun] Giving up after 5 tries...'
sys.exit(1)
try:
connection = psycopg2.connect(conn_str)
except psycopg2.Error as e:
print str(e)
print '[prerun] Unable to connect to the database, waiting...'
time.sleep(10)
check_db_connection(conn_str, retry=retry - 1)
else:
connection.close()
def check_solr_connection(retry=None):
if retry is None:
retry = RETRY
elif retry == 0:
print '[prerun] Giving up after 5 tries...'
sys.exit(1)
url = os.environ.get('CKAN_SOLR_URL', '')
search_url = '{url}/select/?q=*&wt=json'.format(url=url)
try:
connection = urllib2.urlopen(search_url)
except urllib2.URLError as e:
print str(e)
print '[prerun] Unable to connect to solr, waiting...'
time.sleep(10)
check_solr_connection(retry=retry - 1)
else:
eval(connection.read())
def init_db():
db_command = ['paster', '--plugin=ckan', 'db',
'init', '-c', ckan_ini]
print '[prerun] Initializing or upgrading db - start'
try:
subprocess.check_output(db_command, stderr=subprocess.STDOUT)
print '[prerun] Initializing or upgrading db - end'
except subprocess.CalledProcessError, e:
if 'OperationalError' in e.output:
print e.output
print '[prerun] Database not ready, waiting a bit before exit...'
time.sleep(5)
sys.exit(1)
else:
print e.output
raise e
def init_datastore_db():
conn_str = os.environ.get('CKAN_DATASTORE_WRITE_URL')
if not conn_str:
print '[prerun] Skipping datastore initialization'
return
datastore_perms_command = ['paster', '--plugin=ckan', 'datastore',
'set-permissions', '-c', ckan_ini]
connection = psycopg2.connect(conn_str)
cursor = connection.cursor()
print '[prerun] Initializing datastore db - start'
try:
datastore_perms = subprocess.Popen(
datastore_perms_command,
stdout=subprocess.PIPE)
perms_sql = datastore_perms.stdout.read()
# Remove internal pg command as psycopg2 does not like it
perms_sql = re.sub('\\\\connect \"(.*)\"', '', perms_sql)
cursor.execute(perms_sql)
for notice in connection.notices:
print notice
connection.commit()
print '[prerun] Initializing datastore db - end'
print datastore_perms.stdout.read()
except psycopg2.Error as e:
print '[prerun] Could not initialize datastore'
print str(e)
except subprocess.CalledProcessError, e:
if 'OperationalError' in e.output:
print e.output
print '[prerun] Database not ready, waiting a bit before exit...'
time.sleep(5)
sys.exit(1)
else:
print e.output
raise e
finally:
cursor.close()
connection.close()
def create_sysadmin():
name = os.environ.get('CKAN_SYSADMIN_NAME')
password = os.environ.get('CKAN_SYSADMIN_PASSWORD')
email = os.environ.get('CKAN_SYSADMIN_EMAIL')
if name and password and email:
# Check if user exists
command = ['paster', '--plugin=ckan', 'user', name, '-c', ckan_ini]
out = subprocess.check_output(command)
if 'User:None' not in re.sub(r'\s', '', out):
print '[prerun] Sysadmin user exists, skipping creation'
return
# Create user
command = ['paster', '--plugin=ckan', 'user', 'add',
name,
'password=' + password,
'email=' + email,
'-c', ckan_ini]
subprocess.call(command)
print '[prerun] Created user {0}'.format(name)
# Make it sysadmin
command = ['paster', '--plugin=ckan', 'sysadmin', 'add',
name,
'-c', ckan_ini]
subprocess.call(command)
print '[prerun] Made user {0} a sysadmin'.format(name)
if __name__ == '__main__':
maintenance = os.environ.get('MAINTENANCE_MODE', '').lower() == 'true'
if maintenance:
print '[prerun] Maintenance mode, skipping setup...'
else:
check_main_db_connection()
init_db()
update_plugins()
check_datastore_db_connection()
init_datastore_db()
check_solr_connection()
create_sysadmin()

47
ckan-base/2.8/setup/start_ckan.sh Executable file
View File

@ -0,0 +1,47 @@
#!/bin/bash
# Run the prerun script to init CKAN and create the default admin user
sudo -u ckan -EH python prerun.py
# Run any startup scripts provided by images extending this one
if [[ -d "/docker-entrypoint.d" ]]
then
for f in /docker-entrypoint.d/*; do
case "$f" in
*.sh) echo "$0: Running init file $f"; . "$f" ;;
*.py) echo "$0: Running init file $f"; python "$f"; echo ;;
*) echo "$0: Ignoring $f (not an sh or py file)" ;;
esac
echo
done
fi
# Set the common uwsgi options
UWSGI_OPTS="--plugins http,python,gevent --socket /tmp/uwsgi.sock --uid 92 --gid 92 --http :5000 --master --enable-threads --paste config:/srv/app/production.ini --paste-logger --lazy-apps --gevent 2000 -p 2 -L"
# Check whether http basic auth password protection is enabled and enable basicauth routing on uwsgi respecfully
if [ $? -eq 0 ]
then
if [ "$PASSWORD_PROTECT" = true ]
then
if [ "$HTPASSWD_USER" ] || [ "$HTPASSWD_PASSWORD" ]
then
# Generate htpasswd file for basicauth
htpasswd -d -b -c /srv/app/.htpasswd $HTPASSWD_USER $HTPASSWD_PASSWORD
# Start supervisord
supervisord --configuration /etc/supervisord.conf &
# Start uwsgi with basicauth
sudo -u ckan -EH uwsgi --ini /srv/app/uwsgi.conf --pcre-jit $UWSGI_OPTS
else
echo "Missing HTPASSWD_USER or HTPASSWD_PASSWORD environment variables. Exiting..."
exit 1
fi
else
# Start supervisord
supervisord --configuration /etc/supervisord.conf &
# Start uwsgi
sudo -u ckan -EH uwsgi $UWSGI_OPTS
fi
else
echo "[prerun] failed...not starting CKAN."
fi

13
ckan-base/2.8/setup/supervisor.worker.conf Executable file
View File

@ -0,0 +1,13 @@
[program:ckan-worker]
command=paster --plugin=ckan jobs worker -c /srv/app/production.ini
priority=501
autostart=true
autorestart=true
redirect_stderr=true
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stdout
stderr_logfile_maxbytes=0
user=ckan
environment=HOME="/srv/app",USER="ckan"

23
ckan-base/2.8/setup/supervisord.conf Executable file
View File

@ -0,0 +1,23 @@
[unix_http_server]
file = /tmp/supervisor.sock
chmod = 0777
chown = nobody:nogroup
[supervisord]
logfile = /tmp/supervisord.log
logfile_maxbytes = 50MB
logfile_backups=10
loglevel = info
pidfile = /tmp/supervisord.pid
nodaemon = true
umask = 022
identifier = supervisor
[supervisorctl]
serverurl = unix:///tmp/supervisor.sock
[rpcinterface:supervisor]
supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
[include]
files = /etc/supervisord.d/*.conf

2
ckan-base/2.8/setup/uwsgi.conf Executable file
View File

@ -0,0 +1,2 @@
[uwsgi]
route = ^(?!/api).*$ basicauth:Restricted,/srv/app/.htpasswd

109
ckan-base/2.9/Dockerfile Executable file
View File

@ -0,0 +1,109 @@
FROM alpine:3.13
# Internal environment variables
ENV APP_DIR=/srv/app
ENV SRC_DIR=/srv/app/src
ENV CKAN_INI=${APP_DIR}/ckan.ini
ENV PIP_SRC=${SRC_DIR}
ENV CKAN_STORAGE_PATH=/var/lib/ckan
ENV GIT_URL=https://github.com/ckan/ckan.git
# CKAN version to build
ENV GIT_BRANCH=ckan-2.9.4
# Customize these on the .env file if needed
ENV CKAN_SITE_URL=http://localhost:5000
ENV CKAN__PLUGINS image_view text_view recline_view datastore datapusher envvars
WORKDIR ${APP_DIR}
# Install necessary packages to run CKAN
RUN apk add --no-cache tzdata \
git \
gettext \
postgresql-client \
python3 \
apache2-utils \
libxml2 \
libxslt \
musl-dev \
uwsgi-http \
uwsgi-corerouter \
uwsgi-python3 \
py3-gevent \
uwsgi-gevent \
libmagic \
curl \
sudo && \
# Packages to build CKAN requirements and plugins
apk add --no-cache --virtual .build-deps \
postgresql-dev \
gcc \
make \
g++ \
autoconf \
automake \
libtool \
python3-dev \
py3-virtualenv \
libxml2-dev \
libxslt-dev \
linux-headers
# Create src directory (SRC_DIR)
RUN mkdir -p ${SRC_DIR}
# Install pip
RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/get-pip.py && \
python3 ${SRC_DIR}/get-pip.py
# Set up Python3 virtual environment
RUN cd ${APP_DIR} && \
python3 -m venv ${APP_DIR} && \
source ${APP_DIR}/bin/activate
# Virtual environment binaries/scripts to be used first
ENV PATH=${APP_DIR}/bin:${PATH}
# Install CKAN code
RUN pip3 install -e git+${GIT_URL}@${GIT_BRANCH}#egg=ckan
# Install uwsgi
RUN pip3 install uwsgi
# Complete CKAN install
RUN cd ${SRC_DIR}/ckan && \
cp who.ini ${APP_DIR} && \
pip3 install -r requirement-setuptools.txt && \
pip3 install --no-binary :all: -r requirements.txt && \
# Install CKAN envvars to support loading config from environment variables
pip3 install -e git+https://github.com/okfn/ckanext-envvars.git#egg=ckanext-envvars
# Create and update CKAN config
RUN ckan generate config ${CKAN_INI}
# Install and configure supervisor
RUN pip3 install supervisor && \
mkdir /etc/supervisord.d
# Copy all setup files
COPY 2.9/setup ${APP_DIR}
COPY 2.9/setup/supervisor.worker.conf /etc/supervisord.d/worker.conf
COPY 2.9/setup/supervisord.conf /etc/supervisord.conf
# Create a local user and group to run the app
RUN addgroup -g 92 -S ckan && \
adduser -u 92 -h /srv/app -H -D -S -G ckan ckan
# Create local storage folder
RUN mkdir -p $CKAN_STORAGE_PATH && \
chown -R ckan:ckan $CKAN_STORAGE_PATH
# Create entrypoint directory for children image scripts
ONBUILD RUN mkdir /docker-entrypoint.d
RUN chown ckan -R /srv/app
EXPOSE 5000
HEALTHCHECK --interval=10s --timeout=5s --retries=5 CMD curl --fail http://localhost:5000/api/3/action/status_show || exit 1
CMD ["/srv/app/start_ckan.sh"]

15
ckan-base/2.9/setup/ckan-uwsgi.ini Normal file
View File

@ -0,0 +1,15 @@
[uwsgi]
http-socket = :5000
uid = ckan
guid = ckan
plugins = python3
wsgi-file = /srv/app/wsgi.py
virtualenv = /srv/app
module = wsgi:application
master = true
processes = 5
pidfile = /tmp/%n.pid
harakiri = 50
max-requests = 5000
vacuum = true
callable = application

194
ckan-base/2.9/setup/prerun.py Executable file
View File

@ -0,0 +1,194 @@
import os
import sys
import subprocess
import psycopg2
import urllib3
import time
import re
ckan_ini = os.environ.get('CKAN_INI', '/srv/app/ckan.ini')
RETRY = 5
def update_plugins():
plugins = os.environ.get('CKAN__PLUGINS', '')
print('[prerun] Setting the following plugins in {}:'.format(ckan_ini))
print(plugins)
cmd = ['ckan', 'config-tool', ckan_ini,
'ckan.plugins = {}'.format(plugins)]
subprocess.check_output(cmd, stderr=subprocess.STDOUT)
print('[prerun] Plugins set.')
def check_main_db_connection(retry=None):
conn_str = os.environ.get('CKAN_SQLALCHEMY_URL')
if not conn_str:
print('[prerun] CKAN_SQLALCHEMY_URL not defined, not checking db')
return check_db_connection(conn_str, retry)
def check_datastore_db_connection(retry=None):
conn_str = os.environ.get('CKAN_DATASTORE_WRITE_URL')
if not conn_str:
print('[prerun] CKAN_DATASTORE_WRITE_URL not defined, not checking db')
return check_db_connection(conn_str, retry)
def check_db_connection(conn_str, retry=None):
if retry is None:
retry = RETRY
elif retry == 0:
print('[prerun] Giving up after 5 tries...')
sys.exit(1)
try:
connection = psycopg2.connect(conn_str)
except psycopg2.Error as e:
print(str(e))
print('[prerun] Unable to connect to the database, waiting...')
time.sleep(10)
check_db_connection(conn_str, retry=retry - 1)
else:
connection.close()
def check_solr_connection(retry=None):
if retry is None:
retry = RETRY
elif retry == 0:
print('[prerun] Giving up after 5 tries...')
sys.exit(1)
url = os.environ.get('CKAN_SOLR_URL', '')
search_url = '{url}/select/?q=*&wt=json'.format(url=url)
http = urllib3.PoolManager()
try:
r = http.request('GET', search_url)
except urllib3.exceptions.ConnectionError as e:
print(str(e))
print('[prerun] Unable to connect to solr, waiting...')
time.sleep(10)
check_solr_connection(retry=retry - 1)
else:
print('[prerun] Connection Status from SOLR is ', (r.status))
def init_db():
db_command = ['ckan', '-c', ckan_ini,
'db', 'init']
print('[prerun] Initializing or upgrading db - start')
try:
subprocess.check_output(db_command, stderr=subprocess.STDOUT)
print('[prerun] Initializing or upgrading db - end')
except subprocess.CalledProcessError as e:
if 'OperationalError' in e.output:
print(e.output)
print('[prerun] Database not ready, waiting a bit before exit...')
time.sleep(5)
sys.exit(1)
else:
print(e.output)
raise e
def init_datastore_db():
conn_str = os.environ.get('CKAN_DATASTORE_WRITE_URL')
if not conn_str:
print('[prerun] Skipping datastore initialization')
return
datastore_perms_command = ['ckan', '-c', ckan_ini,
'datastore', 'set-permissions']
connection = psycopg2.connect(conn_str)
cursor = connection.cursor()
print('[prerun] Initializing datastore db - start')
try:
datastore_perms = subprocess.Popen(
datastore_perms_command,
stdout=subprocess.PIPE)
perms_sql = datastore_perms.stdout.read().decode('utf-8')
# Remove internal pg command as psycopg2 does not like it
perms_sql = re.sub('\\\\connect \"(.*)\"', '', perms_sql)
cursor.execute(perms_sql)
for notice in connection.notices:
print(notice)
connection.commit()
print('[prerun] Initializing datastore db - end')
print(datastore_perms.stdout.read().decode('utf-8'))
except psycopg2.Error as e:
print('[prerun] Could not initialize datastore')
print(str(e))
except subprocess.CalledProcessError as e:
if 'OperationalError' in e.output:
print(e.output)
print('[prerun] Database not ready, waiting a bit before exit...')
time.sleep(5)
sys.exit(1)
else:
print(e.output)
raise e
finally:
cursor.close()
connection.close()
def create_sysadmin():
name = os.environ.get('CKAN_SYSADMIN_NAME')
password = os.environ.get('CKAN_SYSADMIN_PASSWORD')
email = os.environ.get('CKAN_SYSADMIN_EMAIL')
if name and password and email:
# Check if user exists
command = ['ckan', '-c', ckan_ini, 'user', 'show', name,]
out = subprocess.check_output(command)
if 'User:None' not in re.sub(r'\s', '', out.decode()):
print('[prerun] Sysadmin user exists, skipping creation')
return
# Create user
command = ['ckan', '-c', ckan_ini, 'user', 'add',
name,
'password=' + password,
'email=' + email]
subprocess.call(command)
print('[prerun] Created user {0}'.format(name))
# Make it sysadmin
command = ['ckan', '-c', ckan_ini, 'sysadmin', 'add',
name]
subprocess.call(command)
print('[prerun] Made user {0} a sysadmin'.format(name))
if __name__ == '__main__':
maintenance = os.environ.get('MAINTENANCE_MODE', '').lower() == 'true'
if maintenance:
print('[prerun] Maintenance mode, skipping setup...')
else:
check_main_db_connection()
init_db()
update_plugins()
check_datastore_db_connection()
init_datastore_db()
check_solr_connection()
create_sysadmin()

46
ckan-base/2.9/setup/start_ckan.sh Executable file
View File

@ -0,0 +1,46 @@
#!/bin/bash
# Run the prerun script to init CKAN and create the default admin user
sudo -u ckan -EH python3 prerun.py
# Run any startup scripts provided by images extending this one
if [[ -d "/docker-entrypoint.d" ]]
then
for f in /docker-entrypoint.d/*; do
case "$f" in
*.sh) echo "$0: Running init file $f"; . "$f" ;;
*.py) echo "$0: Running init file $f"; python "$f"; echo ;;
*) echo "$0: Ignoring $f (not an sh or py file)" ;;
esac
echo
done
fi
# Check whether http basic auth password protection is enabled and enable basicauth routing on uwsgi respecfully
if [ $? -eq 0 ]
then
if [ "$PASSWORD_PROTECT" = true ]
then
if [ "$HTPASSWD_USER" ] || [ "$HTPASSWD_PASSWORD" ]
then
# Generate htpasswd file for basicauth
htpasswd -d -b -c /srv/app/.htpasswd $HTPASSWD_USER $HTPASSWD_PASSWORD
# Start supervisord
supervisord --configuration /etc/supervisord.conf &
# Start uwsgi with basicauth
sudo -u ckan -EH uwsgi -i ckan-uwsgi.ini
else
echo "Missing HTPASSWD_USER or HTPASSWD_PASSWORD environment variables. Exiting..."
exit 1
fi
else
# Start supervisord
supervisord --configuration /etc/supervisord.conf &
# Start uwsgi
sudo -u ckan -EH uwsgi -i ckan-uwsgi.ini
fi
else
echo "[prerun] failed...not starting CKAN."
fi

12
ckan-base/2.9/setup/supervisor.worker.conf Normal file
View File

@ -0,0 +1,12 @@
[program:ckan-worker]
command=ckan -c /srv/app/ckan.ini jobs worker
priority=501
autostart=true
autorestart=true
redirect_stderr=true
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stdout
stderr_logfile_maxbytes=0
user=ckan
environment=HOME="/srv/app",USER="ckan"

23
ckan-base/2.9/setup/supervisord.conf Normal file
View File

@ -0,0 +1,23 @@
[unix_http_server]
file = /tmp/supervisor.sock
chmod = 0777
chown = nobody:nogroup
[supervisord]
logfile = /tmp/supervisord.log
logfile_maxbytes = 50MB
logfile_backups=10
loglevel = info
pidfile = /tmp/supervisord.pid
nodaemon = true
umask = 022
identifier = supervisor
[supervisorctl]
serverurl = unix:///tmp/supervisor.sock
[rpcinterface:supervisor]
supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
[include]
files = /etc/supervisord.d/*.conf

9
ckan-base/2.9/setup/wsgi.py Normal file
View File

@ -0,0 +1,9 @@
import os
from ckan.config.middleware import make_app
from ckan.cli import CKANConfigLoader
from logging.config import fileConfig as loggingFileConfig
config_filepath = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'ckan.ini')
abspath = os.path.join(os.path.dirname(os.path.abspath(__file__)))
loggingFileConfig(config_filepath)
config = CKANConfigLoader(config_filepath).get_config()
application = make_app(config)

20
ckan-dev/2.7/Dockerfile Executable file
View File

@ -0,0 +1,20 @@
FROM ckan/ckan-base:testing-only.2.7
LABEL maintainer="brett@kowh.ai"
ENV APP_DIR=/srv/app
ENV SRC_EXTENSIONS_DIR=/srv/app/src_extensions
# Install packages needed by the dev requirements
RUN apk add --no-cache libffi-dev
# Install CKAN dev requirements
RUN pip install --no-binary :all: -r https://raw.githubusercontent.com/ckan/ckan/${GIT_BRANCH}/dev-requirements.txt
# Create folder for local extensions sources
RUN mkdir $SRC_EXTENSIONS_DIR
COPY setup/start_ckan_development.sh ${APP_DIR}
CMD ["/srv/app/start_ckan_development.sh"]

20
ckan-dev/2.8/Dockerfile Executable file
View File

@ -0,0 +1,20 @@
FROM ckan/ckan-base:testing-only.2.8
LABEL maintainer="brett@kowh.ai"
ENV APP_DIR=/srv/app
ENV SRC_EXTENSIONS_DIR=/srv/app/src_extensions
# Install packages needed by the dev requirements
RUN apk add --no-cache libffi-dev
# Install CKAN dev requirements
RUN pip install --no-binary :all: -r https://raw.githubusercontent.com/ckan/ckan/${GIT_BRANCH}/dev-requirements.txt
# Create folder for local extensions sources
RUN mkdir $SRC_EXTENSIONS_DIR
COPY setup/start_ckan_development.sh ${APP_DIR}
CMD ["/srv/app/start_ckan_development.sh"]

29
ckan-dev/2.9/Dockerfile Executable file
View File

@ -0,0 +1,29 @@
FROM ckan/ckan-base:testing-only.2.9
LABEL maintainer="brett@kowh.ai"
ENV APP_DIR=/srv/app
ENV SRC_EXTENSIONS_DIR=/srv/app/src_extensions
# Install packages needed by the dev requirements
RUN apk add --no-cache libffi-dev
# Set up Python3 virtual environment
RUN cd ${APP_DIR} && \
source ${APP_DIR}/bin/activate
# Virtual environment binaries/scripts to be used first
ENV PATH=${APP_DIR}/bin:${PATH}
# Install CKAN dev requirements
# Will need to change this eventually - when CKAN 2.9 is out
# wget https://raw.githubusercontent.com/ckan/ckan/master/dev-requirements.txt
# RUN pip3 install --no-binary :all: -r https://raw.githubusercontent.com/ckan/ckan/master/dev-requirements.txt
RUN pip3 install -r https://raw.githubusercontent.com/ckan/ckan/master/dev-requirements.txt
# Create folder for local extensions sources
RUN mkdir $SRC_EXTENSIONS_DIR
COPY setup/start_ckan_development.sh ${APP_DIR}
CMD ["/srv/app/start_ckan_development.sh"]

82
ckan-dev/setup/start_ckan_development.sh Executable file
View File

@ -0,0 +1,82 @@
#!/bin/bash
# Install any local extensions in the src_extensions volume
echo "Looking for local extensions to install..."
echo "Extension dir contents:"
ls -la $SRC_EXTENSIONS_DIR
for i in $SRC_EXTENSIONS_DIR/*
do
if [ -d $i ];
then
if [ -f $i/pip-requirements.txt ];
then
pip install -r $i/pip-requirements.txt
echo "Found requirements file in $i"
fi
if [ -f $i/requirements.txt ];
then
pip install -r $i/requirements.txt
echo "Found requirements file in $i"
fi
if [ -f $i/dev-requirements.txt ];
then
pip install -r $i/dev-requirements.txt
echo "Found dev-requirements file in $i"
fi
if [ -f $i/setup.py ];
then
cd $i
python $i/setup.py develop
echo "Found setup.py file in $i"
cd $APP_DIR
fi
# Point `use` in test.ini to location of `test-core.ini`
if [ -f $i/test.ini ];
then
echo "Updating \`test.ini\` reference to \`test-core.ini\` for plugin $i"
paster --plugin=ckan config-tool $i/test.ini "use = config:../../src/ckan/test-core.ini"
fi
fi
done
# Set debug to true
echo "Enabling debug mode"
ckan config-tool $CKAN_INI -s DEFAULT "debug = true"
# Update the plugins setting in the ini file with the values defined in the env var
echo "Loading the following plugins: $CKAN__PLUGINS"
ckan config-tool $CKAN_INI "ckan.plugins = $CKAN__PLUGINS"
# Update test-core.ini DB, SOLR & Redis settings
echo "Loading test settings into test-core.ini"
ckan config-tool $SRC_DIR/ckan/test-core.ini \
"sqlalchemy.url = $TEST_CKAN_SQLALCHEMY_URL" \
"ckan.datstore.write_url = $TEST_CKAN_DATASTORE_WRITE_URL" \
"ckan.datstore.read_url = $TEST_CKAN_DATASTORE_READ_URL" \
"solr_url = $TEST_CKAN_SOLR_URL" \
"ckan.redis_url = $TEST_CKAN_REDIS_URL"
# Run the prerun script to init CKAN and create the default admin user
sudo -u ckan -EH python prerun.py
# Run any startup scripts provided by images extending this one
if [[ -d "/docker-entrypoint.d" ]]
then
for f in /docker-entrypoint.d/*; do
case "$f" in
*.sh) echo "$0: Running init file $f"; . "$f" ;;
*.py) echo "$0: Running init file $f"; python "$f"; echo ;;
*) echo "$0: Ignoring $f (not an sh or py file)" ;;
esac
echo
done
fi
# Start supervisord
supervisord --configuration /etc/supervisord.conf &
# Start the development server with automatic reload
# Check the --reloader options sudo -u ckan -EH ckan -c $CKAN_INI run --reloader <TEXT>
sudo -u ckan -EH ckan -c $CKAN_INI run

48
docker-compose.yml
View File

@ -1,5 +1,10 @@
version: "3" version: "3"
volumes:
pg_data:
solr_data:
ckan_storage:
services: services:
nginx: nginx:
@ -11,7 +16,7 @@ services:
- ckan - ckan
ports: ports:
- "0.0.0.0:80:80" - "0.0.0.0:80:80"
ckan: ckan:
container_name: ckan container_name: ckan
build: build:
@ -29,18 +34,13 @@ services:
ports: ports:
- "0.0.0.0:${CKAN_PORT}:5000" - "0.0.0.0:${CKAN_PORT}:5000"
volumes: volumes:
- ckan_storage:/var/lib/ckan - ckan_storage:/var/lib/ckan
datapusher:
container_name: datapusher
image: kowhai/datapusher:0.0.17
ports:
- "8800:8800"
db: db:
container_name: db container_name: db
build: build:
context: postgresql/ context: .
dockerfile: postgresql/Dockerfile
args: args:
- DATASTORE_READONLY_PASSWORD=${DATASTORE_READONLY_PASSWORD} - DATASTORE_READONLY_PASSWORD=${DATASTORE_READONLY_PASSWORD}
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD} - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
@ -50,19 +50,31 @@ services:
- PGDATA=/var/lib/postgresql/data/db - PGDATA=/var/lib/postgresql/data/db
volumes: volumes:
- pg_data:/var/lib/postgresql/data - pg_data:/var/lib/postgresql/data
healthcheck:
test: ["CMD", "pg_isready", "-U", "ckan"]
datapusher:
container_name: datapusher
build:
context: datapusher
ports:
- "8800:8800"
environment:
- DATAPUSHER_REWRITE_RESOURCES=${DATAPUSHER_REWRITE_RESOURCES}
- DATAPUSHER_REWRITE_URL=${DATAPUSHER_REWRITE_URL}
- CKAN__DATAPUSHER__CALLBACK_URL_BASE=${CKAN__DATAPUSHER__CALLBACK_URL_BASE}
solr: solr:
container_name: solr container_name: solr
build: build:
context: solr/ context: .
dockerfile: solr/Dockerfile
args:
- SOLR_VERSION=${SOLR_VERSION}
- CKAN_VERSION=${CKAN_VERSION}
volumes: volumes:
- solr_data:/opt/solr/server/solr/ckan/data/index - solr_data:/opt/solr/server/solr/ckan/data
redis: redis:
container_name: redis container_name: redis
image: redis:alpine image: redis:${REDIS_VERSION}
volumes:
ckan_storage:
pg_data:
solr_data:

4
docker-entrypoint-initdb.d/10_create_datastore.sql Executable file
View File

@ -0,0 +1,4 @@
\set datastore_ro_password '\'' `echo $DATASTORE_READONLY_PASSWORD` '\''
CREATE ROLE datastore_ro NOSUPERUSER NOCREATEDB NOCREATEROLE LOGIN PASSWORD :datastore_ro_password;
CREATE DATABASE datastore OWNER ckan ENCODING 'utf-8';

2
docker-entrypoint-initdb.d/20_setup_test_databases.sql Executable file
View File

@ -0,0 +1,2 @@
CREATE DATABASE ckan_test OWNER ckan ENCODING 'utf-8';
CREATE DATABASE datastore_test OWNER ckan ENCODING 'utf-8';

22
index.html Normal file
View File

@ -0,0 +1,22 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<meta http-equiv="X-UA-Compatible" content="ie=edge">
<title>CKAN Docker NGINX landing page</title>
<style>
h1{
font-weight:lighter;
font-family: Arial, Helvetica, sans-serif;
}
</style>
</head>
<body>
<h2>
CKAN Docker NGINX landing page
</h2>
</body>
</html>

69
notes.txt Normal file
View File

@ -0,0 +1,69 @@
Environment variables are in the .env file and loaded into the docker-compose.yml file at runtime.
The Dockerfile will need an ARG <value> line also
List all env variables and decide where they go
.env
env-ckan
ARG
ENV
NB: .env variables can be referenced in the docker-compose.yml file (environment: part of the service)
The docker-compose.yml file can pass these variables to the Dockerfile as ARG parameters
--------------------------- from OKFN ---------------------------
# DB image settings
POSTGRES_PASSWORD=ckan
DATASTORE_READONLY_PASSWORD=datastore
# Basic
CKAN_SITE_ID=default
CKAN_SITE_URL=http://ckan:5000
CKAN_PORT=5000
CKAN_SYSADMIN_NAME=ckan_admin
CKAN_SYSADMIN_PASSWORD=test1234
CKAN_SYSADMIN_EMAIL=your_email@example.com
TZ=UTC
# Database connections (TODO: avoid duplication)
CKAN_SQLALCHEMY_URL=postgresql://ckan:ckan@db/ckan
CKAN_DATASTORE_WRITE_URL=postgresql://ckan:ckan@db/datastore
CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:datastore@db/datastore
# Test database connections
TEST_CKAN_SQLALCHEMY_URL=postgres://ckan:ckan@db/ckan_test
TEST_CKAN_DATASTORE_WRITE_URL=postgresql://ckan:ckan@db/datastore_test
TEST_CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:datastore@db/datastore_test
# Other services connections
CKAN_SOLR_URL=http://solr:8983/solr/ckan
CKAN_REDIS_URL=redis://redis:6379/1
CKAN_DATAPUSHER_URL=http://datapusher:8800
CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://ckan:5000
TEST_CKAN_SOLR_URL=http://solr:8983/solr/ckan
TEST_CKAN_REDIS_URL=redis://redis:6379/1
# Core settings
CKAN__STORAGE_PATH=/var/lib/ckan
CKAN_SMTP_SERVER=smtp.corporateict.domain:25
CKAN_SMTP_STARTTLS=True
CKAN_SMTP_USER=user
CKAN_SMTP_PASSWORD=pass
CKAN_SMTP_MAIL_FROM=ckan@localhost
# Extensions
CKAN__PLUGINS=envvars image_view text_view recline_view datastore datapusher
CKAN__HARVEST__MQ__TYPE=redis
CKAN__HARVEST__MQ__HOSTNAME=redis
CKAN__HARVEST__MQ__PORT=6379
CKAN__HARVEST__MQ__REDIS_DB=1
docker-compose.yml
DATASTORE_READONLY_PASSWORD=${DATASTORE_READONLY_PASSWORD}
POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
DATASTORE_READONLY_PASSWORD=${DATASTORE_READONLY_PASSWORD}
POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
PGDATA=/var/lib/postgresql/data/db

11
patches/00_patch_sql_url.patch Normal file
View File

@ -0,0 +1,11 @@
--- ckan/ckan/model/__init__.py 2021-02-16 14:47:06.168327441 +0100
+++ ckan/ckan/model/__init__.py 2021-02-16 14:48:00.740780218 +0100
@@ -266,7 +266,7 @@
self.reset_alembic_output()
alembic_config = AlembicConfig(self._alembic_ini)
alembic_config.set_main_option(
- "sqlalchemy.url", str(self.metadata.bind.url)
+ "sqlalchemy.url", str(self.metadata.bind.url).replace('%', '%%')
)
try:
sqlalchemy_migrate_version = self.metadata.bind.execute(

11
patches/01_patch_resource_replace_upload.patch Normal file
View File

@ -0,0 +1,11 @@
--- ckan/ckan/logic/action/update.py 2021-02-17 16:46:55.673578728 +0100
+++ ckan/ckan/logic/action/update-edit.py 2021-02-17 16:47:28.905879170 +0100
@@ -929,7 +929,7 @@
'''
model = context['model']
- session = model.Session
+ session = model.meta.create_local_session()
context['session'] = session
user = context['user']

11
patches/02_patch_postgres_username_split.patch Normal file
View File

@ -0,0 +1,11 @@
--- ckan/ckanext/datastore/backend/postgres.py 2021-02-18 11:01:56.692267462 +0100
+++ ckan/ckanext/datastore/backend/postgres-patch.py 2021-02-18 13:45:16.033193435 +0100
@@ -1690,7 +1690,7 @@
read only user.
'''
write_connection = self._get_write_engine().connect()
- read_connection_user = sa_url.make_url(self.read_url).username
+ read_connection_user = sa_url.make_url(self.read_url).username.split("@")[0]
drop_foo_sql = u'DROP TABLE IF EXISTS _foo'

5
scripts/apply_ckan_patches.sh Executable file
View File

@ -0,0 +1,5 @@
#!/bin/bash
shopt -s nullglob
for patch in patches/*.patch; do
/usr/bin/patch -p0 -i $patch
done

15
setup/ckan-uwsgi.ini Normal file
View File

@ -0,0 +1,15 @@
[uwsgi]
http-socket = :5000
uid = ckan
guid = ckan
plugins = python3
wsgi-file = /srv/app/wsgi.py
virtualenv = /srv/app
module = wsgi:application
master = true
processes = 5
pidfile = /tmp/%n.pid
harakiri = 50
max-requests = 5000
vacuum = true
callable = application

14
setup/datapusher-uwsgi.ini Normal file
View File

@ -0,0 +1,14 @@
[uwsgi]
http = 0.0.0.0:8800
uid = www-data
guid = www-data
wsgi-file = /srv/app/datapusher.wsgi
master = true
socket = /tmp/uwsgi.sock
plugins = http,python
pidfile = /tmp/%n.pid
harakiri = 50
max-requests = 5000
vacuum = true
callable = application
buffer-size = 32768

37
setup/datapusher_settings.py Normal file
View File

@ -0,0 +1,37 @@
import os
import uuid
DEBUG = False
TESTING = False
SECRET_KEY = str(uuid.uuid4())
USERNAME = str(uuid.uuid4())
PASSWORD = str(uuid.uuid4())
NAME = 'datapusher'
# Webserver host and port
HOST = os.environ.get('DATAPUSHER_HOST', '0.0.0.0')
PORT = os.environ.get('DATAPUSHER_PORT', 8800)
# Database
SQLALCHEMY_DATABASE_URI = os.environ.get('DATAPUSHER_SQLALCHEMY_DATABASE_URI', 'sqlite:////tmp/job_store.db')
# Download and streaming settings
MAX_CONTENT_LENGTH = int(os.environ.get('DATAPUSHER_MAX_CONTENT_LENGTH', '1024000'))
CHUNK_SIZE = int(os.environ.get('DATAPUSHER_CHUNK_SIZE', '16384'))
CHUNK_INSERT_ROWS = int(os.environ.get('DATAPUSHER_CHUNK_INSERT_ROWS', '250'))
DOWNLOAD_TIMEOUT = int(os.environ.get('DATAPUSHER_DOWNLOAD_TIMEOUT', '30'))
# Verify SSL
SSL_VERIFY = os.environ.get('DATAPUSHER_SSL_VERIFY', True)
# logging
#LOG_FILE = '/tmp/ckan_service.log'
STDERR = True
# Rewrite resource URL's when ckan callback url base is used
REWRITE_RESOURCES = os.environ.get('DATAPUSHER_REWRITE_RESOURCES', True)
REWRITE_URL = os.environ.get('DATAPUSHER_REWRITE_URL', 'http://ckan:5000/')

34
setup/nginx.conf Normal file
View File

@ -0,0 +1,34 @@
user nginx;
worker_processes auto;
error_log /var/log/nginx/error.log warn;
pid /var/run/nginx.pid;
events {
worker_connections 1024;
}
http {
include /etc/nginx/mime.types;
default_type application/octet-stream;
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';
access_log /var/log/nginx/access.log main;
sendfile on;
#tcp_nopush on;
keepalive_timeout 65;
#gzip on;
include /etc/nginx/sites-enabled/*.conf;
}

194
setup/prerun.py Executable file
View File

@ -0,0 +1,194 @@
import os
import sys
import subprocess
import psycopg2
import urllib3
import time
import re
ckan_ini = os.environ.get('CKAN_INI', '/srv/app/ckan.ini')
RETRY = 5
def update_plugins():
plugins = os.environ.get('CKAN__PLUGINS', '')
print('[prerun] Setting the following plugins in {}:'.format(ckan_ini))
print(plugins)
cmd = ['ckan', 'config-tool', ckan_ini,
'ckan.plugins = {}'.format(plugins)]
subprocess.check_output(cmd, stderr=subprocess.STDOUT)
print('[prerun] Plugins set.')
def check_main_db_connection(retry=None):
conn_str = os.environ.get('CKAN_SQLALCHEMY_URL')
if not conn_str:
print('[prerun] CKAN_SQLALCHEMY_URL not defined, not checking db')
return check_db_connection(conn_str, retry)
def check_datastore_db_connection(retry=None):
conn_str = os.environ.get('CKAN_DATASTORE_WRITE_URL')
if not conn_str:
print('[prerun] CKAN_DATASTORE_WRITE_URL not defined, not checking db')
return check_db_connection(conn_str, retry)
def check_db_connection(conn_str, retry=None):
if retry is None:
retry = RETRY
elif retry == 0:
print('[prerun] Giving up after 5 tries...')
sys.exit(1)
try:
connection = psycopg2.connect(conn_str)
except psycopg2.Error as e:
print(str(e))
print('[prerun] Unable to connect to the database, waiting...')
time.sleep(10)
check_db_connection(conn_str, retry=retry - 1)
else:
connection.close()
def check_solr_connection(retry=None):
if retry is None:
retry = RETRY
elif retry == 0:
print('[prerun] Giving up after 5 tries...')
sys.exit(1)
url = os.environ.get('CKAN_SOLR_URL', '')
search_url = '{url}/select/?q=*&wt=json'.format(url=url)
http = urllib3.PoolManager()
try:
r = http.request('GET', search_url)
except urllib3.exceptions.ConnectionError as e:
print(str(e))
print('[prerun] Unable to connect to solr, waiting...')
time.sleep(10)
check_solr_connection(retry=retry - 1)
else:
print('[prerun] Connection Status from SOLR is ', (r.status))
def init_db():
db_command = ['ckan', '-c', ckan_ini,
'db', 'init']
print('[prerun] Initializing or upgrading db - start')
try:
subprocess.check_output(db_command, stderr=subprocess.STDOUT)
print('[prerun] Initializing or upgrading db - end')
except subprocess.CalledProcessError as e:
if 'OperationalError' in e.output:
print(e.output)
print('[prerun] Database not ready, waiting a bit before exit...')
time.sleep(5)
sys.exit(1)
else:
print(e.output)
raise e
def init_datastore_db():
conn_str = os.environ.get('CKAN_DATASTORE_WRITE_URL')
if not conn_str:
print('[prerun] Skipping datastore initialization')
return
datastore_perms_command = ['ckan', '-c', ckan_ini,
'datastore', 'set-permissions']
connection = psycopg2.connect(conn_str)
cursor = connection.cursor()
print('[prerun] Initializing datastore db - start')
try:
datastore_perms = subprocess.Popen(
datastore_perms_command,
stdout=subprocess.PIPE)
perms_sql = datastore_perms.stdout.read().decode('utf-8')
# Remove internal pg command as psycopg2 does not like it
perms_sql = re.sub('\\\\connect \"(.*)\"', '', perms_sql)
cursor.execute(perms_sql)
for notice in connection.notices:
print(notice)
connection.commit()
print('[prerun] Initializing datastore db - end')
print(datastore_perms.stdout.read().decode('utf-8'))
except psycopg2.Error as e:
print('[prerun] Could not initialize datastore')
print(str(e))
except subprocess.CalledProcessError as e:
if 'OperationalError' in e.output:
print(e.output)
print('[prerun] Database not ready, waiting a bit before exit...')
time.sleep(5)
sys.exit(1)
else:
print(e.output)
raise e
finally:
cursor.close()
connection.close()
def create_sysadmin():
name = os.environ.get('CKAN_SYSADMIN_NAME')
password = os.environ.get('CKAN_SYSADMIN_PASSWORD')
email = os.environ.get('CKAN_SYSADMIN_EMAIL')
if name and password and email:
# Check if user exists
command = ['ckan', '-c', ckan_ini, 'user', 'show', name,]
out = subprocess.check_output(command)
if 'User:None' not in re.sub(r'\s', '', out.decode()):
print('[prerun] Sysadmin user exists, skipping creation')
return
# Create user
command = ['ckan', '-c', ckan_ini, 'user', 'add',
name,
'password=' + password,
'email=' + email]
subprocess.call(command)
print('[prerun] Created user {0}'.format(name))
# Make it sysadmin
command = ['ckan', '-c', ckan_ini, 'sysadmin', 'add',
name]
subprocess.call(command)
print('[prerun] Made user {0} a sysadmin'.format(name))
if __name__ == '__main__':
maintenance = os.environ.get('MAINTENANCE_MODE', '').lower() == 'true'
if maintenance:
print('[prerun] Maintenance mode, skipping setup...')
else:
check_main_db_connection()
init_db()
update_plugins()
check_datastore_db_connection()
init_datastore_db()
check_solr_connection()
create_sysadmin()

18
setup/sites-available/ckan.conf Normal file
View File

@ -0,0 +1,18 @@
proxy_cache_path /tmp/nginx_cache levels=1:2 keys_zone=cache:30m max_size=250m;
proxy_temp_path /tmp/nginx_proxy 1 2;
server {
client_max_body_size 100M;
location / {
proxy_pass http://ckan:5000/;
proxy_set_header X-Forwarded-For $remote_addr;
proxy_set_header Host $host;
proxy_cache cache;
proxy_cache_bypass $cookie_auth_tkt;
proxy_no_cache $cookie_auth_tkt;
proxy_cache_valid 30m;
proxy_cache_key $host$scheme$proxy_host$request_uri;
# In emergency comment out line to force caching
# proxy_ignore_headers X-Accel-Expires Expires Cache-Control;
}
}

46
setup/start_ckan.sh Executable file
View File

@ -0,0 +1,46 @@
#!/bin/bash
# Run the prerun script to init CKAN and create the default admin user
sudo -u ckan -EH python3 prerun.py
# Run any startup scripts provided by images extending this one
if [[ -d "/docker-entrypoint.d" ]]
then
for f in /docker-entrypoint.d/*; do
case "$f" in
*.sh) echo "$0: Running init file $f"; . "$f" ;;
*.py) echo "$0: Running init file $f"; python "$f"; echo ;;
*) echo "$0: Ignoring $f (not an sh or py file)" ;;
esac
echo
done
fi
# Check whether http basic auth password protection is enabled and enable basicauth routing on uwsgi respecfully
if [ $? -eq 0 ]
then
if [ "$PASSWORD_PROTECT" = true ]
then
if [ "$HTPASSWD_USER" ] || [ "$HTPASSWD_PASSWORD" ]
then
# Generate htpasswd file for basicauth
htpasswd -d -b -c /srv/app/.htpasswd $HTPASSWD_USER $HTPASSWD_PASSWORD
# Start supervisord
supervisord --configuration /etc/supervisord.conf &
# Start uwsgi with basicauth
sudo -u ckan -EH uwsgi -i ckan-uwsgi.ini
else
echo "Missing HTPASSWD_USER or HTPASSWD_PASSWORD environment variables. Exiting..."
exit 1
fi
else
# Start supervisord
supervisord --configuration /etc/supervisord.conf &
# Start uwsgi
sudo -u ckan -EH uwsgi -i ckan-uwsgi.ini
fi
else
echo "[prerun] failed...not starting CKAN."
fi

5
setup/start_datapusher.sh Executable file
View File

@ -0,0 +1,5 @@
#!/bin/sh
echo "[start_datapusher.sh] Starting supervisord."
# Start supervisord
supervisord --configuration /etc/supervisord.conf

12
setup/supervisor.uwsgi.conf Normal file
View File

@ -0,0 +1,12 @@
[program:datapusher-uwsgi]
command=/usr/sbin/uwsgi -i /srv/app/datapusher-uwsgi.ini
priority=501
autostart=true
autorestart=true
redirect_stderr=true
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stdout
stderr_logfile_maxbytes=0
user=www-data
environment=HOME="/srv/app",USER="www-data"

12
setup/supervisor.worker.conf Normal file
View File

@ -0,0 +1,12 @@
[program:ckan-worker]
command=ckan -c /srv/app/ckan.ini jobs worker
priority=501
autostart=true
autorestart=true
redirect_stderr=true
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stdout
stderr_logfile_maxbytes=0
user=ckan
environment=HOME="/srv/app",USER="ckan"

23
setup/supervisord.conf Normal file
View File

@ -0,0 +1,23 @@
[unix_http_server]
file = /tmp/supervisor.sock
chmod = 0777
chown = nobody:nogroup
[supervisord]
logfile = /tmp/supervisord.log
logfile_maxbytes = 50MB
logfile_backups=10
loglevel = trace
pidfile = /tmp/supervisord.pid
nodaemon = true
umask = 022
identifier = supervisor
[supervisorctl]
serverurl = unix:///tmp/supervisor.sock
[rpcinterface:supervisor]
supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
[include]
files = /etc/supervisord.d/*.conf

9
setup/wsgi.py Normal file
View File

@ -0,0 +1,9 @@
import os
from ckan.config.middleware import make_app
from ckan.cli import CKANConfigLoader
from logging.config import fileConfig as loggingFileConfig
config_filepath = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'ckan.ini')
abspath = os.path.join(os.path.dirname(os.path.abspath(__file__)))
loggingFileConfig(config_filepath)
config = CKANConfigLoader(config_filepath).get_config()
application = make_app(config)

24
solr/Dockerfile
View File

@ -2,9 +2,10 @@ FROM solr:6.6.6
# Enviroment variables # Enviroment variables
ENV SOLR_CORE ckan ENV SOLR_CORE ckan
ENV SOLR_VERSION 6.6.6
ENV CKAN_VERSION 2.9.1 # Build Arguments
###TODO!!! CKAN_VERSION to be passed in as an ARG ARG CKAN_VERSION
ARG SOLR_VERSION
# root user for initial config # root user for initial config
USER root USER root
@ -15,14 +16,14 @@ RUN mkdir -p /opt/solr/server/solr/${SOLR_CORE}/conf && \
mkdir -p /opt/solr/server/solr/${SOLR_CORE}/data/index mkdir -p /opt/solr/server/solr/${SOLR_CORE}/data/index
# Add files # Add files
ADD solrconfig.xml \ COPY ./solr/solrconfig-${CKAN_VERSION}.xml /opt/solr/server/solr/${SOLR_CORE}/conf/solrconfig.xml
https://raw.githubusercontent.com/ckan/ckan/ckan-${CKAN_VERSION}/ckan/config/solr/schema.xml \ ADD https://raw.githubusercontent.com/ckan/ckan/ckan-${CKAN_VERSION}/ckan/config/solr/schema.xml \
https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/$SOLR_VERSION/solr/server/solr/configsets/basic_configs/conf/currency.xml \ https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/${SOLR_VERSION}/solr/server/solr/configsets/basic_configs/conf/currency.xml \
https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/$SOLR_VERSION/solr/server/solr/configsets/basic_configs/conf/synonyms.txt \ https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/${SOLR_VERSION}/solr/server/solr/configsets/basic_configs/conf/synonyms.txt \
https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/$SOLR_VERSION/solr/server/solr/configsets/basic_configs/conf/stopwords.txt \ https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/${SOLR_VERSION}/solr/server/solr/configsets/basic_configs/conf/stopwords.txt \
https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/$SOLR_VERSION/solr/server/solr/configsets/basic_configs/conf/protwords.txt \ https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/${SOLR_VERSION}/solr/server/solr/configsets/basic_configs/conf/protwords.txt \
https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/$SOLR_VERSION/solr/server/solr/configsets/data_driven_schema_configs/conf/elevate.xml \ https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/${SOLR_VERSION}/solr/server/solr/configsets/data_driven_schema_configs/conf/elevate.xml \
/opt/solr/server/solr/$SOLR_CORE/conf/ /opt/solr/server/solr/${SOLR_CORE}/conf/
# Create core.properties # Create core.properties
RUN echo name=${SOLR_CORE} > /opt/solr/server/solr/${SOLR_CORE}/core.properties RUN echo name=${SOLR_CORE} > /opt/solr/server/solr/${SOLR_CORE}/core.properties
@ -32,3 +33,4 @@ RUN chown -R ${SOLR_USER}:${SOLR_USER} /opt/solr/server/solr/${SOLR_CORE}
# non-root user for runtime # non-root user for runtime
USER ${SOLR_USER}:${SOLR_USER} USER ${SOLR_USER}:${SOLR_USER}

2
solr/solrconfig.xml → solr/solrconfig-2.9.3.xml Executable file → Normal file
View File

@ -342,4 +342,4 @@
<int name="xsltCacheLifetimeSeconds">5</int> <int name="xsltCacheLifetimeSeconds">5</int>
</queryResponseWriter> </queryResponseWriter>
</config> </config>

12
todo.txt Normal file
View File

@ -0,0 +1,12 @@
Get the NGINX container running <done>
Add the redis container <done>
Add the solr container <done>
Add the db container <done>
Get a datapusher container working with supervisord <done>
Test the datapusher container works
Take out ARG's from docker-compose.yml is not needed
Test with new versions of CKAN (ie: if problems with Python 3.9)
Create (and push) the datapusher image
Create the ckan container
Create the ckanext container