Commit new images directory
This commit is contained in:
Brett
parent
9088ac295d
commit
6a8c9ecc5c
10
README
Executable file
10
README
Executable file
@ -0,0 +1,10 @@
|
|||||||
|
Build the images from this directory using:
|
||||||
|
|
||||||
|
cd $REPO_HOME
|
||||||
|
cd images/ckan-base
|
||||||
|
docker build -t ckan/ckan-base:testing-only-2.9 -f 2.9/Dockerfile .
|
||||||
|
docker push ckan/ckan-base:testing-only-2.9
|
||||||
|
docker build -t ckan/ckan-base:testing-only-2.8 -f 2.8/Dockerfile .
|
||||||
|
docker push ckan/ckan-base:testing-only-2.8
|
||||||
|
docker build -t ckan/ckan-base:testing-only-2.7 -f 2.7/Dockerfile .
|
||||||
|
docker push ckan/ckan-base:testing-only-2.7
|
||||||
101
ckan-base/2.7/Dockerfile
Executable file
101
ckan-base/2.7/Dockerfile
Executable file
@ -0,0 +1,101 @@
|
|||||||
|
FROM alpine:3.13
|
||||||
|
|
||||||
|
# Internals, you probably don't need to change these
|
||||||
|
ENV APP_DIR=/srv/app
|
||||||
|
ENV SRC_DIR=/srv/app/src
|
||||||
|
ENV CKAN_INI=${APP_DIR}/production.ini
|
||||||
|
ENV PIP_SRC=${SRC_DIR}
|
||||||
|
ENV CKAN_STORAGE_PATH=/var/lib/ckan
|
||||||
|
ENV GIT_URL=https://github.com/ckan/ckan.git
|
||||||
|
# CKAN version to build
|
||||||
|
ENV GIT_BRANCH=ckan-2.7.11
|
||||||
|
# Customize these on the .env file if needed
|
||||||
|
ENV CKAN_SITE_URL=http://localhost:5000
|
||||||
|
ENV CKAN__PLUGINS image_view text_view recline_view datastore datapusher envvars
|
||||||
|
|
||||||
|
WORKDIR ${APP_DIR}
|
||||||
|
|
||||||
|
# Install necessary packages to run CKAN
|
||||||
|
RUN apk add --no-cache tzdata \
|
||||||
|
git \
|
||||||
|
gettext \
|
||||||
|
postgresql-client \
|
||||||
|
python \
|
||||||
|
apache2-utils \
|
||||||
|
libxml2 \
|
||||||
|
libxslt \
|
||||||
|
musl-dev \
|
||||||
|
uwsgi \
|
||||||
|
uwsgi-http \
|
||||||
|
uwsgi-corerouter \
|
||||||
|
uwsgi-python \
|
||||||
|
py2-gevent \
|
||||||
|
uwsgi-gevent \
|
||||||
|
libmagic \
|
||||||
|
curl \
|
||||||
|
sudo && \
|
||||||
|
# Packages to build CKAN requirements and plugins
|
||||||
|
apk add --no-cache --virtual .build-deps \
|
||||||
|
postgresql-dev \
|
||||||
|
gcc \
|
||||||
|
make \
|
||||||
|
g++ \
|
||||||
|
autoconf \
|
||||||
|
automake \
|
||||||
|
libtool \
|
||||||
|
python-dev \
|
||||||
|
libxml2-dev \
|
||||||
|
libxslt-dev \
|
||||||
|
linux-headers
|
||||||
|
|
||||||
|
# Create SRC_DIR
|
||||||
|
RUN mkdir -p ${SRC_DIR}
|
||||||
|
|
||||||
|
# Install pip
|
||||||
|
RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/pip/2.7/get-pip.py && \
|
||||||
|
python ${SRC_DIR}/get-pip.py 'pip==20.3.3'
|
||||||
|
|
||||||
|
# Install pip, supervisord and uwsgi
|
||||||
|
RUN pip install supervisor && \
|
||||||
|
mkdir /etc/supervisord.d && \
|
||||||
|
#pip wheel --wheel-dir=/wheels uwsgi gevent && \
|
||||||
|
rm -rf ${SRC_DIR}/get-pip.py
|
||||||
|
|
||||||
|
COPY 2.7/setup/supervisord.conf /etc
|
||||||
|
|
||||||
|
# Install CKAN
|
||||||
|
RUN pip install -e git+${GIT_URL}@${GIT_BRANCH}#egg=ckan && \
|
||||||
|
cd ${SRC_DIR}/ckan && \
|
||||||
|
cp who.ini ${APP_DIR} && \
|
||||||
|
# Workaround to solve https://github.com/psycopg/psycopg2/issues/594 in Alpine 3.7
|
||||||
|
sed -i -e "s/psycopg2==2.4.5/psycopg2==2.7.3.2/g" requirements.txt && \
|
||||||
|
pip install --no-binary :all: -r requirements.txt && \
|
||||||
|
# Install CKAN envvars to support loading config from environment variables
|
||||||
|
pip install -e git+https://github.com/okfn/ckanext-envvars.git#egg=ckanext-envvars && \
|
||||||
|
# Create and update CKAN config
|
||||||
|
paster --plugin=ckan make-config ckan ${CKAN_INI} && \
|
||||||
|
paster --plugin=ckan config-tool ${CKAN_INI} "ckan.plugins = ${CKAN__PLUGINS}" && \
|
||||||
|
paster --plugin=ckan config-tool ${CKAN_INI} "ckan.site_url = ${CKAN__SITE_URL}"
|
||||||
|
|
||||||
|
# Create a local user and group to run the app
|
||||||
|
RUN addgroup -g 92 -S ckan && \
|
||||||
|
adduser -u 92 -h /srv/app -H -D -S -G ckan ckan
|
||||||
|
|
||||||
|
# Create local storage folder
|
||||||
|
RUN mkdir -p $CKAN_STORAGE_PATH && \
|
||||||
|
chown -R ckan:ckan $CKAN_STORAGE_PATH
|
||||||
|
|
||||||
|
COPY 2.7/setup ${APP_DIR}
|
||||||
|
COPY 2.7/setup/supervisor.worker.conf /etc/supervisord.d/worker.conf
|
||||||
|
COPY 2.7/setup/uwsgi.conf /srv/app/uwsgi.conf
|
||||||
|
|
||||||
|
# Create entrypoint directory for children image scripts
|
||||||
|
ONBUILD RUN mkdir /docker-entrypoint.d
|
||||||
|
|
||||||
|
RUN chown ckan -R /srv/app
|
||||||
|
|
||||||
|
EXPOSE 5000
|
||||||
|
|
||||||
|
HEALTHCHECK --interval=10s --timeout=5s --retries=5 CMD curl --fail http://localhost:5000/api/3/action/status_show || exit 1
|
||||||
|
|
||||||
|
CMD ["/srv/app/start_ckan.sh"]
|
||||||
197
ckan-base/2.7/setup/prerun.py
Executable file
197
ckan-base/2.7/setup/prerun.py
Executable file
@ -0,0 +1,197 @@
|
|||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import subprocess
|
||||||
|
import psycopg2
|
||||||
|
import urllib2
|
||||||
|
import time
|
||||||
|
import re
|
||||||
|
|
||||||
|
ckan_ini = os.environ.get('CKAN_INI', '/srv/app/production.ini')
|
||||||
|
|
||||||
|
RETRY = 5
|
||||||
|
|
||||||
|
def update_plugins():
|
||||||
|
|
||||||
|
plugins = os.environ.get('CKAN__PLUGINS', '')
|
||||||
|
print('[prerun] Setting the following plugins in {}:'.format(ckan_ini))
|
||||||
|
print(plugins)
|
||||||
|
cmd = ['paster', '--plugin=ckan', 'config-tool',
|
||||||
|
ckan_ini, 'ckan.plugins = {}'.format(plugins)]
|
||||||
|
subprocess.check_output(cmd, stderr=subprocess.STDOUT)
|
||||||
|
print '[prerun] Plugins set.'
|
||||||
|
|
||||||
|
|
||||||
|
def check_main_db_connection(retry=None):
|
||||||
|
|
||||||
|
conn_str = os.environ.get('CKAN_SQLALCHEMY_URL')
|
||||||
|
if not conn_str:
|
||||||
|
print '[prerun] CKAN_SQLALCHEMY_URL not defined, not checking db'
|
||||||
|
return check_db_connection(conn_str, retry)
|
||||||
|
|
||||||
|
|
||||||
|
def check_datastore_db_connection(retry=None):
|
||||||
|
|
||||||
|
conn_str = os.environ.get('CKAN_DATASTORE_WRITE_URL')
|
||||||
|
if not conn_str:
|
||||||
|
print '[prerun] CKAN_DATASTORE_WRITE_URL not defined, not checking db'
|
||||||
|
return check_db_connection(conn_str, retry)
|
||||||
|
|
||||||
|
|
||||||
|
def check_db_connection(conn_str, retry=None):
|
||||||
|
|
||||||
|
if retry is None:
|
||||||
|
retry = RETRY
|
||||||
|
elif retry == 0:
|
||||||
|
print '[prerun] Giving up after 5 tries...'
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
try:
|
||||||
|
connection = psycopg2.connect(conn_str)
|
||||||
|
|
||||||
|
except psycopg2.Error as e:
|
||||||
|
print str(e)
|
||||||
|
print '[prerun] Unable to connect to the database, waiting...'
|
||||||
|
time.sleep(10)
|
||||||
|
check_db_connection(conn_str, retry=retry - 1)
|
||||||
|
else:
|
||||||
|
connection.close()
|
||||||
|
|
||||||
|
|
||||||
|
def check_solr_connection(retry=None):
|
||||||
|
|
||||||
|
if retry is None:
|
||||||
|
retry = RETRY
|
||||||
|
elif retry == 0:
|
||||||
|
print '[prerun] Giving up after 5 tries...'
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
url = os.environ.get('CKAN_SOLR_URL', '')
|
||||||
|
search_url = '{url}/select/?q=*&wt=json'.format(url=url)
|
||||||
|
|
||||||
|
try:
|
||||||
|
connection = urllib2.urlopen(search_url)
|
||||||
|
except urllib2.URLError as e:
|
||||||
|
print str(e)
|
||||||
|
print '[prerun] Unable to connect to solr, waiting...'
|
||||||
|
time.sleep(10)
|
||||||
|
check_solr_connection(retry=retry - 1)
|
||||||
|
else:
|
||||||
|
eval(connection.read())
|
||||||
|
|
||||||
|
|
||||||
|
def init_db():
|
||||||
|
|
||||||
|
db_command = ['paster', '--plugin=ckan', 'db',
|
||||||
|
'init', '-c', ckan_ini]
|
||||||
|
print '[prerun] Initializing or upgrading db - start'
|
||||||
|
try:
|
||||||
|
subprocess.check_output(db_command, stderr=subprocess.STDOUT)
|
||||||
|
print '[prerun] Initializing or upgrading db - end'
|
||||||
|
except subprocess.CalledProcessError, e:
|
||||||
|
if 'OperationalError' in e.output:
|
||||||
|
print e.output
|
||||||
|
print '[prerun] Database not ready, waiting a bit before exit...'
|
||||||
|
time.sleep(5)
|
||||||
|
sys.exit(1)
|
||||||
|
else:
|
||||||
|
print e.output
|
||||||
|
raise e
|
||||||
|
|
||||||
|
|
||||||
|
def init_datastore_db():
|
||||||
|
|
||||||
|
conn_str = os.environ.get('CKAN_DATASTORE_WRITE_URL')
|
||||||
|
if not conn_str:
|
||||||
|
print '[prerun] Skipping datastore initialization'
|
||||||
|
return
|
||||||
|
|
||||||
|
datastore_perms_command = ['paster', '--plugin=ckan', 'datastore',
|
||||||
|
'set-permissions', '-c', ckan_ini]
|
||||||
|
|
||||||
|
connection = psycopg2.connect(conn_str)
|
||||||
|
cursor = connection.cursor()
|
||||||
|
|
||||||
|
print '[prerun] Initializing datastore db - start'
|
||||||
|
try:
|
||||||
|
datastore_perms = subprocess.Popen(
|
||||||
|
datastore_perms_command,
|
||||||
|
stdout=subprocess.PIPE)
|
||||||
|
|
||||||
|
perms_sql = datastore_perms.stdout.read()
|
||||||
|
# Remove internal pg command as psycopg2 does not like it
|
||||||
|
perms_sql = re.sub('\\\\connect \"(.*)\"', '', perms_sql)
|
||||||
|
cursor.execute(perms_sql)
|
||||||
|
for notice in connection.notices:
|
||||||
|
print notice
|
||||||
|
|
||||||
|
connection.commit()
|
||||||
|
|
||||||
|
print '[prerun] Initializing datastore db - end'
|
||||||
|
print datastore_perms.stdout.read()
|
||||||
|
except psycopg2.Error as e:
|
||||||
|
print '[prerun] Could not initialize datastore'
|
||||||
|
print str(e)
|
||||||
|
|
||||||
|
except subprocess.CalledProcessError, e:
|
||||||
|
if 'OperationalError' in e.output:
|
||||||
|
print e.output
|
||||||
|
print '[prerun] Database not ready, waiting a bit before exit...'
|
||||||
|
time.sleep(5)
|
||||||
|
sys.exit(1)
|
||||||
|
else:
|
||||||
|
print e.output
|
||||||
|
raise e
|
||||||
|
finally:
|
||||||
|
cursor.close()
|
||||||
|
connection.close()
|
||||||
|
|
||||||
|
|
||||||
|
def create_sysadmin():
|
||||||
|
|
||||||
|
name = os.environ.get('CKAN_SYSADMIN_NAME')
|
||||||
|
password = os.environ.get('CKAN_SYSADMIN_PASSWORD')
|
||||||
|
email = os.environ.get('CKAN_SYSADMIN_EMAIL')
|
||||||
|
|
||||||
|
if name and password and email:
|
||||||
|
|
||||||
|
# Check if user exists
|
||||||
|
command = ['paster', '--plugin=ckan', 'user', name, '-c', ckan_ini]
|
||||||
|
|
||||||
|
out = subprocess.check_output(command)
|
||||||
|
if 'User:None' not in re.sub(r'\s', '', out):
|
||||||
|
print '[prerun] Sysadmin user exists, skipping creation'
|
||||||
|
return
|
||||||
|
|
||||||
|
# Create user
|
||||||
|
command = ['paster', '--plugin=ckan', 'user', 'add',
|
||||||
|
name,
|
||||||
|
'password=' + password,
|
||||||
|
'email=' + email,
|
||||||
|
'-c', ckan_ini]
|
||||||
|
|
||||||
|
subprocess.call(command)
|
||||||
|
print '[prerun] Created user {0}'.format(name)
|
||||||
|
|
||||||
|
# Make it sysadmin
|
||||||
|
command = ['paster', '--plugin=ckan', 'sysadmin', 'add',
|
||||||
|
name,
|
||||||
|
'-c', ckan_ini]
|
||||||
|
|
||||||
|
subprocess.call(command)
|
||||||
|
print '[prerun] Made user {0} a sysadmin'.format(name)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
|
||||||
|
maintenance = os.environ.get('MAINTENANCE_MODE', '').lower() == 'true'
|
||||||
|
|
||||||
|
if maintenance:
|
||||||
|
print '[prerun] Maintenance mode, skipping setup...'
|
||||||
|
else:
|
||||||
|
check_main_db_connection()
|
||||||
|
init_db()
|
||||||
|
update_plugins()
|
||||||
|
check_datastore_db_connection()
|
||||||
|
init_datastore_db()
|
||||||
|
check_solr_connection()
|
||||||
|
create_sysadmin()
|
||||||
47
ckan-base/2.7/setup/start_ckan.sh
Executable file
47
ckan-base/2.7/setup/start_ckan.sh
Executable file
@ -0,0 +1,47 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
# Run the prerun script to init CKAN and create the default admin user
|
||||||
|
sudo -u ckan -EH python prerun.py
|
||||||
|
|
||||||
|
# Run any startup scripts provided by images extending this one
|
||||||
|
if [[ -d "/docker-entrypoint.d" ]]
|
||||||
|
then
|
||||||
|
for f in /docker-entrypoint.d/*; do
|
||||||
|
case "$f" in
|
||||||
|
*.sh) echo "$0: Running init file $f"; . "$f" ;;
|
||||||
|
*.py) echo "$0: Running init file $f"; python "$f"; echo ;;
|
||||||
|
*) echo "$0: Ignoring $f (not an sh or py file)" ;;
|
||||||
|
esac
|
||||||
|
echo
|
||||||
|
done
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Set the common uwsgi options
|
||||||
|
UWSGI_OPTS="--plugins http,python,gevent --socket /tmp/uwsgi.sock --uid 92 --gid 92 --http :5000 --master --enable-threads --paste config:/srv/app/production.ini --paste-logger --lazy-apps --gevent 2000 -p 2 -L"
|
||||||
|
|
||||||
|
# Check whether http basic auth password protection is enabled and enable basicauth routing on uwsgi respecfully
|
||||||
|
if [ $? -eq 0 ]
|
||||||
|
then
|
||||||
|
if [ "$PASSWORD_PROTECT" = true ]
|
||||||
|
then
|
||||||
|
if [ "$HTPASSWD_USER" ] || [ "$HTPASSWD_PASSWORD" ]
|
||||||
|
then
|
||||||
|
# Generate htpasswd file for basicauth
|
||||||
|
htpasswd -d -b -c /srv/app/.htpasswd $HTPASSWD_USER $HTPASSWD_PASSWORD
|
||||||
|
# Start supervisord
|
||||||
|
supervisord --configuration /etc/supervisord.conf &
|
||||||
|
# Start uwsgi with basicauth
|
||||||
|
sudo -u ckan -EH uwsgi --ini /srv/app/uwsgi.conf --pcre-jit $UWSGI_OPTS
|
||||||
|
else
|
||||||
|
echo "Missing HTPASSWD_USER or HTPASSWD_PASSWORD environment variables. Exiting..."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
# Start supervisord
|
||||||
|
supervisord --configuration /etc/supervisord.conf &
|
||||||
|
# Start uwsgi
|
||||||
|
sudo -u ckan -EH uwsgi $UWSGI_OPTS
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
echo "[prerun] failed...not starting CKAN."
|
||||||
|
fi
|
||||||
13
ckan-base/2.7/setup/supervisor.worker.conf
Executable file
13
ckan-base/2.7/setup/supervisor.worker.conf
Executable file
@ -0,0 +1,13 @@
|
|||||||
|
[program:ckan-worker]
|
||||||
|
command=paster --plugin=ckan jobs worker -c /srv/app/production.ini
|
||||||
|
priority=501
|
||||||
|
autostart=true
|
||||||
|
autorestart=true
|
||||||
|
redirect_stderr=true
|
||||||
|
stdout_logfile=/dev/stdout
|
||||||
|
stdout_logfile_maxbytes=0
|
||||||
|
stderr_logfile=/dev/stdout
|
||||||
|
stderr_logfile_maxbytes=0
|
||||||
|
user=ckan
|
||||||
|
environment=HOME="/srv/app",USER="ckan"
|
||||||
|
|
||||||
23
ckan-base/2.7/setup/supervisord.conf
Executable file
23
ckan-base/2.7/setup/supervisord.conf
Executable file
@ -0,0 +1,23 @@
|
|||||||
|
[unix_http_server]
|
||||||
|
file = /tmp/supervisor.sock
|
||||||
|
chmod = 0777
|
||||||
|
chown = nobody:nogroup
|
||||||
|
|
||||||
|
[supervisord]
|
||||||
|
logfile = /tmp/supervisord.log
|
||||||
|
logfile_maxbytes = 50MB
|
||||||
|
logfile_backups=10
|
||||||
|
loglevel = info
|
||||||
|
pidfile = /tmp/supervisord.pid
|
||||||
|
nodaemon = true
|
||||||
|
umask = 022
|
||||||
|
identifier = supervisor
|
||||||
|
|
||||||
|
[supervisorctl]
|
||||||
|
serverurl = unix:///tmp/supervisor.sock
|
||||||
|
|
||||||
|
[rpcinterface:supervisor]
|
||||||
|
supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
|
||||||
|
|
||||||
|
[include]
|
||||||
|
files = /etc/supervisord.d/*.conf
|
||||||
2
ckan-base/2.7/setup/uwsgi.conf
Executable file
2
ckan-base/2.7/setup/uwsgi.conf
Executable file
@ -0,0 +1,2 @@
|
|||||||
|
[uwsgi]
|
||||||
|
route = ^(?!/api).*$ basicauth:Restricted,/srv/app/.htpasswd
|
||||||
99
ckan-base/2.8/Dockerfile
Executable file
99
ckan-base/2.8/Dockerfile
Executable file
@ -0,0 +1,99 @@
|
|||||||
|
FROM alpine:3.13
|
||||||
|
|
||||||
|
# Internals, you probably don't need to change these
|
||||||
|
ENV APP_DIR=/srv/app
|
||||||
|
ENV SRC_DIR=/srv/app/src
|
||||||
|
ENV CKAN_INI=${APP_DIR}/production.ini
|
||||||
|
ENV PIP_SRC=${SRC_DIR}
|
||||||
|
ENV CKAN_STORAGE_PATH=/var/lib/ckan
|
||||||
|
ENV GIT_URL=https://github.com/ckan/ckan.git
|
||||||
|
# CKAN version to build
|
||||||
|
ENV GIT_BRANCH=ckan-2.8.8
|
||||||
|
# Customize these on the .env file if needed
|
||||||
|
ENV CKAN_SITE_URL=http://localhost:5000
|
||||||
|
ENV CKAN__PLUGINS image_view text_view recline_view datastore datapusher envvars
|
||||||
|
|
||||||
|
WORKDIR ${APP_DIR}
|
||||||
|
|
||||||
|
# Install necessary packages to run CKAN
|
||||||
|
RUN apk add --no-cache tzdata \
|
||||||
|
git \
|
||||||
|
gettext \
|
||||||
|
postgresql-client \
|
||||||
|
python \
|
||||||
|
apache2-utils \
|
||||||
|
libxml2 \
|
||||||
|
libxslt \
|
||||||
|
musl-dev \
|
||||||
|
uwsgi \
|
||||||
|
uwsgi-http \
|
||||||
|
uwsgi-corerouter \
|
||||||
|
uwsgi-python \
|
||||||
|
py2-gevent \
|
||||||
|
uwsgi-gevent \
|
||||||
|
libmagic \
|
||||||
|
curl \
|
||||||
|
sudo && \
|
||||||
|
# Packages to build CKAN requirements and plugins
|
||||||
|
apk add --no-cache --virtual .build-deps \
|
||||||
|
postgresql-dev \
|
||||||
|
gcc \
|
||||||
|
make \
|
||||||
|
g++ \
|
||||||
|
autoconf \
|
||||||
|
automake \
|
||||||
|
libtool \
|
||||||
|
python-dev \
|
||||||
|
libxml2-dev \
|
||||||
|
libxslt-dev \
|
||||||
|
linux-headers
|
||||||
|
|
||||||
|
# Create SRC_DIR
|
||||||
|
RUN mkdir -p ${SRC_DIR}
|
||||||
|
|
||||||
|
# Install pip
|
||||||
|
RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/pip/2.7/get-pip.py && \
|
||||||
|
python ${SRC_DIR}/get-pip.py 'pip==20.3.3'
|
||||||
|
|
||||||
|
# Install supervisord and uwsgi
|
||||||
|
RUN pip install supervisor && \
|
||||||
|
mkdir /etc/supervisord.d && \
|
||||||
|
#pip wheel --wheel-dir=/wheels uwsgi gevent && \
|
||||||
|
rm -rf ${SRC_DIR}/get-pip.py
|
||||||
|
|
||||||
|
COPY 2.8/setup/supervisord.conf /etc
|
||||||
|
|
||||||
|
# Install CKAN
|
||||||
|
RUN pip install -e git+${GIT_URL}@${GIT_BRANCH}#egg=ckan && \
|
||||||
|
cd ${SRC_DIR}/ckan && \
|
||||||
|
cp who.ini ${APP_DIR} && \
|
||||||
|
pip install --no-binary :all: -r requirements.txt && \
|
||||||
|
# Install CKAN envvars to support loading config from environment variables
|
||||||
|
pip install -e git+https://github.com/okfn/ckanext-envvars.git#egg=ckanext-envvars && \
|
||||||
|
# Create and update CKAN config
|
||||||
|
paster --plugin=ckan make-config ckan ${CKAN_INI} && \
|
||||||
|
paster --plugin=ckan config-tool ${CKAN_INI} "ckan.plugins = ${CKAN__PLUGINS}" && \
|
||||||
|
paster --plugin=ckan config-tool ${CKAN_INI} "ckan.site_url = ${CKAN__SITE_URL}"
|
||||||
|
|
||||||
|
# Create a local user and group to run the app
|
||||||
|
RUN addgroup -g 92 -S ckan && \
|
||||||
|
adduser -u 92 -h /srv/app -H -D -S -G ckan ckan
|
||||||
|
|
||||||
|
# Create local storage folder
|
||||||
|
RUN mkdir -p $CKAN_STORAGE_PATH && \
|
||||||
|
chown -R ckan:ckan $CKAN_STORAGE_PATH
|
||||||
|
|
||||||
|
COPY 2.8/setup ${APP_DIR}
|
||||||
|
COPY 2.8/setup/supervisor.worker.conf /etc/supervisord.d/worker.conf
|
||||||
|
COPY 2.8/setup/uwsgi.conf /srv/app/uwsgi.conf
|
||||||
|
|
||||||
|
# Create entrypoint directory for children image scripts
|
||||||
|
ONBUILD RUN mkdir /docker-entrypoint.d
|
||||||
|
|
||||||
|
RUN chown ckan -R /srv/app
|
||||||
|
|
||||||
|
EXPOSE 5000
|
||||||
|
|
||||||
|
HEALTHCHECK --interval=10s --timeout=5s --retries=5 CMD curl --fail http://localhost:5000/api/3/action/status_show || exit 1
|
||||||
|
|
||||||
|
CMD ["/srv/app/start_ckan.sh"]
|
||||||
197
ckan-base/2.8/setup/prerun.py
Executable file
197
ckan-base/2.8/setup/prerun.py
Executable file
@ -0,0 +1,197 @@
|
|||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import subprocess
|
||||||
|
import psycopg2
|
||||||
|
import urllib2
|
||||||
|
import time
|
||||||
|
import re
|
||||||
|
|
||||||
|
ckan_ini = os.environ.get('CKAN_INI', '/srv/app/production.ini')
|
||||||
|
|
||||||
|
RETRY = 5
|
||||||
|
|
||||||
|
def update_plugins():
|
||||||
|
|
||||||
|
plugins = os.environ.get('CKAN__PLUGINS', '')
|
||||||
|
print('[prerun] Setting the following plugins in {}:'.format(ckan_ini))
|
||||||
|
print(plugins)
|
||||||
|
cmd = ['paster', '--plugin=ckan', 'config-tool',
|
||||||
|
ckan_ini, 'ckan.plugins = {}'.format(plugins)]
|
||||||
|
subprocess.check_output(cmd, stderr=subprocess.STDOUT)
|
||||||
|
print '[prerun] Plugins set.'
|
||||||
|
|
||||||
|
|
||||||
|
def check_main_db_connection(retry=None):
|
||||||
|
|
||||||
|
conn_str = os.environ.get('CKAN_SQLALCHEMY_URL')
|
||||||
|
if not conn_str:
|
||||||
|
print '[prerun] CKAN_SQLALCHEMY_URL not defined, not checking db'
|
||||||
|
return check_db_connection(conn_str, retry)
|
||||||
|
|
||||||
|
|
||||||
|
def check_datastore_db_connection(retry=None):
|
||||||
|
|
||||||
|
conn_str = os.environ.get('CKAN_DATASTORE_WRITE_URL')
|
||||||
|
if not conn_str:
|
||||||
|
print '[prerun] CKAN_DATASTORE_WRITE_URL not defined, not checking db'
|
||||||
|
return check_db_connection(conn_str, retry)
|
||||||
|
|
||||||
|
|
||||||
|
def check_db_connection(conn_str, retry=None):
|
||||||
|
|
||||||
|
if retry is None:
|
||||||
|
retry = RETRY
|
||||||
|
elif retry == 0:
|
||||||
|
print '[prerun] Giving up after 5 tries...'
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
try:
|
||||||
|
connection = psycopg2.connect(conn_str)
|
||||||
|
|
||||||
|
except psycopg2.Error as e:
|
||||||
|
print str(e)
|
||||||
|
print '[prerun] Unable to connect to the database, waiting...'
|
||||||
|
time.sleep(10)
|
||||||
|
check_db_connection(conn_str, retry=retry - 1)
|
||||||
|
else:
|
||||||
|
connection.close()
|
||||||
|
|
||||||
|
|
||||||
|
def check_solr_connection(retry=None):
|
||||||
|
|
||||||
|
if retry is None:
|
||||||
|
retry = RETRY
|
||||||
|
elif retry == 0:
|
||||||
|
print '[prerun] Giving up after 5 tries...'
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
url = os.environ.get('CKAN_SOLR_URL', '')
|
||||||
|
search_url = '{url}/select/?q=*&wt=json'.format(url=url)
|
||||||
|
|
||||||
|
try:
|
||||||
|
connection = urllib2.urlopen(search_url)
|
||||||
|
except urllib2.URLError as e:
|
||||||
|
print str(e)
|
||||||
|
print '[prerun] Unable to connect to solr, waiting...'
|
||||||
|
time.sleep(10)
|
||||||
|
check_solr_connection(retry=retry - 1)
|
||||||
|
else:
|
||||||
|
eval(connection.read())
|
||||||
|
|
||||||
|
|
||||||
|
def init_db():
|
||||||
|
|
||||||
|
db_command = ['paster', '--plugin=ckan', 'db',
|
||||||
|
'init', '-c', ckan_ini]
|
||||||
|
print '[prerun] Initializing or upgrading db - start'
|
||||||
|
try:
|
||||||
|
subprocess.check_output(db_command, stderr=subprocess.STDOUT)
|
||||||
|
print '[prerun] Initializing or upgrading db - end'
|
||||||
|
except subprocess.CalledProcessError, e:
|
||||||
|
if 'OperationalError' in e.output:
|
||||||
|
print e.output
|
||||||
|
print '[prerun] Database not ready, waiting a bit before exit...'
|
||||||
|
time.sleep(5)
|
||||||
|
sys.exit(1)
|
||||||
|
else:
|
||||||
|
print e.output
|
||||||
|
raise e
|
||||||
|
|
||||||
|
|
||||||
|
def init_datastore_db():
|
||||||
|
|
||||||
|
conn_str = os.environ.get('CKAN_DATASTORE_WRITE_URL')
|
||||||
|
if not conn_str:
|
||||||
|
print '[prerun] Skipping datastore initialization'
|
||||||
|
return
|
||||||
|
|
||||||
|
datastore_perms_command = ['paster', '--plugin=ckan', 'datastore',
|
||||||
|
'set-permissions', '-c', ckan_ini]
|
||||||
|
|
||||||
|
connection = psycopg2.connect(conn_str)
|
||||||
|
cursor = connection.cursor()
|
||||||
|
|
||||||
|
print '[prerun] Initializing datastore db - start'
|
||||||
|
try:
|
||||||
|
datastore_perms = subprocess.Popen(
|
||||||
|
datastore_perms_command,
|
||||||
|
stdout=subprocess.PIPE)
|
||||||
|
|
||||||
|
perms_sql = datastore_perms.stdout.read()
|
||||||
|
# Remove internal pg command as psycopg2 does not like it
|
||||||
|
perms_sql = re.sub('\\\\connect \"(.*)\"', '', perms_sql)
|
||||||
|
cursor.execute(perms_sql)
|
||||||
|
for notice in connection.notices:
|
||||||
|
print notice
|
||||||
|
|
||||||
|
connection.commit()
|
||||||
|
|
||||||
|
print '[prerun] Initializing datastore db - end'
|
||||||
|
print datastore_perms.stdout.read()
|
||||||
|
except psycopg2.Error as e:
|
||||||
|
print '[prerun] Could not initialize datastore'
|
||||||
|
print str(e)
|
||||||
|
|
||||||
|
except subprocess.CalledProcessError, e:
|
||||||
|
if 'OperationalError' in e.output:
|
||||||
|
print e.output
|
||||||
|
print '[prerun] Database not ready, waiting a bit before exit...'
|
||||||
|
time.sleep(5)
|
||||||
|
sys.exit(1)
|
||||||
|
else:
|
||||||
|
print e.output
|
||||||
|
raise e
|
||||||
|
finally:
|
||||||
|
cursor.close()
|
||||||
|
connection.close()
|
||||||
|
|
||||||
|
|
||||||
|
def create_sysadmin():
|
||||||
|
|
||||||
|
name = os.environ.get('CKAN_SYSADMIN_NAME')
|
||||||
|
password = os.environ.get('CKAN_SYSADMIN_PASSWORD')
|
||||||
|
email = os.environ.get('CKAN_SYSADMIN_EMAIL')
|
||||||
|
|
||||||
|
if name and password and email:
|
||||||
|
|
||||||
|
# Check if user exists
|
||||||
|
command = ['paster', '--plugin=ckan', 'user', name, '-c', ckan_ini]
|
||||||
|
|
||||||
|
out = subprocess.check_output(command)
|
||||||
|
if 'User:None' not in re.sub(r'\s', '', out):
|
||||||
|
print '[prerun] Sysadmin user exists, skipping creation'
|
||||||
|
return
|
||||||
|
|
||||||
|
# Create user
|
||||||
|
command = ['paster', '--plugin=ckan', 'user', 'add',
|
||||||
|
name,
|
||||||
|
'password=' + password,
|
||||||
|
'email=' + email,
|
||||||
|
'-c', ckan_ini]
|
||||||
|
|
||||||
|
subprocess.call(command)
|
||||||
|
print '[prerun] Created user {0}'.format(name)
|
||||||
|
|
||||||
|
# Make it sysadmin
|
||||||
|
command = ['paster', '--plugin=ckan', 'sysadmin', 'add',
|
||||||
|
name,
|
||||||
|
'-c', ckan_ini]
|
||||||
|
|
||||||
|
subprocess.call(command)
|
||||||
|
print '[prerun] Made user {0} a sysadmin'.format(name)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
|
||||||
|
maintenance = os.environ.get('MAINTENANCE_MODE', '').lower() == 'true'
|
||||||
|
|
||||||
|
if maintenance:
|
||||||
|
print '[prerun] Maintenance mode, skipping setup...'
|
||||||
|
else:
|
||||||
|
check_main_db_connection()
|
||||||
|
init_db()
|
||||||
|
update_plugins()
|
||||||
|
check_datastore_db_connection()
|
||||||
|
init_datastore_db()
|
||||||
|
check_solr_connection()
|
||||||
|
create_sysadmin()
|
||||||
47
ckan-base/2.8/setup/start_ckan.sh
Executable file
47
ckan-base/2.8/setup/start_ckan.sh
Executable file
@ -0,0 +1,47 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
# Run the prerun script to init CKAN and create the default admin user
|
||||||
|
sudo -u ckan -EH python prerun.py
|
||||||
|
|
||||||
|
# Run any startup scripts provided by images extending this one
|
||||||
|
if [[ -d "/docker-entrypoint.d" ]]
|
||||||
|
then
|
||||||
|
for f in /docker-entrypoint.d/*; do
|
||||||
|
case "$f" in
|
||||||
|
*.sh) echo "$0: Running init file $f"; . "$f" ;;
|
||||||
|
*.py) echo "$0: Running init file $f"; python "$f"; echo ;;
|
||||||
|
*) echo "$0: Ignoring $f (not an sh or py file)" ;;
|
||||||
|
esac
|
||||||
|
echo
|
||||||
|
done
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Set the common uwsgi options
|
||||||
|
UWSGI_OPTS="--plugins http,python,gevent --socket /tmp/uwsgi.sock --uid 92 --gid 92 --http :5000 --master --enable-threads --paste config:/srv/app/production.ini --paste-logger --lazy-apps --gevent 2000 -p 2 -L"
|
||||||
|
|
||||||
|
# Check whether http basic auth password protection is enabled and enable basicauth routing on uwsgi respecfully
|
||||||
|
if [ $? -eq 0 ]
|
||||||
|
then
|
||||||
|
if [ "$PASSWORD_PROTECT" = true ]
|
||||||
|
then
|
||||||
|
if [ "$HTPASSWD_USER" ] || [ "$HTPASSWD_PASSWORD" ]
|
||||||
|
then
|
||||||
|
# Generate htpasswd file for basicauth
|
||||||
|
htpasswd -d -b -c /srv/app/.htpasswd $HTPASSWD_USER $HTPASSWD_PASSWORD
|
||||||
|
# Start supervisord
|
||||||
|
supervisord --configuration /etc/supervisord.conf &
|
||||||
|
# Start uwsgi with basicauth
|
||||||
|
sudo -u ckan -EH uwsgi --ini /srv/app/uwsgi.conf --pcre-jit $UWSGI_OPTS
|
||||||
|
else
|
||||||
|
echo "Missing HTPASSWD_USER or HTPASSWD_PASSWORD environment variables. Exiting..."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
# Start supervisord
|
||||||
|
supervisord --configuration /etc/supervisord.conf &
|
||||||
|
# Start uwsgi
|
||||||
|
sudo -u ckan -EH uwsgi $UWSGI_OPTS
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
echo "[prerun] failed...not starting CKAN."
|
||||||
|
fi
|
||||||
13
ckan-base/2.8/setup/supervisor.worker.conf
Executable file
13
ckan-base/2.8/setup/supervisor.worker.conf
Executable file
@ -0,0 +1,13 @@
|
|||||||
|
[program:ckan-worker]
|
||||||
|
command=paster --plugin=ckan jobs worker -c /srv/app/production.ini
|
||||||
|
priority=501
|
||||||
|
autostart=true
|
||||||
|
autorestart=true
|
||||||
|
redirect_stderr=true
|
||||||
|
stdout_logfile=/dev/stdout
|
||||||
|
stdout_logfile_maxbytes=0
|
||||||
|
stderr_logfile=/dev/stdout
|
||||||
|
stderr_logfile_maxbytes=0
|
||||||
|
user=ckan
|
||||||
|
environment=HOME="/srv/app",USER="ckan"
|
||||||
|
|
||||||
23
ckan-base/2.8/setup/supervisord.conf
Executable file
23
ckan-base/2.8/setup/supervisord.conf
Executable file
@ -0,0 +1,23 @@
|
|||||||
|
[unix_http_server]
|
||||||
|
file = /tmp/supervisor.sock
|
||||||
|
chmod = 0777
|
||||||
|
chown = nobody:nogroup
|
||||||
|
|
||||||
|
[supervisord]
|
||||||
|
logfile = /tmp/supervisord.log
|
||||||
|
logfile_maxbytes = 50MB
|
||||||
|
logfile_backups=10
|
||||||
|
loglevel = info
|
||||||
|
pidfile = /tmp/supervisord.pid
|
||||||
|
nodaemon = true
|
||||||
|
umask = 022
|
||||||
|
identifier = supervisor
|
||||||
|
|
||||||
|
[supervisorctl]
|
||||||
|
serverurl = unix:///tmp/supervisor.sock
|
||||||
|
|
||||||
|
[rpcinterface:supervisor]
|
||||||
|
supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
|
||||||
|
|
||||||
|
[include]
|
||||||
|
files = /etc/supervisord.d/*.conf
|
||||||
2
ckan-base/2.8/setup/uwsgi.conf
Executable file
2
ckan-base/2.8/setup/uwsgi.conf
Executable file
@ -0,0 +1,2 @@
|
|||||||
|
[uwsgi]
|
||||||
|
route = ^(?!/api).*$ basicauth:Restricted,/srv/app/.htpasswd
|
||||||
109
ckan-base/2.9/Dockerfile
Executable file
109
ckan-base/2.9/Dockerfile
Executable file
@ -0,0 +1,109 @@
|
|||||||
|
FROM alpine:3.13
|
||||||
|
|
||||||
|
# Internal environment variables
|
||||||
|
ENV APP_DIR=/srv/app
|
||||||
|
ENV SRC_DIR=/srv/app/src
|
||||||
|
ENV CKAN_INI=${APP_DIR}/ckan.ini
|
||||||
|
ENV PIP_SRC=${SRC_DIR}
|
||||||
|
ENV CKAN_STORAGE_PATH=/var/lib/ckan
|
||||||
|
ENV GIT_URL=https://github.com/ckan/ckan.git
|
||||||
|
# CKAN version to build
|
||||||
|
ENV GIT_BRANCH=ckan-2.9.4
|
||||||
|
# Customize these on the .env file if needed
|
||||||
|
ENV CKAN_SITE_URL=http://localhost:5000
|
||||||
|
ENV CKAN__PLUGINS image_view text_view recline_view datastore datapusher envvars
|
||||||
|
|
||||||
|
WORKDIR ${APP_DIR}
|
||||||
|
|
||||||
|
# Install necessary packages to run CKAN
|
||||||
|
RUN apk add --no-cache tzdata \
|
||||||
|
git \
|
||||||
|
gettext \
|
||||||
|
postgresql-client \
|
||||||
|
python3 \
|
||||||
|
apache2-utils \
|
||||||
|
libxml2 \
|
||||||
|
libxslt \
|
||||||
|
musl-dev \
|
||||||
|
uwsgi-http \
|
||||||
|
uwsgi-corerouter \
|
||||||
|
uwsgi-python3 \
|
||||||
|
py3-gevent \
|
||||||
|
uwsgi-gevent \
|
||||||
|
libmagic \
|
||||||
|
curl \
|
||||||
|
sudo && \
|
||||||
|
# Packages to build CKAN requirements and plugins
|
||||||
|
apk add --no-cache --virtual .build-deps \
|
||||||
|
postgresql-dev \
|
||||||
|
gcc \
|
||||||
|
make \
|
||||||
|
g++ \
|
||||||
|
autoconf \
|
||||||
|
automake \
|
||||||
|
libtool \
|
||||||
|
python3-dev \
|
||||||
|
py3-virtualenv \
|
||||||
|
libxml2-dev \
|
||||||
|
libxslt-dev \
|
||||||
|
linux-headers
|
||||||
|
|
||||||
|
# Create src directory (SRC_DIR)
|
||||||
|
RUN mkdir -p ${SRC_DIR}
|
||||||
|
|
||||||
|
# Install pip
|
||||||
|
RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/get-pip.py && \
|
||||||
|
python3 ${SRC_DIR}/get-pip.py
|
||||||
|
|
||||||
|
# Set up Python3 virtual environment
|
||||||
|
RUN cd ${APP_DIR} && \
|
||||||
|
python3 -m venv ${APP_DIR} && \
|
||||||
|
source ${APP_DIR}/bin/activate
|
||||||
|
|
||||||
|
# Virtual environment binaries/scripts to be used first
|
||||||
|
ENV PATH=${APP_DIR}/bin:${PATH}
|
||||||
|
|
||||||
|
# Install CKAN code
|
||||||
|
RUN pip3 install -e git+${GIT_URL}@${GIT_BRANCH}#egg=ckan
|
||||||
|
|
||||||
|
# Install uwsgi
|
||||||
|
RUN pip3 install uwsgi
|
||||||
|
|
||||||
|
# Complete CKAN install
|
||||||
|
RUN cd ${SRC_DIR}/ckan && \
|
||||||
|
cp who.ini ${APP_DIR} && \
|
||||||
|
pip3 install -r requirement-setuptools.txt && \
|
||||||
|
pip3 install --no-binary :all: -r requirements.txt && \
|
||||||
|
# Install CKAN envvars to support loading config from environment variables
|
||||||
|
pip3 install -e git+https://github.com/okfn/ckanext-envvars.git#egg=ckanext-envvars
|
||||||
|
|
||||||
|
# Create and update CKAN config
|
||||||
|
RUN ckan generate config ${CKAN_INI}
|
||||||
|
|
||||||
|
# Install and configure supervisor
|
||||||
|
RUN pip3 install supervisor && \
|
||||||
|
mkdir /etc/supervisord.d
|
||||||
|
|
||||||
|
# Copy all setup files
|
||||||
|
COPY 2.9/setup ${APP_DIR}
|
||||||
|
COPY 2.9/setup/supervisor.worker.conf /etc/supervisord.d/worker.conf
|
||||||
|
COPY 2.9/setup/supervisord.conf /etc/supervisord.conf
|
||||||
|
|
||||||
|
# Create a local user and group to run the app
|
||||||
|
RUN addgroup -g 92 -S ckan && \
|
||||||
|
adduser -u 92 -h /srv/app -H -D -S -G ckan ckan
|
||||||
|
|
||||||
|
# Create local storage folder
|
||||||
|
RUN mkdir -p $CKAN_STORAGE_PATH && \
|
||||||
|
chown -R ckan:ckan $CKAN_STORAGE_PATH
|
||||||
|
|
||||||
|
# Create entrypoint directory for children image scripts
|
||||||
|
ONBUILD RUN mkdir /docker-entrypoint.d
|
||||||
|
|
||||||
|
RUN chown ckan -R /srv/app
|
||||||
|
|
||||||
|
EXPOSE 5000
|
||||||
|
|
||||||
|
HEALTHCHECK --interval=10s --timeout=5s --retries=5 CMD curl --fail http://localhost:5000/api/3/action/status_show || exit 1
|
||||||
|
|
||||||
|
CMD ["/srv/app/start_ckan.sh"]
|
||||||
15
ckan-base/2.9/setup/ckan-uwsgi.ini
Normal file
15
ckan-base/2.9/setup/ckan-uwsgi.ini
Normal file
@ -0,0 +1,15 @@
|
|||||||
|
[uwsgi]
|
||||||
|
http-socket = :5000
|
||||||
|
uid = ckan
|
||||||
|
guid = ckan
|
||||||
|
plugins = python3
|
||||||
|
wsgi-file = /srv/app/wsgi.py
|
||||||
|
virtualenv = /srv/app
|
||||||
|
module = wsgi:application
|
||||||
|
master = true
|
||||||
|
processes = 5
|
||||||
|
pidfile = /tmp/%n.pid
|
||||||
|
harakiri = 50
|
||||||
|
max-requests = 5000
|
||||||
|
vacuum = true
|
||||||
|
callable = application
|
||||||
194
ckan-base/2.9/setup/prerun.py
Executable file
194
ckan-base/2.9/setup/prerun.py
Executable file
@ -0,0 +1,194 @@
|
|||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import subprocess
|
||||||
|
import psycopg2
|
||||||
|
import urllib3
|
||||||
|
import time
|
||||||
|
import re
|
||||||
|
|
||||||
|
ckan_ini = os.environ.get('CKAN_INI', '/srv/app/ckan.ini')
|
||||||
|
|
||||||
|
RETRY = 5
|
||||||
|
|
||||||
|
def update_plugins():
|
||||||
|
|
||||||
|
plugins = os.environ.get('CKAN__PLUGINS', '')
|
||||||
|
print('[prerun] Setting the following plugins in {}:'.format(ckan_ini))
|
||||||
|
print(plugins)
|
||||||
|
cmd = ['ckan', 'config-tool', ckan_ini,
|
||||||
|
'ckan.plugins = {}'.format(plugins)]
|
||||||
|
subprocess.check_output(cmd, stderr=subprocess.STDOUT)
|
||||||
|
print('[prerun] Plugins set.')
|
||||||
|
|
||||||
|
|
||||||
|
def check_main_db_connection(retry=None):
|
||||||
|
|
||||||
|
conn_str = os.environ.get('CKAN_SQLALCHEMY_URL')
|
||||||
|
if not conn_str:
|
||||||
|
print('[prerun] CKAN_SQLALCHEMY_URL not defined, not checking db')
|
||||||
|
return check_db_connection(conn_str, retry)
|
||||||
|
|
||||||
|
|
||||||
|
def check_datastore_db_connection(retry=None):
|
||||||
|
|
||||||
|
conn_str = os.environ.get('CKAN_DATASTORE_WRITE_URL')
|
||||||
|
if not conn_str:
|
||||||
|
print('[prerun] CKAN_DATASTORE_WRITE_URL not defined, not checking db')
|
||||||
|
return check_db_connection(conn_str, retry)
|
||||||
|
|
||||||
|
|
||||||
|
def check_db_connection(conn_str, retry=None):
|
||||||
|
|
||||||
|
if retry is None:
|
||||||
|
retry = RETRY
|
||||||
|
elif retry == 0:
|
||||||
|
print('[prerun] Giving up after 5 tries...')
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
try:
|
||||||
|
connection = psycopg2.connect(conn_str)
|
||||||
|
|
||||||
|
except psycopg2.Error as e:
|
||||||
|
print(str(e))
|
||||||
|
print('[prerun] Unable to connect to the database, waiting...')
|
||||||
|
time.sleep(10)
|
||||||
|
check_db_connection(conn_str, retry=retry - 1)
|
||||||
|
else:
|
||||||
|
connection.close()
|
||||||
|
|
||||||
|
|
||||||
|
def check_solr_connection(retry=None):
|
||||||
|
|
||||||
|
if retry is None:
|
||||||
|
retry = RETRY
|
||||||
|
elif retry == 0:
|
||||||
|
print('[prerun] Giving up after 5 tries...')
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
url = os.environ.get('CKAN_SOLR_URL', '')
|
||||||
|
search_url = '{url}/select/?q=*&wt=json'.format(url=url)
|
||||||
|
http = urllib3.PoolManager()
|
||||||
|
try:
|
||||||
|
r = http.request('GET', search_url)
|
||||||
|
except urllib3.exceptions.ConnectionError as e:
|
||||||
|
print(str(e))
|
||||||
|
print('[prerun] Unable to connect to solr, waiting...')
|
||||||
|
time.sleep(10)
|
||||||
|
check_solr_connection(retry=retry - 1)
|
||||||
|
else:
|
||||||
|
print('[prerun] Connection Status from SOLR is ', (r.status))
|
||||||
|
|
||||||
|
def init_db():
|
||||||
|
|
||||||
|
db_command = ['ckan', '-c', ckan_ini,
|
||||||
|
'db', 'init']
|
||||||
|
print('[prerun] Initializing or upgrading db - start')
|
||||||
|
try:
|
||||||
|
subprocess.check_output(db_command, stderr=subprocess.STDOUT)
|
||||||
|
print('[prerun] Initializing or upgrading db - end')
|
||||||
|
except subprocess.CalledProcessError as e:
|
||||||
|
if 'OperationalError' in e.output:
|
||||||
|
print(e.output)
|
||||||
|
print('[prerun] Database not ready, waiting a bit before exit...')
|
||||||
|
time.sleep(5)
|
||||||
|
sys.exit(1)
|
||||||
|
else:
|
||||||
|
print(e.output)
|
||||||
|
raise e
|
||||||
|
|
||||||
|
|
||||||
|
def init_datastore_db():
|
||||||
|
|
||||||
|
conn_str = os.environ.get('CKAN_DATASTORE_WRITE_URL')
|
||||||
|
if not conn_str:
|
||||||
|
print('[prerun] Skipping datastore initialization')
|
||||||
|
return
|
||||||
|
|
||||||
|
datastore_perms_command = ['ckan', '-c', ckan_ini,
|
||||||
|
'datastore', 'set-permissions']
|
||||||
|
|
||||||
|
connection = psycopg2.connect(conn_str)
|
||||||
|
cursor = connection.cursor()
|
||||||
|
|
||||||
|
print('[prerun] Initializing datastore db - start')
|
||||||
|
try:
|
||||||
|
datastore_perms = subprocess.Popen(
|
||||||
|
datastore_perms_command,
|
||||||
|
stdout=subprocess.PIPE)
|
||||||
|
|
||||||
|
perms_sql = datastore_perms.stdout.read().decode('utf-8')
|
||||||
|
# Remove internal pg command as psycopg2 does not like it
|
||||||
|
perms_sql = re.sub('\\\\connect \"(.*)\"', '', perms_sql)
|
||||||
|
cursor.execute(perms_sql)
|
||||||
|
for notice in connection.notices:
|
||||||
|
print(notice)
|
||||||
|
|
||||||
|
connection.commit()
|
||||||
|
|
||||||
|
print('[prerun] Initializing datastore db - end')
|
||||||
|
print(datastore_perms.stdout.read().decode('utf-8'))
|
||||||
|
except psycopg2.Error as e:
|
||||||
|
print('[prerun] Could not initialize datastore')
|
||||||
|
print(str(e))
|
||||||
|
|
||||||
|
except subprocess.CalledProcessError as e:
|
||||||
|
if 'OperationalError' in e.output:
|
||||||
|
print(e.output)
|
||||||
|
print('[prerun] Database not ready, waiting a bit before exit...')
|
||||||
|
time.sleep(5)
|
||||||
|
sys.exit(1)
|
||||||
|
else:
|
||||||
|
print(e.output)
|
||||||
|
raise e
|
||||||
|
finally:
|
||||||
|
cursor.close()
|
||||||
|
connection.close()
|
||||||
|
|
||||||
|
|
||||||
|
def create_sysadmin():
|
||||||
|
|
||||||
|
name = os.environ.get('CKAN_SYSADMIN_NAME')
|
||||||
|
password = os.environ.get('CKAN_SYSADMIN_PASSWORD')
|
||||||
|
email = os.environ.get('CKAN_SYSADMIN_EMAIL')
|
||||||
|
|
||||||
|
if name and password and email:
|
||||||
|
|
||||||
|
# Check if user exists
|
||||||
|
command = ['ckan', '-c', ckan_ini, 'user', 'show', name,]
|
||||||
|
|
||||||
|
out = subprocess.check_output(command)
|
||||||
|
if 'User:None' not in re.sub(r'\s', '', out.decode()):
|
||||||
|
print('[prerun] Sysadmin user exists, skipping creation')
|
||||||
|
return
|
||||||
|
|
||||||
|
# Create user
|
||||||
|
command = ['ckan', '-c', ckan_ini, 'user', 'add',
|
||||||
|
name,
|
||||||
|
'password=' + password,
|
||||||
|
'email=' + email]
|
||||||
|
|
||||||
|
subprocess.call(command)
|
||||||
|
print('[prerun] Created user {0}'.format(name))
|
||||||
|
|
||||||
|
# Make it sysadmin
|
||||||
|
command = ['ckan', '-c', ckan_ini, 'sysadmin', 'add',
|
||||||
|
name]
|
||||||
|
|
||||||
|
subprocess.call(command)
|
||||||
|
print('[prerun] Made user {0} a sysadmin'.format(name))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
|
||||||
|
maintenance = os.environ.get('MAINTENANCE_MODE', '').lower() == 'true'
|
||||||
|
|
||||||
|
if maintenance:
|
||||||
|
print('[prerun] Maintenance mode, skipping setup...')
|
||||||
|
else:
|
||||||
|
check_main_db_connection()
|
||||||
|
init_db()
|
||||||
|
update_plugins()
|
||||||
|
check_datastore_db_connection()
|
||||||
|
init_datastore_db()
|
||||||
|
check_solr_connection()
|
||||||
|
create_sysadmin()
|
||||||
46
ckan-base/2.9/setup/start_ckan.sh
Executable file
46
ckan-base/2.9/setup/start_ckan.sh
Executable file
@ -0,0 +1,46 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
# Run the prerun script to init CKAN and create the default admin user
|
||||||
|
sudo -u ckan -EH python3 prerun.py
|
||||||
|
|
||||||
|
# Run any startup scripts provided by images extending this one
|
||||||
|
if [[ -d "/docker-entrypoint.d" ]]
|
||||||
|
then
|
||||||
|
for f in /docker-entrypoint.d/*; do
|
||||||
|
case "$f" in
|
||||||
|
*.sh) echo "$0: Running init file $f"; . "$f" ;;
|
||||||
|
*.py) echo "$0: Running init file $f"; python "$f"; echo ;;
|
||||||
|
*) echo "$0: Ignoring $f (not an sh or py file)" ;;
|
||||||
|
esac
|
||||||
|
echo
|
||||||
|
done
|
||||||
|
fi
|
||||||
|
|
||||||
|
|
||||||
|
# Check whether http basic auth password protection is enabled and enable basicauth routing on uwsgi respecfully
|
||||||
|
if [ $? -eq 0 ]
|
||||||
|
then
|
||||||
|
if [ "$PASSWORD_PROTECT" = true ]
|
||||||
|
then
|
||||||
|
if [ "$HTPASSWD_USER" ] || [ "$HTPASSWD_PASSWORD" ]
|
||||||
|
then
|
||||||
|
# Generate htpasswd file for basicauth
|
||||||
|
htpasswd -d -b -c /srv/app/.htpasswd $HTPASSWD_USER $HTPASSWD_PASSWORD
|
||||||
|
# Start supervisord
|
||||||
|
supervisord --configuration /etc/supervisord.conf &
|
||||||
|
# Start uwsgi with basicauth
|
||||||
|
sudo -u ckan -EH uwsgi -i ckan-uwsgi.ini
|
||||||
|
else
|
||||||
|
echo "Missing HTPASSWD_USER or HTPASSWD_PASSWORD environment variables. Exiting..."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
# Start supervisord
|
||||||
|
supervisord --configuration /etc/supervisord.conf &
|
||||||
|
# Start uwsgi
|
||||||
|
sudo -u ckan -EH uwsgi -i ckan-uwsgi.ini
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
echo "[prerun] failed...not starting CKAN."
|
||||||
|
fi
|
||||||
|
|
||||||
12
ckan-base/2.9/setup/supervisor.worker.conf
Normal file
12
ckan-base/2.9/setup/supervisor.worker.conf
Normal file
@ -0,0 +1,12 @@
|
|||||||
|
[program:ckan-worker]
|
||||||
|
command=ckan -c /srv/app/ckan.ini jobs worker
|
||||||
|
priority=501
|
||||||
|
autostart=true
|
||||||
|
autorestart=true
|
||||||
|
redirect_stderr=true
|
||||||
|
stdout_logfile=/dev/stdout
|
||||||
|
stdout_logfile_maxbytes=0
|
||||||
|
stderr_logfile=/dev/stdout
|
||||||
|
stderr_logfile_maxbytes=0
|
||||||
|
user=ckan
|
||||||
|
environment=HOME="/srv/app",USER="ckan"
|
||||||
23
ckan-base/2.9/setup/supervisord.conf
Normal file
23
ckan-base/2.9/setup/supervisord.conf
Normal file
@ -0,0 +1,23 @@
|
|||||||
|
[unix_http_server]
|
||||||
|
file = /tmp/supervisor.sock
|
||||||
|
chmod = 0777
|
||||||
|
chown = nobody:nogroup
|
||||||
|
|
||||||
|
[supervisord]
|
||||||
|
logfile = /tmp/supervisord.log
|
||||||
|
logfile_maxbytes = 50MB
|
||||||
|
logfile_backups=10
|
||||||
|
loglevel = info
|
||||||
|
pidfile = /tmp/supervisord.pid
|
||||||
|
nodaemon = true
|
||||||
|
umask = 022
|
||||||
|
identifier = supervisor
|
||||||
|
|
||||||
|
[supervisorctl]
|
||||||
|
serverurl = unix:///tmp/supervisor.sock
|
||||||
|
|
||||||
|
[rpcinterface:supervisor]
|
||||||
|
supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
|
||||||
|
|
||||||
|
[include]
|
||||||
|
files = /etc/supervisord.d/*.conf
|
||||||
9
ckan-base/2.9/setup/wsgi.py
Normal file
9
ckan-base/2.9/setup/wsgi.py
Normal file
@ -0,0 +1,9 @@
|
|||||||
|
import os
|
||||||
|
from ckan.config.middleware import make_app
|
||||||
|
from ckan.cli import CKANConfigLoader
|
||||||
|
from logging.config import fileConfig as loggingFileConfig
|
||||||
|
config_filepath = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'ckan.ini')
|
||||||
|
abspath = os.path.join(os.path.dirname(os.path.abspath(__file__)))
|
||||||
|
loggingFileConfig(config_filepath)
|
||||||
|
config = CKANConfigLoader(config_filepath).get_config()
|
||||||
|
application = make_app(config)
|
||||||
20
ckan-dev/2.7/Dockerfile
Executable file
20
ckan-dev/2.7/Dockerfile
Executable file
@ -0,0 +1,20 @@
|
|||||||
|
FROM ckan/ckan-base:testing-only.2.7
|
||||||
|
|
||||||
|
LABEL maintainer="brett@kowh.ai"
|
||||||
|
|
||||||
|
ENV APP_DIR=/srv/app
|
||||||
|
ENV SRC_EXTENSIONS_DIR=/srv/app/src_extensions
|
||||||
|
|
||||||
|
# Install packages needed by the dev requirements
|
||||||
|
RUN apk add --no-cache libffi-dev
|
||||||
|
|
||||||
|
# Install CKAN dev requirements
|
||||||
|
RUN pip install --no-binary :all: -r https://raw.githubusercontent.com/ckan/ckan/${GIT_BRANCH}/dev-requirements.txt
|
||||||
|
|
||||||
|
# Create folder for local extensions sources
|
||||||
|
RUN mkdir $SRC_EXTENSIONS_DIR
|
||||||
|
|
||||||
|
COPY setup/start_ckan_development.sh ${APP_DIR}
|
||||||
|
|
||||||
|
|
||||||
|
CMD ["/srv/app/start_ckan_development.sh"]
|
||||||
20
ckan-dev/2.8/Dockerfile
Executable file
20
ckan-dev/2.8/Dockerfile
Executable file
@ -0,0 +1,20 @@
|
|||||||
|
FROM ckan/ckan-base:testing-only.2.8
|
||||||
|
|
||||||
|
LABEL maintainer="brett@kowh.ai"
|
||||||
|
|
||||||
|
ENV APP_DIR=/srv/app
|
||||||
|
ENV SRC_EXTENSIONS_DIR=/srv/app/src_extensions
|
||||||
|
|
||||||
|
# Install packages needed by the dev requirements
|
||||||
|
RUN apk add --no-cache libffi-dev
|
||||||
|
|
||||||
|
# Install CKAN dev requirements
|
||||||
|
RUN pip install --no-binary :all: -r https://raw.githubusercontent.com/ckan/ckan/${GIT_BRANCH}/dev-requirements.txt
|
||||||
|
|
||||||
|
# Create folder for local extensions sources
|
||||||
|
RUN mkdir $SRC_EXTENSIONS_DIR
|
||||||
|
|
||||||
|
COPY setup/start_ckan_development.sh ${APP_DIR}
|
||||||
|
|
||||||
|
|
||||||
|
CMD ["/srv/app/start_ckan_development.sh"]
|
||||||
29
ckan-dev/2.9/Dockerfile
Executable file
29
ckan-dev/2.9/Dockerfile
Executable file
@ -0,0 +1,29 @@
|
|||||||
|
FROM ckan/ckan-base:testing-only.2.9
|
||||||
|
|
||||||
|
LABEL maintainer="brett@kowh.ai"
|
||||||
|
|
||||||
|
ENV APP_DIR=/srv/app
|
||||||
|
ENV SRC_EXTENSIONS_DIR=/srv/app/src_extensions
|
||||||
|
|
||||||
|
# Install packages needed by the dev requirements
|
||||||
|
RUN apk add --no-cache libffi-dev
|
||||||
|
|
||||||
|
# Set up Python3 virtual environment
|
||||||
|
RUN cd ${APP_DIR} && \
|
||||||
|
source ${APP_DIR}/bin/activate
|
||||||
|
|
||||||
|
# Virtual environment binaries/scripts to be used first
|
||||||
|
ENV PATH=${APP_DIR}/bin:${PATH}
|
||||||
|
|
||||||
|
# Install CKAN dev requirements
|
||||||
|
# Will need to change this eventually - when CKAN 2.9 is out
|
||||||
|
# wget https://raw.githubusercontent.com/ckan/ckan/master/dev-requirements.txt
|
||||||
|
# RUN pip3 install --no-binary :all: -r https://raw.githubusercontent.com/ckan/ckan/master/dev-requirements.txt
|
||||||
|
RUN pip3 install -r https://raw.githubusercontent.com/ckan/ckan/master/dev-requirements.txt
|
||||||
|
|
||||||
|
# Create folder for local extensions sources
|
||||||
|
RUN mkdir $SRC_EXTENSIONS_DIR
|
||||||
|
|
||||||
|
COPY setup/start_ckan_development.sh ${APP_DIR}
|
||||||
|
|
||||||
|
CMD ["/srv/app/start_ckan_development.sh"]
|
||||||
82
ckan-dev/setup/start_ckan_development.sh
Executable file
82
ckan-dev/setup/start_ckan_development.sh
Executable file
@ -0,0 +1,82 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
# Install any local extensions in the src_extensions volume
|
||||||
|
echo "Looking for local extensions to install..."
|
||||||
|
echo "Extension dir contents:"
|
||||||
|
ls -la $SRC_EXTENSIONS_DIR
|
||||||
|
for i in $SRC_EXTENSIONS_DIR/*
|
||||||
|
do
|
||||||
|
if [ -d $i ];
|
||||||
|
then
|
||||||
|
|
||||||
|
if [ -f $i/pip-requirements.txt ];
|
||||||
|
then
|
||||||
|
pip install -r $i/pip-requirements.txt
|
||||||
|
echo "Found requirements file in $i"
|
||||||
|
fi
|
||||||
|
if [ -f $i/requirements.txt ];
|
||||||
|
then
|
||||||
|
pip install -r $i/requirements.txt
|
||||||
|
echo "Found requirements file in $i"
|
||||||
|
fi
|
||||||
|
if [ -f $i/dev-requirements.txt ];
|
||||||
|
then
|
||||||
|
pip install -r $i/dev-requirements.txt
|
||||||
|
echo "Found dev-requirements file in $i"
|
||||||
|
fi
|
||||||
|
if [ -f $i/setup.py ];
|
||||||
|
then
|
||||||
|
cd $i
|
||||||
|
python $i/setup.py develop
|
||||||
|
echo "Found setup.py file in $i"
|
||||||
|
cd $APP_DIR
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Point `use` in test.ini to location of `test-core.ini`
|
||||||
|
if [ -f $i/test.ini ];
|
||||||
|
then
|
||||||
|
echo "Updating \`test.ini\` reference to \`test-core.ini\` for plugin $i"
|
||||||
|
paster --plugin=ckan config-tool $i/test.ini "use = config:../../src/ckan/test-core.ini"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
# Set debug to true
|
||||||
|
echo "Enabling debug mode"
|
||||||
|
ckan config-tool $CKAN_INI -s DEFAULT "debug = true"
|
||||||
|
|
||||||
|
# Update the plugins setting in the ini file with the values defined in the env var
|
||||||
|
echo "Loading the following plugins: $CKAN__PLUGINS"
|
||||||
|
ckan config-tool $CKAN_INI "ckan.plugins = $CKAN__PLUGINS"
|
||||||
|
|
||||||
|
# Update test-core.ini DB, SOLR & Redis settings
|
||||||
|
echo "Loading test settings into test-core.ini"
|
||||||
|
ckan config-tool $SRC_DIR/ckan/test-core.ini \
|
||||||
|
"sqlalchemy.url = $TEST_CKAN_SQLALCHEMY_URL" \
|
||||||
|
"ckan.datstore.write_url = $TEST_CKAN_DATASTORE_WRITE_URL" \
|
||||||
|
"ckan.datstore.read_url = $TEST_CKAN_DATASTORE_READ_URL" \
|
||||||
|
"solr_url = $TEST_CKAN_SOLR_URL" \
|
||||||
|
"ckan.redis_url = $TEST_CKAN_REDIS_URL"
|
||||||
|
|
||||||
|
# Run the prerun script to init CKAN and create the default admin user
|
||||||
|
sudo -u ckan -EH python prerun.py
|
||||||
|
|
||||||
|
# Run any startup scripts provided by images extending this one
|
||||||
|
if [[ -d "/docker-entrypoint.d" ]]
|
||||||
|
then
|
||||||
|
for f in /docker-entrypoint.d/*; do
|
||||||
|
case "$f" in
|
||||||
|
*.sh) echo "$0: Running init file $f"; . "$f" ;;
|
||||||
|
*.py) echo "$0: Running init file $f"; python "$f"; echo ;;
|
||||||
|
*) echo "$0: Ignoring $f (not an sh or py file)" ;;
|
||||||
|
esac
|
||||||
|
echo
|
||||||
|
done
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Start supervisord
|
||||||
|
supervisord --configuration /etc/supervisord.conf &
|
||||||
|
|
||||||
|
# Start the development server with automatic reload
|
||||||
|
# Check the --reloader options sudo -u ckan -EH ckan -c $CKAN_INI run --reloader <TEXT>
|
||||||
|
sudo -u ckan -EH ckan -c $CKAN_INI run
|
||||||
Loading…
x
Reference in New Issue
Block a user