diff --git a/.env b/.env index a3d9300..29e9bcf 100644 --- a/.env +++ b/.env @@ -28,18 +28,25 @@ CKAN_SMTP_PASSWORD=pass CKAN_SMTP_MAIL_FROM=ckan@localhost TZ=UTC -# Other services +# Solr +SOLR_VERSION=6.6.6 CKAN_SOLR_URL=http://solr:8983/solr/ckan -CKAN_REDIS_URL=redis://redis:6379/1 -CKAN_DATAPUSHER_URL=http://datapusher:8801 -CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://ckan:5000 - -# test connections TEST_CKAN_SOLR_URL=http://solr:8983/solr/ckan + +# Redis +REDIS_VERSION=6.0.7 +CKAN_REDIS_URL=redis://redis:6379/1 TEST_CKAN_REDIS_URL=redis://redis:6379/1 +# Datapusher +DATAPUSHER_VERSION=0.0.17 +CKAN_DATAPUSHER_URL=http://datapusher:8800 +CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://ckan:5000 +DATAPUSHER_REWRITE_RESOURCES=True +DATAPUSHER_REWRITE_URL=http://ckan:5000 + # Extensions -CKAN__PLUGINS=envvars image_view text_view recline_view datastore datapusher +CKAN__PLUGINS="envvars image_view text_view recline_view datastore datapusher" CKAN__HARVEST__MQ__TYPE=redis CKAN__HARVEST__MQ__HOSTNAME=redis CKAN__HARVEST__MQ__PORT=6379 diff --git a/.env.template b/.env.template index a3d9300..864d5d4 100644 --- a/.env.template +++ b/.env.template @@ -28,16 +28,23 @@ CKAN_SMTP_PASSWORD=pass CKAN_SMTP_MAIL_FROM=ckan@localhost TZ=UTC -# Other services +# Solr +SOLR_VERSION=6.6.6 CKAN_SOLR_URL=http://solr:8983/solr/ckan -CKAN_REDIS_URL=redis://redis:6379/1 -CKAN_DATAPUSHER_URL=http://datapusher:8801 -CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://ckan:5000 - -# test connections TEST_CKAN_SOLR_URL=http://solr:8983/solr/ckan + +# Redis +REDIS_VERSION=6.0.7 +CKAN_REDIS_URL=redis://redis:6379/1 TEST_CKAN_REDIS_URL=redis://redis:6379/1 +# Datapusher +DATAPUSHER_VERSION=0.0.17 +CKAN_DATAPUSHER_URL=http://datapusher:8800 +CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://ckan:5000 +DATAPUSHER_REWRITE_RESOURCES=True +DATAPUSHER_REWRITE_URL=http://ckan:5000 + # Extensions CKAN__PLUGINS=envvars image_view text_view recline_view datastore datapusher CKAN__HARVEST__MQ__TYPE=redis diff --git a/.gitignore b/.gitignore index 1d88d85..06c7c30 100755 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ _service-provider/* _solr/schema.xml _src/* +local/* diff --git a/README.txt b/README.txt index 90e45fd..022f23c 100644 --- a/README.txt +++ b/README.txt @@ -1,24 +1,33 @@ ### Please note that this re-purposed CKAN Docker repo is a WORK IN PROGRESS ### ### It should not be used to install CKAN via Docker until this page is updated ### -- Maybe use build/up time variables loaded via the .env file, runtime variables loaded via a .ckan-env file - both -located at the root directory -- Look at using ghcr.io (GitHub Packages) to store Docker Images rather than DockerHub -- all username/passwords as environment variables rather than hardcoded -- make sure the "development mode" path is taken cared of with any change -- Create an admin user during the container deployment -- should there be a datapusher image pre-built like ckan? maybe an xloader image? +------------------------------------- +(From Adria) +Be limited in scope, and act as a base that users can extend to their own needs +Be opinionated, and provide one way to do things +Be automatically tested +------------------------------------- +should we have an nginx container included? maybe just instructions on how to include one -### Difference between ckan-base and ckan-dev ### +ARG should be used for sensitive variables so as to keep the values of these variables +out of the build and hence will not show with an "inspect image" command -ckan-base -docker-compose up -d --build -docker-compose build -docker-compose up +Do we pre-build a CKAN image and use that (and extend) as the base image OR + just build it from scratch and while saving to storage what we need to -ckan-dev -docker-compose -f docker-compose.dev.yml up -d --build -docker-compose -f docker-compose.dev.yml build -docker-compose -f docker-compose.dev.yml up +To use local storage for the ckan.ini file and the CKAN src code, do the following: +# docker-compose up -d --build +# mkdir local +# docker cp ckan:/srv/app/ckan.ini ./local/ckan.ini +# docker cp ckan:/srv/app/src ./local/src +Stop/Remove ckan container +Start ckan container as follows +Use a bind mount for the config file (ckan.ini) +# docker run -p 0.0.0.0:5000:5000 --net ckan-docker_default --hostname ckan --name ckan \ + --env-file=./environ --mount type=bind,source=$(pwd)/local/ckan.ini,target=/srv/app/ckan.ini \ + --mount type=bind,source=$(pwd)/local/src,target=/srv/app/src \ + -d ckan-docker_ckan + +Maybe include a script to replace docker-compose if required diff --git a/docker-compose.yml b/docker-compose.yml index b2f5aa1..0d8dffa 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -33,7 +33,7 @@ services: datapusher: container_name: datapusher - image: kowhai/datapusher:0.0.17 + image: kowhai/datapusher:${DATAPUSHER_VERSION} ports: - "8800:8800" @@ -60,7 +60,7 @@ services: redis: container_name: redis - image: redis:alpine + image: redis:${REDIS_VERSION} volumes: ckan_storage: diff --git a/environ b/environ new file mode 100644 index 0000000..df4e7f3 --- /dev/null +++ b/environ @@ -0,0 +1,34 @@ +REDIS_CONTAINERNAME=redis +REDIS_HOSTNAME=redis + +SOLR_CONTAINERNAME=solr +SOLR_HOSTNAME=solr +SOLR_IMAGENAME=solr + +DB_CONTAINERNAME=db +DB_HOSTNAME=db +DB_IMAGENAME=db +POSTGRES_PASSWORD=ckan +DS_RO_PASS=datastore +PGDATA=/var/lib/postgresql/data/db + +CKAN_CONTAINERNAME=ckan +CKAN_HOSTNAME=ckan +CKAN_IMAGENAME=ckan + +CKAN_SQLALCHEMY_URL=postgresql://ckan:ckan@db/ckan +CKAN_DATASTORE_WRITE_URL=postgresql://ckan:ckan@db/datastore +CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:datastore@db/datastore +CKAN_SOLR_URL=http://solr:8983/solr/ckan +CKAN_REDIS_URL=redis://redis:6379/1 +CKAN_SITE_URL=http://localhost +CKAN_MAX_UPLOAD_SIZE_MB=32 +POSTGRES_PASSWORD=ckan +DS_RO_PASS=datastore +CKAN_PORT=5000 +CKAN_DATAPUSHER_URL=http://datapusher:8800 +CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://ckan:5000 + +NGINX_CONTAINERNAME=nginx +NGINX_HOSTNAME=nginx +NGINX_IMAGENAME=nginx diff --git a/images/README b/images/README index 44e18b4..ac994c2 100755 --- a/images/README +++ b/images/README @@ -1,4 +1,13 @@ Build the images from this directory using: + cd ckan-base/ + docker build -t ckan/ckan-base:testing-only.2.9 -f 2.9/Dockerfile . + docker build -t ckan/ckan-base:testing-only.2.8 -f 2.8/Dockerfile . + docker build -t ckan/ckan-base:testing-only.2.7 -f 2.7/Dockerfile . - docker build -t openknowledge/ckan-dev:2.7 -f 2.7/Dockerfile . - docker build -t openknowledge/ckan-dev:2.8 -f 2.8/Dockerfile . +Login to Dockerhub +docker login --username=kowhai +(will then ask for a password) + +Push image to (ckan) Dockerhub +docker push ckan/ckan-base:testing-only.2.9 + diff --git a/images/ckan-base/2.7/Dockerfile b/images/ckan-base/2.7/Dockerfile index f3b7099..83b33c9 100755 --- a/images/ckan-base/2.7/Dockerfile +++ b/images/ckan-base/2.7/Dockerfile @@ -8,7 +8,7 @@ ENV PIP_SRC=${SRC_DIR} ENV CKAN_STORAGE_PATH=/var/lib/ckan ENV GIT_URL=https://github.com/ckan/ckan.git # CKAN version to build -ENV GIT_BRANCH=ckan-2.7.8 +ENV GIT_BRANCH=ckan-2.7.12 # Customize these on the .env file if needed ENV CKAN_SITE_URL=http://localhost:5000 ENV CKAN__PLUGINS image_view text_view recline_view datastore datapusher envvars @@ -46,11 +46,11 @@ RUN apk add --no-cache tzdata \ python-dev \ libxml2-dev \ libxslt-dev \ - linux-headers && \ + linux-headers # Create SRC_DIR - mkdir -p ${SRC_DIR} && \ + RUN mkdir -p ${SRC_DIR} && \ # Install pip, supervisord and uwsgi - curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/2.7/get-pip.py && \ + curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/pip/2.7/get-pip.py && \ python ${SRC_DIR}/get-pip.py 'pip==20.3.3' && \ pip install supervisor && \ mkdir /etc/supervisord.d && \ @@ -81,9 +81,11 @@ RUN addgroup -g 92 -S ckan && \ RUN mkdir -p $CKAN_STORAGE_PATH && \ chown -R ckan:ckan $CKAN_STORAGE_PATH -COPY 2.7/setup ${APP_DIR} -COPY 2.7/setup/supervisor.worker.conf /etc/supervisord.d/worker.conf -COPY 2.7/setup/uwsgi.conf /srv/app/uwsgi.conf +COPY 2.7/setup/ckan-uwsgi.ini ${APP_DIR} +COPY 2.7/setup/wsgi.py ${APP_DIR} +COPY 2.7/setup/prerun.py ${APP_DIR} +COPY 2.7/setup/start_ckan.sh ${APP_DIR} +COPY 2.7/setup/supervisord.conf /etc/supervisord.conf # Create entrypoint directory for children image scripts ONBUILD RUN mkdir /docker-entrypoint.d diff --git a/images/ckan-base/2.7/setup/ckan-uwsgi.ini b/images/ckan-base/2.7/setup/ckan-uwsgi.ini new file mode 100644 index 0000000..2361f36 --- /dev/null +++ b/images/ckan-base/2.7/setup/ckan-uwsgi.ini @@ -0,0 +1,15 @@ +[uwsgi] +http-socket = :5000 +uid = ckan +guid = ckan +plugins = python3 +wsgi-file = /srv/app/wsgi.py +virtualenv = /srv/app +module = wsgi:application +master = true +processes = 5 +pidfile = /tmp/%n.pid +harakiri = 50 +max-requests = 5000 +vacuum = true +callable = application diff --git a/images/ckan-base/2.7/setup/supervisor.worker.conf b/images/ckan-base/2.7/setup/supervisor.worker.conf deleted file mode 100755 index 0ca43ba..0000000 --- a/images/ckan-base/2.7/setup/supervisor.worker.conf +++ /dev/null @@ -1,13 +0,0 @@ -[program:ckan-worker] -command=paster --plugin=ckan jobs worker -c /srv/app/production.ini -priority=501 -autostart=true -autorestart=true -redirect_stderr=true -stdout_logfile=/dev/stdout -stdout_logfile_maxbytes=0 -stderr_logfile=/dev/stdout -stderr_logfile_maxbytes=0 -user=ckan -environment=HOME="/srv/app",USER="ckan" - diff --git a/images/ckan-base/2.7/setup/wsgi.py b/images/ckan-base/2.7/setup/wsgi.py new file mode 100644 index 0000000..b37d80e --- /dev/null +++ b/images/ckan-base/2.7/setup/wsgi.py @@ -0,0 +1,9 @@ +import os +from ckan.config.middleware import make_app +from ckan.cli import CKANConfigLoader +from logging.config import fileConfig as loggingFileConfig +config_filepath = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'ckan.ini') +abspath = os.path.join(os.path.dirname(os.path.abspath(__file__))) +loggingFileConfig(config_filepath) +config = CKANConfigLoader(config_filepath).get_config() +application = make_app(config) diff --git a/images/ckan-base/2.8/Dockerfile b/images/ckan-base/2.8/Dockerfile index 48847e7..e2cec51 100755 --- a/images/ckan-base/2.8/Dockerfile +++ b/images/ckan-base/2.8/Dockerfile @@ -8,7 +8,7 @@ ENV PIP_SRC=${SRC_DIR} ENV CKAN_STORAGE_PATH=/var/lib/ckan ENV GIT_URL=https://github.com/ckan/ckan.git # CKAN version to build -ENV GIT_BRANCH=ckan-2.8.5 +ENV GIT_BRANCH=ckan-2.8.9 # Customize these on the .env file if needed ENV CKAN_SITE_URL=http://localhost:5000 ENV CKAN__PLUGINS image_view text_view recline_view datastore datapusher envvars @@ -46,11 +46,11 @@ RUN apk add --no-cache tzdata \ python-dev \ libxml2-dev \ libxslt-dev \ - linux-headers && \ + linux-headers # Create SRC_DIR - mkdir -p ${SRC_DIR} && \ + RUN mkdir -p ${SRC_DIR} && \ # Install pip, supervisord and uwsgi - curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/2.7/get-pip.py && \ + curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/pip/2.7/get-pip.py && \ python ${SRC_DIR}/get-pip.py 'pip==20.3.3' && \ pip install supervisor && \ mkdir /etc/supervisord.d && \ @@ -79,9 +79,11 @@ RUN addgroup -g 92 -S ckan && \ RUN mkdir -p $CKAN_STORAGE_PATH && \ chown -R ckan:ckan $CKAN_STORAGE_PATH -COPY 2.8/setup ${APP_DIR} -COPY 2.8/setup/supervisor.worker.conf /etc/supervisord.d/worker.conf -COPY 2.8/setup/uwsgi.conf /srv/app/uwsgi.conf +COPY 2.8/setup/ckan-uwsgi.ini ${APP_DIR} +COPY 2.8/setup/wsgi.py ${APP_DIR} +COPY 2.8/setup/prerun.py ${APP_DIR} +COPY 2.8/setup/start_ckan.sh ${APP_DIR} +COPY 2.8/setup/supervisord.conf /etc/supervisord.conf # Create entrypoint directory for children image scripts ONBUILD RUN mkdir /docker-entrypoint.d diff --git a/images/ckan-base/2.8/setup/ckan-uwsgi.ini b/images/ckan-base/2.8/setup/ckan-uwsgi.ini new file mode 100644 index 0000000..2361f36 --- /dev/null +++ b/images/ckan-base/2.8/setup/ckan-uwsgi.ini @@ -0,0 +1,15 @@ +[uwsgi] +http-socket = :5000 +uid = ckan +guid = ckan +plugins = python3 +wsgi-file = /srv/app/wsgi.py +virtualenv = /srv/app +module = wsgi:application +master = true +processes = 5 +pidfile = /tmp/%n.pid +harakiri = 50 +max-requests = 5000 +vacuum = true +callable = application diff --git a/images/ckan-base/2.8/setup/supervisor.worker.conf b/images/ckan-base/2.8/setup/supervisor.worker.conf deleted file mode 100755 index 0ca43ba..0000000 --- a/images/ckan-base/2.8/setup/supervisor.worker.conf +++ /dev/null @@ -1,13 +0,0 @@ -[program:ckan-worker] -command=paster --plugin=ckan jobs worker -c /srv/app/production.ini -priority=501 -autostart=true -autorestart=true -redirect_stderr=true -stdout_logfile=/dev/stdout -stdout_logfile_maxbytes=0 -stderr_logfile=/dev/stdout -stderr_logfile_maxbytes=0 -user=ckan -environment=HOME="/srv/app",USER="ckan" - diff --git a/images/ckan-base/2.8/setup/wsgi.py b/images/ckan-base/2.8/setup/wsgi.py new file mode 100644 index 0000000..b37d80e --- /dev/null +++ b/images/ckan-base/2.8/setup/wsgi.py @@ -0,0 +1,9 @@ +import os +from ckan.config.middleware import make_app +from ckan.cli import CKANConfigLoader +from logging.config import fileConfig as loggingFileConfig +config_filepath = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'ckan.ini') +abspath = os.path.join(os.path.dirname(os.path.abspath(__file__))) +loggingFileConfig(config_filepath) +config = CKANConfigLoader(config_filepath).get_config() +application = make_app(config) diff --git a/images/ckan-base/2.9/Dockerfile b/images/ckan-base/2.9/Dockerfile index d2665cd..a0d28d4 100755 --- a/images/ckan-base/2.9/Dockerfile +++ b/images/ckan-base/2.9/Dockerfile @@ -1,4 +1,4 @@ -FROM alpine:3.7 +FROM alpine:3.13 # Internal environment variables ENV APP_DIR=/srv/app @@ -8,7 +8,7 @@ ENV PIP_SRC=${SRC_DIR} ENV CKAN_STORAGE_PATH=/var/lib/ckan ENV GIT_URL=https://github.com/ckan/ckan.git # CKAN version to build -ENV GIT_BRANCH=ckan-2.9.0 +ENV GIT_BRANCH=ckan-2.9.4 # Customize these on the .env file if needed ENV CKAN_SITE_URL=http://localhost:5000 ENV CKAN__PLUGINS image_view text_view recline_view datastore datapusher envvars @@ -46,12 +46,13 @@ RUN apk add --no-cache tzdata \ py3-virtualenv \ libxml2-dev \ libxslt-dev \ - linux-headers && \ - # Create SRC_DIR - mkdir -p ${SRC_DIR} && \ + linux-headers + +# Create SRC_DIR and install pip +RUN mkdir -p ${SRC_DIR} && \ # Install pip - curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/2.7/get-pip.py && \ - python3 ${SRC_DIR}/get-pip.py 'pip==20.3.3' && \ + curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/get-pip.py && \ + python3 ${SRC_DIR}/get-pip.py && \ rm -rf ${SRC_DIR}/get-pip.py # Set up Python3 virtual environment @@ -62,12 +63,13 @@ RUN cd ${APP_DIR} && \ # Virtual environment binaries/scripts to be used first ENV PATH=${APP_DIR}/bin:${PATH} -# Install CKAN and uwsgi +# Install CKAN, uwsgi, ckanext-envvars and generate CKAN config file RUN pip3 install -e git+${GIT_URL}@${GIT_BRANCH}#egg=ckan && \ pip3 install uwsgi && \ cd ${SRC_DIR}/ckan && \ cp who.ini ${APP_DIR} && \ - pip install --no-binary :all: -r requirements.txt && \ + pip3 install -r requirement-setuptools.txt && \ + pip3 install --no-binary :all: -r requirements.txt && \ # Install CKAN envvars to support loading config from environment variables pip3 install -e git+https://github.com/okfn/ckanext-envvars.git#egg=ckanext-envvars && \ # Create and update CKAN config @@ -78,8 +80,10 @@ RUN pip3 install supervisor && \ mkdir /etc/supervisord.d # Copy all setup files -COPY 2.9/setup ${APP_DIR} -COPY 2.9/setup/supervisor.worker.conf /etc/supervisord.d/worker.conf +COPY 2.9/setup/ckan-uwsgi.ini ${APP_DIR} +COPY 2.9/setup/wsgi.py ${APP_DIR} +COPY 2.9/setup/prerun.py ${APP_DIR} +COPY 2.9/setup/start_ckan.sh ${APP_DIR} COPY 2.9/setup/supervisord.conf /etc/supervisord.conf # Create a local user and group to run the app diff --git a/images/ckan-base/2.9/setup/supervisor.worker.conf b/images/ckan-base/2.9/setup/supervisor.worker.conf deleted file mode 100644 index 9d46f37..0000000 --- a/images/ckan-base/2.9/setup/supervisor.worker.conf +++ /dev/null @@ -1,12 +0,0 @@ -[program:ckan-worker] -command=ckan -c /srv/app/ckan.ini jobs worker -priority=501 -autostart=true -autorestart=true -redirect_stderr=true -stdout_logfile=/dev/stdout -stdout_logfile_maxbytes=0 -stderr_logfile=/dev/stdout -stderr_logfile_maxbytes=0 -user=ckan -environment=HOME="/srv/app",USER="ckan" \ No newline at end of file diff --git a/local/ckan.ini b/local/ckan.ini new file mode 100644 index 0000000..90721b8 --- /dev/null +++ b/local/ckan.ini @@ -0,0 +1,252 @@ +# +# CKAN - Pylons configuration +# +# These are some of the configuration options available for your CKAN +# instance. Check the documentation in 'doc/configuration.rst' or at the +# following URL for a description of what they do and the full list of +# available options: +# +# http://docs.ckan.org/en/latest/maintaining/configuration.html +# +# The %(here)s variable will be replaced with the parent directory of this file +# + +[DEFAULT] + +# WARNING: *THIS SETTING MUST BE SET TO FALSE ON A PUBLIC ENVIRONMENT* +# With debug mode enabled, a visitor to your site could execute malicious commands. +debug = false + +[app:main] +use = egg:ckan + +## Development settings +ckan.devserver.host = localhost +ckan.devserver.port = 5000 + + +## Session settings +cache_dir = /tmp/%(ckan.site_id)s/ +beaker.session.key = ckan + +# This is the secret token that the beaker library uses to hash the cookie sent +# to the client. `ckan generate config` generates a unique value for this each +# time it generates a config file. +beaker.session.secret = EHU6ef6rWzspiSQutl2F4G28n + +# `ckan generate config` generates a unique value for this each time it generates +# a config file. +app_instance_uuid = 52ba6253-cf07-4f0b-ba3f-1c685b4a708c + +# repoze.who config +who.config_file = %(here)s/who.ini +who.log_level = warning +who.log_file = %(cache_dir)s/who_log.ini +# Session timeout (user logged out after period of inactivity, in seconds). +# Inactive by default, so the session doesn't expire. +# who.timeout = 86400 + +## Database Settings +sqlalchemy.url = postgresql://ckan_default:pass@localhost/ckan_default + +#ckan.datastore.write_url = postgresql://ckan_default:pass@localhost/datastore_default +#ckan.datastore.read_url = postgresql://datastore_default:pass@localhost/datastore_default + +# PostgreSQL' full-text search parameters +ckan.datastore.default_fts_lang = english +ckan.datastore.default_fts_index_method = gist + + +## Site Settings + +ckan.site_url = +#ckan.use_pylons_response_cleanup_middleware = true + +## Authorization Settings + +ckan.auth.anon_create_dataset = false +ckan.auth.create_unowned_dataset = false +ckan.auth.create_dataset_if_not_in_organization = false +ckan.auth.user_create_groups = false +ckan.auth.user_create_organizations = false +ckan.auth.user_delete_groups = true +ckan.auth.user_delete_organizations = true +ckan.auth.create_user_via_api = false +ckan.auth.create_user_via_web = true +ckan.auth.roles_that_cascade_to_sub_groups = admin +ckan.auth.public_user_details = true +ckan.auth.public_activity_stream_detail = true +ckan.auth.allow_dataset_collaborators = false +ckan.auth.create_default_api_keys = false + +## API Token Settings +api_token.nbytes = 60 +api_token.jwt.encode.secret = string:EHU6ef6rWzspiSQutl2F4G28n +api_token.jwt.decode.secret = string:EHU6ef6rWzspiSQutl2F4G28n +api_token.jwt.algorithm = HS256 + +## API Token: expire_api_token plugin +expire_api_token.default_lifetime = 3600 + +## Search Settings + +ckan.site_id = default +#solr_url = http://127.0.0.1:8983/solr + + +## Redis Settings + +# URL to your Redis instance, including the database to be used. +#ckan.redis.url = redis://localhost:6379/0 + + +## CORS Settings + +# If cors.origin_allow_all is true, all origins are allowed. +# If false, the cors.origin_whitelist is used. +# ckan.cors.origin_allow_all = true +# cors.origin_whitelist is a space separated list of allowed domains. +# ckan.cors.origin_whitelist = http://example1.com http://example2.com + + +## Plugins Settings + +# Note: Add ``datastore`` to enable the CKAN DataStore +# Add ``datapusher`` to enable DataPusher +# Add ``resource_proxy`` to enable resorce proxying and get around the +# same origin policy +ckan.plugins = image_view text_view recline_view datastore datapusher envvars + +# Define which views should be created by default +# (plugins must be loaded in ckan.plugins) +ckan.views.default_views = image_view text_view recline_view + +# Customize which text formats the text_view plugin will show +#ckan.preview.json_formats = json +#ckan.preview.xml_formats = xml rdf rdf+xml owl+xml atom rss +#ckan.preview.text_formats = text plain text/plain + +# Customize which image formats the image_view plugin will show +#ckan.preview.image_formats = png jpeg jpg gif + +## Front-End Settings + +ckan.site_title = CKAN +ckan.site_logo = /base/images/ckan-logo.png +ckan.site_description = +ckan.favicon = /base/images/ckan.ico +ckan.gravatar_default = identicon +ckan.preview.direct = png jpg gif +ckan.preview.loadable = html htm rdf+xml owl+xml xml n3 n-triples turtle plain atom csv tsv rss txt json +ckan.display_timezone = server + +# package_hide_extras = for_search_index_only +#package_edit_return_url = http://another.frontend/dataset/ +#package_new_return_url = http://another.frontend/dataset/ +#ckan.recaptcha.publickey = +#ckan.recaptcha.privatekey = +#licenses_group_url = http://licenses.opendefinition.org/licenses/groups/ckan.json +# ckan.template_footer_end = + + +## Internationalisation Settings +ckan.locale_default = en +ckan.locale_order = en pt_BR ja it cs_CZ ca es fr el sv sr sr@latin no sk fi ru de pl nl bg ko_KR hu sa sl lv +ckan.locales_offered = +ckan.locales_filtered_out = en_GB + +## Feeds Settings + +ckan.feeds.authority_name = +ckan.feeds.date = +ckan.feeds.author_name = +ckan.feeds.author_link = + +## Storage Settings + +#ckan.storage_path = /var/lib/ckan +#ckan.max_resource_size = 10 +#ckan.max_image_size = 2 + +## Webassets Settings +#ckan.webassets.use_x_sendfile = false +#ckan.webassets.path = /var/lib/ckan/webassets + + +## Datapusher settings + +# Make sure you have set up the DataStore + +#ckan.datapusher.formats = csv xls xlsx tsv application/csv application/vnd.ms-excel application/vnd.openxmlformats-officedocument.spreadsheetml.sheet +#ckan.datapusher.url = http://127.0.0.1:8800/ +#ckan.datapusher.assume_task_stale_after = 3600 + +# Resource Proxy settings +# Preview size limit, default: 1MB +#ckan.resource_proxy.max_file_size = 1048576 +# Size of chunks to read/write. +#ckan.resource_proxy.chunk_size = 4096 + +## Activity Streams Settings + +#ckan.activity_streams_enabled = true +#ckan.activity_list_limit = 31 +#ckan.activity_streams_email_notifications = true +#ckan.email_notifications_since = 2 days +ckan.hide_activity_from_users = %(ckan.site_id)s + + +## Email settings + +#email_to = errors@example.com +#error_email_from = ckan-errors@example.com +#smtp.server = localhost +#smtp.starttls = False +#smtp.user = username@example.com +#smtp.password = your_password +#smtp.mail_from = +#smtp.reply_to = + +## Background Job Settings +ckan.jobs.timeout = 180 + +## Logging configuration +[loggers] +keys = root, ckan, ckanext, werkzeug + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARNING +handlers = console + +[logger_werkzeug] +level = WARNING +handlers = console +qualname = werkzeug +propagate = 0 + +[logger_ckan] +level = INFO +handlers = console +qualname = ckan +propagate = 0 + +[logger_ckanext] +level = DEBUG +handlers = console +qualname = ckanext +propagate = 0 + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(asctime)s %(levelname)-5.5s [%(name)s] %(message)s diff --git a/nginx/Dockerfile b/nginx/Dockerfile index a62d0d3..8bdcef2 100644 --- a/nginx/Dockerfile +++ b/nginx/Dockerfile @@ -2,7 +2,7 @@ FROM nginx:alpine ENV NGINX_DIR=/etc/nginx -COPY index.html /usr/share/nginx/html/index.html +COPY setup/index.html /usr/share/nginx/html/index.html RUN mkdir -p ${NGINX_DIR}/sites-available RUN mkdir -p ${NGINX_DIR}/sites-enabled @@ -12,4 +12,4 @@ COPY setup/sites-available/* ${NGINX_DIR}/sites-available RUN ln -s ${NGINX_DIR}/sites-available/ckan.conf ${NGINX_DIR}/sites-enabled/ckan.conf -EXPOSE 80 \ No newline at end of file +EXPOSE 80 diff --git a/nginx/index.html b/nginx/setup/index.html similarity index 100% rename from nginx/index.html rename to nginx/setup/index.html diff --git a/not-compose.sh b/not-compose.sh new file mode 100755 index 0000000..791bb28 --- /dev/null +++ b/not-compose.sh @@ -0,0 +1,71 @@ +#!/bin/bash + +# +# Create the network first +# +docker network create ckan-net +# + + +# Redis - Run only (image already built) +# +REDIS_CONTAINERNAME=redis +REDIS_HOSTNAME=redis + +docker run -d -it --net ckan-net --name ${REDIS_CONTAINERNAME} --hostname ${REDIS_HOSTNAME} redis:latest + +# +# Solr - Run/Build +# +SOLR_CONTAINERNAME=solr +SOLR_HOSTNAME=solr +SOLR_IMAGENAME=solr + +cd solr +docker image build -t ${SOLR_IMAGENAME} . +docker run -d -it --net ckan-net --name ${SOLR_CONTAINERNAME} --hostname ${SOLR_HOSTNAME} \ + -v solr_core:/opt/solr/server/solr ${SOLR_IMAGENAME} +cd - + +# +# PostgreSQL (DB) - Run/Build +# +DB_CONTAINERNAME=db +DB_HOSTNAME=db +DB_IMAGENAME=db + +cd postgresql +docker image build -t ${DB_IMAGENAME} . +docker run -d -it --net ckan-net --name ${DB_CONTAINERNAME} --hostname ${DB_HOSTNAME} --env-file=../environ \ + -v pg_data:/var/lib/postgresql/data ${DB_IMAGENAME} +cd - + +# +# CKAN - Run/Build +# +CKAN_CONTAINERNAME=ckan +CKAN_HOSTNAME=ckan +CKAN_IMAGENAME=ckan + +cd ckan +docker image build -t ${CKAN_IMAGENAME} . +docker run -d -it --net ckan-net --name ${CKAN_CONTAINERNAME} --hostname ${CKAN_HOSTNAME} --env-file=../environ \ + -v ckan_storage:/var/lib/ckan ${CKAN_IMAGENAME} +cd - + +# +# Nginx - Run/Build +# +NGINX_CONTAINERNAME=nginx +NGINX_HOSTNAME=nginx +NGINX_IMAGENAME=nginx + +cd nginx +docker image build -t ${NGINX_IMAGENAME} . +docker run -d -it --net ckan-net --name ${NGINX_CONTAINERNAME} --hostname ${NGINX_HOSTNAME} -p 0.0.0.0:80:80 ${NGINX_IMAGENAME} +cd - + +# +# PostgreSQL client +# +docker run -d -it --net ckan-net --name postgres-client --hostname postgres-client -e POSTGRES_PASSWORD=ckan -p 5432:5432 postgres diff --git a/pruneall.sh b/pruneall.sh new file mode 100755 index 0000000..d1a8d1f --- /dev/null +++ b/pruneall.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +# Stop all containers +docker kill $(docker ps -q) + +# Remove all stopped containers +docker rm -f $(docker ps -aq) + +# Remove all docker images +docker rmi $(docker images -q) + +# Remove all volumes +docker volume rm $(docker volume ls -q) + +# Remove ckan-docker_default network +docker network rm ckan-docker_default diff --git a/solr/Dockerfile b/solr/Dockerfile index 8922342..feeb423 100755 --- a/solr/Dockerfile +++ b/solr/Dockerfile @@ -3,7 +3,7 @@ FROM solr:6.6.6 # Enviroment variables ENV SOLR_CORE ckan ENV SOLR_VERSION 6.6.6 -ENV CKAN_VERSION 2.9.1 +ENV CKAN_VERSION 2.9.4 ###TODO!!! CKAN_VERSION to be passed in as an ARG # root user for initial config @@ -15,14 +15,14 @@ RUN mkdir -p /opt/solr/server/solr/${SOLR_CORE}/conf && \ mkdir -p /opt/solr/server/solr/${SOLR_CORE}/data/index # Add files -ADD solrconfig.xml \ -https://raw.githubusercontent.com/ckan/ckan/ckan-${CKAN_VERSION}/ckan/config/solr/schema.xml \ -https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/$SOLR_VERSION/solr/server/solr/configsets/basic_configs/conf/currency.xml \ -https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/$SOLR_VERSION/solr/server/solr/configsets/basic_configs/conf/synonyms.txt \ -https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/$SOLR_VERSION/solr/server/solr/configsets/basic_configs/conf/stopwords.txt \ -https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/$SOLR_VERSION/solr/server/solr/configsets/basic_configs/conf/protwords.txt \ -https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/$SOLR_VERSION/solr/server/solr/configsets/data_driven_schema_configs/conf/elevate.xml \ -/opt/solr/server/solr/$SOLR_CORE/conf/ +COPY solrconfig-${CKAN_VERSION}.xml /opt/solr/server/solr/${SOLR_CORE}/conf/solrconfig.xml +ADD https://raw.githubusercontent.com/ckan/ckan/ckan-${CKAN_VERSION}/ckan/config/solr/schema.xml \ +https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/${SOLR_VERSION}/solr/server/solr/configsets/basic_configs/conf/currency.xml \ +https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/${SOLR_VERSION}/solr/server/solr/configsets/basic_configs/conf/synonyms.txt \ +https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/${SOLR_VERSION}/solr/server/solr/configsets/basic_configs/conf/stopwords.txt \ +https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/${SOLR_VERSION}/solr/server/solr/configsets/basic_configs/conf/protwords.txt \ +https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/${SOLR_VERSION}/solr/server/solr/configsets/data_driven_schema_configs/conf/elevate.xml \ +/opt/solr/server/solr/${SOLR_CORE}/conf/ # Create core.properties RUN echo name=${SOLR_CORE} > /opt/solr/server/solr/${SOLR_CORE}/core.properties diff --git a/solr/solrconfig.xml b/solr/solrconfig-2.9.4.xml similarity index 100% rename from solr/solrconfig.xml rename to solr/solrconfig-2.9.4.xml