From a81447b2caa4ff09f3f625bc18f36e0cdcf2c4eb Mon Sep 17 00:00:00 2001 From: Radu Liviu Carjan Date: Mon, 17 Oct 2022 15:55:41 +0300 Subject: [PATCH] Initial commit --- .gitmodules | 9 ++ README.md | 7 +- docker-compose.yml | 98 +++++++++++++++++++ docker/conf/nginx/nginx.conf | 35 +++++++ docker/conf/nginx/templates/default.conf.conf | 21 ++++ docker/core/20-supervisor.sh | 5 + docker/core/Dockerfile | 39 ++++++++ docker/core/duckling-worker.conf | 10 ++ docker/core/queue-worker.conf | 9 ++ docker/ingest/20-supervisor.sh | 5 + docker/ingest/20-xdebug.ini | 10 ++ docker/ingest/Dockerfile | 44 +++++++++ docker/ingest/queue-worker.conf | 9 ++ sandd.code-workspace | 42 ++++++++ scripts/artisan.sh | 4 + scripts/bash.sh | 4 + scripts/composer.sh | 11 +++ scripts/docker.sh | 4 + scripts/init.sh | 5 + scripts/npm.sh | 9 ++ 20 files changed, 378 insertions(+), 2 deletions(-) create mode 100644 .gitmodules create mode 100644 docker-compose.yml create mode 100644 docker/conf/nginx/nginx.conf create mode 100644 docker/conf/nginx/templates/default.conf.conf create mode 100755 docker/core/20-supervisor.sh create mode 100644 docker/core/Dockerfile create mode 100755 docker/core/duckling-worker.conf create mode 100755 docker/core/queue-worker.conf create mode 100755 docker/ingest/20-supervisor.sh create mode 100644 docker/ingest/20-xdebug.ini create mode 100644 docker/ingest/Dockerfile create mode 100644 docker/ingest/queue-worker.conf create mode 100644 sandd.code-workspace create mode 100755 scripts/artisan.sh create mode 100755 scripts/bash.sh create mode 100755 scripts/composer.sh create mode 100755 scripts/docker.sh create mode 100755 scripts/init.sh create mode 100755 scripts/npm.sh diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..9ad6565 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,9 @@ +[submodule "searchanddisplace-core"] + path = searchanddisplace-core + url = https://git.law/newroco/searchanddisplace-core +[submodule "searchanddisplace-ingest"] + path = searchanddisplace-ingest + url = https://git.law/newroco/searchanddisplace-ingest +[submodule "docker/core/duckling"] + path = docker/core/duckling + url = https://github.com/facebook/duckling.git diff --git a/README.md b/README.md index bbac156..bdecf98 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,6 @@ -# searchanddisplace-docker +# A docker container configuration for search and displace -Installation for S&D \ No newline at end of file +## Initialize the repository +In order to be able to run this application, its submodules first need to be initialized. To do this, after cloning, run `git submodule init` in the root directory. + +After that, the files in `docker/conf/nginx/templates` need to be edited with the corresponding URLs. The `server_name`s need to be added in the operating system's hosts file so they work locally. diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..9804d4c --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,98 @@ +version: '3' +services: + core: + build: + context: ./docker/core + dockerfile: Dockerfile + extra_hosts: + - 'host.docker.internal:host-gateway' + volumes: + - './docker/core/20-supervisor.sh:/docker-entrypoint.d/20-supervisor.sh' + - './docker/core/duckling-worker.conf:/etc/supervisor/conf.d/duckling-worker.conf' + - './docker/core/queue-worker.conf:/etc/supervisor/conf.d/queue-worker.conf' + - './searchanddisplace-core:/var/www/core' + environment: + NGINX_ENVSUBST_TEMPLATE_SUFFIX: '${NGINX_ENVSUBST_TEMPLATE_SUFFIX:-.conf}' + SERVER_ROOT: '/var/www/core' + SERVER_NAME: 'core.sandd' + container_name: sandd_core + networks: + - sandd + expose: + - 9003 + depends_on: + - redis + + ingest: + build: + context: ./docker/ingest + dockerfile: Dockerfile + extra_hosts: + - 'host.docker.internal:host-gateway' + volumes: + - './docker/ingest/20-xdebug.ini:/etc/php/7.4/fpm/conf.d/20-xdebug.ini' + - './docker/ingest/20-supervisor.sh:/docker-entrypoint.d/20-supervisor.sh' + - './docker/ingest/queue-worker.conf:/etc/supervisor/conf.d/queue-worker.conf' + - './searchanddisplace-ingest:/var/www/ingest' + environment: + NGINX_ENVSUBST_TEMPLATE_SUFFIX: '${NGINX_ENVSUBST_TEMPLATE_SUFFIX:-.conf}' + SERVER_ROOT: '/var/www/ingest' + SERVER_NAME: 'ingest.sandd' + container_name: sandd_ingest + networks: + - sandd + expose: + - 9004 + depends_on: + - redis + + nginx: + image: 'nginx:alpine' + container_name: sandd_nginx + volumes: + - './docker/conf/nginx/nginx.conf:/etc/nginx/nginx.conf' + - './docker/conf/nginx/templates:/etc/nginx/templates' + - './docker/cert:/etc/cert' + environment: + NGINX_ENVSUBST_TEMPLATE_SUFFIX: '${NGINX_ENVSUBST_TEMPLATE_SUFFIX}' + ports: + - 80:80 + - 443:443 + networks: + - sandd + depends_on: + - core + - ingest + + redis: + image: "redis:alpine" + container_name: sandd_redis + command: redis-server --requirepass sandd + ports: + - "6379:6379" + volumes: + - $PWD/docker/redis/data:/var/lib/redis + - $PWD/docker/redis/redis.conf:/usr/local/etc/redis/redis.conf + environment: + - REDIS_REPLICATION_MODE=master + networks: + - sandd + + redis-commander: + image: rediscommander/redis-commander:latest + container_name: sandd_redis_commander + environment: + - REDIS_HOSTS=sandd:redis:6379:1:sandd + - HTTP_USER=root + - HTTP_PASSWORD=sandd + ports: + - 8081:8081 + depends_on: + - redis + networks: + - sandd + + +networks: + sandd: + driver: bridge diff --git a/docker/conf/nginx/nginx.conf b/docker/conf/nginx/nginx.conf new file mode 100644 index 0000000..7068928 --- /dev/null +++ b/docker/conf/nginx/nginx.conf @@ -0,0 +1,35 @@ +user nginx; +worker_processes 1; + +error_log /var/log/nginx/error.log warn; +pid /var/run/nginx.pid; + +events { + worker_connections 1024; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + + fastcgi_buffers 8 16k; + fastcgi_buffer_size 32k; + fastcgi_connect_timeout 90; + fastcgi_send_timeout 90; + fastcgi_read_timeout 90; + + sendfile on; + #tcp_nopush on; + + keepalive_timeout 65; + + #gzip on; + + include /etc/nginx/conf.d/*.conf; +} \ No newline at end of file diff --git a/docker/conf/nginx/templates/default.conf.conf b/docker/conf/nginx/templates/default.conf.conf new file mode 100644 index 0000000..7306028 --- /dev/null +++ b/docker/conf/nginx/templates/default.conf.conf @@ -0,0 +1,21 @@ +server { + listen 80; + server_name core.sandd; + + location / { + proxy_pass http://sandd_core/; + proxy_http_version 1.1; + proxy_set_header Connection ""; + } +} + +server { + listen 80; + server_name ingest.sandd; + + location / { + proxy_pass http://sandd_ingest/; + proxy_http_version 1.1; + proxy_set_header Connection ""; + } +} \ No newline at end of file diff --git a/docker/core/20-supervisor.sh b/docker/core/20-supervisor.sh new file mode 100755 index 0000000..ed4c46d --- /dev/null +++ b/docker/core/20-supervisor.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +systemctl enable supervisor; +service supervisor start; +supervisorctl start all; \ No newline at end of file diff --git a/docker/core/Dockerfile b/docker/core/Dockerfile new file mode 100644 index 0000000..5656b59 --- /dev/null +++ b/docker/core/Dockerfile @@ -0,0 +1,39 @@ +FROM rcarjan/nginx-php:7.4 + +LABEL maintainer="Radu Liviu Carjan" + +## Make queue log directory +RUN mkdir /var/log/queue + +WORKDIR /var/www + +## Install libreoffice +RUN apt-get install -y software-properties-common && \ + apt-add-repository -y ppa:libreoffice/ppa && \ + apt-get install -y \ + libreoffice \ + supervisor \ + sqlite3 \ + php7.4-sqlite3 + +RUN mkdir /var/www/duckling +ADD duckling /var/www/duckling + +## Install duckling prerequisites +RUN apt-get update && \ + apt-get install -y \ + libpcre3 \ + libpcre3-dev \ + pkg-config \ + git + +## Change workdir to duckling folder +WORKDIR /var/www/duckling + +## Install haskell stack +RUN curl -sSL https://get.haskellstack.org/ | sh + +## Install duckling +RUN stack build + +WORKDIR ${SERVER_ROOT} \ No newline at end of file diff --git a/docker/core/duckling-worker.conf b/docker/core/duckling-worker.conf new file mode 100755 index 0000000..595b317 --- /dev/null +++ b/docker/core/duckling-worker.conf @@ -0,0 +1,10 @@ +[program:duckling-worker] +process_name=%(program_name)s_%(process_num)02d +directory=/var/www/duckling +command=stack exec duckling-example-exe +autostart=true +autorestart=true +user=root +numprocs=1 +redirect_stderr=true +stdout_logfile=/var/log/queue/duckling.log \ No newline at end of file diff --git a/docker/core/queue-worker.conf b/docker/core/queue-worker.conf new file mode 100755 index 0000000..2ce9571 --- /dev/null +++ b/docker/core/queue-worker.conf @@ -0,0 +1,9 @@ +[program:queue-worker] +process_name=%(program_name)s_%(process_num)02d +command=php /var/www/core/artisan queue:listen --queue=sd_core,default --tries=2 --timeout=180 +autostart=true +autorestart=true +user=www-data +numprocs=3 +redirect_stderr=true +stdout_logfile=/var/log/queue/queues.log \ No newline at end of file diff --git a/docker/ingest/20-supervisor.sh b/docker/ingest/20-supervisor.sh new file mode 100755 index 0000000..ed4c46d --- /dev/null +++ b/docker/ingest/20-supervisor.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +systemctl enable supervisor; +service supervisor start; +supervisorctl start all; \ No newline at end of file diff --git a/docker/ingest/20-xdebug.ini b/docker/ingest/20-xdebug.ini new file mode 100644 index 0000000..aee716b --- /dev/null +++ b/docker/ingest/20-xdebug.ini @@ -0,0 +1,10 @@ +zend_extension=xdebug + +[xdebug] +xdebug.mode=debug +xdebug.client_port=9004 +xdebug.start_with_request = yes +xdebug.client_host=host.docker.internal +xdebug.idekey = VSCODE +xdebug.log=/tmp/xdebug.log +xdebug.log_level=10 \ No newline at end of file diff --git a/docker/ingest/Dockerfile b/docker/ingest/Dockerfile new file mode 100644 index 0000000..e287f35 --- /dev/null +++ b/docker/ingest/Dockerfile @@ -0,0 +1,44 @@ +FROM rcarjan/nginx-php:7.4 + +LABEL maintainer="Radu Liviu Carjan" + +## Install libreoffice +RUN apt-add-repository -y ppa:libreoffice/ppa && \ + apt-get install -y \ + libreoffice \ + software-properties-common + +# Install python & popple PDF convertor +RUN add-apt-repository -y ppa:deadsnakes/ppa && \ + apt-get install -y \ + supervisor \ + python3.8 \ + python3.8-dev \ + python3.8-distutils \ + libpoppler-cpp-dev \ + poppler-utils + +## Install Tesseract OCR, Pandoc & other dependencies +RUN add-apt-repository -y ppa:alex-p/tesseract-ocr-devel && \ + apt-get install -y \ + tesseract-ocr \ + unpaper \ + unoconv \ + pandoc + +## Configure correct python version, install PIP +RUN rm /usr/bin/python3 && \ + ln -s /usr/bin/python3.8 /usr/bin/python3 && \ + apt-get install -y python-is-python3 && \ + curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ + python get-pip.py && \ + rm -rf get-pip.py && \ + pip install --upgrade pip + +## Install PIP packages +RUN pip install \ + pdftotext \ + supervisor \ + opencv-python + +RUN mkdir /var/log/queue \ No newline at end of file diff --git a/docker/ingest/queue-worker.conf b/docker/ingest/queue-worker.conf new file mode 100644 index 0000000..09e7a62 --- /dev/null +++ b/docker/ingest/queue-worker.conf @@ -0,0 +1,9 @@ +[program:queue-worker] +process_name=%(program_name)s_%(process_num)02d +command=php /var/www/ingest/artisan queue:listen --queue=sd_ingest,default --tries=2 --timeout=180 +autostart=true +autorestart=true +user=www-data +numprocs=3 +redirect_stderr=true +stdout_logfile=/var/log/queue/queues.log \ No newline at end of file diff --git a/sandd.code-workspace b/sandd.code-workspace new file mode 100644 index 0000000..ff7ddf6 --- /dev/null +++ b/sandd.code-workspace @@ -0,0 +1,42 @@ +{ + "folders": [ + { + "path": "." + } + ], + "settings": {}, + "launch": { + "version": "0.2.0", + "configurations": [ + { + "name": "[DEBUG] Core", + "type": "php", + "request": "launch", + "port": 9003, + "log": false, + "xdebugSettings": { + "show_hidden": 0, + "max_depth": 3 + }, + "pathMappings": { + "/var/www/core/": "${workspaceFolder}/searchanddisplace-core", + } + }, + + { + "name": "[DEBUG] Ingest", + "type": "php", + "request": "launch", + "port": 9004, + "log": false, + "xdebugSettings": { + "show_hidden": 0, + "max_depth": 3 + }, + "pathMappings": { + "/var/www/ingest/": "${workspaceFolder}/searchanddisplace-ingest", + } + } + ] + } +} \ No newline at end of file diff --git a/scripts/artisan.sh b/scripts/artisan.sh new file mode 100755 index 0000000..4885f41 --- /dev/null +++ b/scripts/artisan.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env zsh + +clear; +docker compose -p sandd exec website php artisan $@ diff --git a/scripts/bash.sh b/scripts/bash.sh new file mode 100755 index 0000000..a652036 --- /dev/null +++ b/scripts/bash.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env zsh + +clear; +docker compose -p sandd exec $@ /bin/bash diff --git a/scripts/composer.sh b/scripts/composer.sh new file mode 100755 index 0000000..65b862c --- /dev/null +++ b/scripts/composer.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +clear; + +if tty -s; then + echo 'Running composer in interactive mode.' + docker compose -p sandd exec website composer $@ +else + echo 'Running composer in non-interactive mode.' + docker compose -p sandd exec -T website composer $@ +fi diff --git a/scripts/docker.sh b/scripts/docker.sh new file mode 100755 index 0000000..1693489 --- /dev/null +++ b/scripts/docker.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env zsh + +clear; +docker compose -p sandd $@ diff --git a/scripts/init.sh b/scripts/init.sh new file mode 100755 index 0000000..0fdad06 --- /dev/null +++ b/scripts/init.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env zsh + +rm -rf "$PWD/website/.env"; +# ln -s "$PWD/.env" "$PWD/api/.env"; +cp "$PWD/.env" "$PWD/website/.env"; \ No newline at end of file diff --git a/scripts/npm.sh b/scripts/npm.sh new file mode 100755 index 0000000..f7aea93 --- /dev/null +++ b/scripts/npm.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +clear; + +container="website" +dir="/var/www/website/" + + +docker compose -p sandd exec -w ${dir} ${container} npm $@ -unsafe-perm; \ No newline at end of file