Initial commit

.gitmodules

@@ -0,0 +1,9 @@
+[submodule "searchanddisplace-core"]
+	path = searchanddisplace-core
+	url = https://git.law/newroco/searchanddisplace-core
+[submodule "searchanddisplace-ingest"]
+	path = searchanddisplace-ingest
+	url = https://git.law/newroco/searchanddisplace-ingest
+[submodule "docker/core/duckling"]
+	path = docker/core/duckling
+	url = https://github.com/facebook/duckling.git

README.md

@@ -1,3 +1,6 @@
-# searchanddisplace-docker
+# A docker container configuration for search and displace
 
-Installation for S&D
+## Initialize the repository
+In order to be able to run this application, its submodules first need to be initialized. To do this, after cloning, run `git submodule init` in the root directory.
+
+After that, the files in `docker/conf/nginx/templates` need to be edited with the corresponding URLs. The `server_name`s need to be added in the operating system's hosts file so they work locally.

docker-compose.yml

@@ -0,0 +1,98 @@
+version: '3'
+services:
+    core:
+        build:
+            context: ./docker/core
+            dockerfile: Dockerfile
+        extra_hosts:
+            - 'host.docker.internal:host-gateway'
+        volumes:
+            - './docker/core/20-supervisor.sh:/docker-entrypoint.d/20-supervisor.sh'
+            - './docker/core/duckling-worker.conf:/etc/supervisor/conf.d/duckling-worker.conf'
+            - './docker/core/queue-worker.conf:/etc/supervisor/conf.d/queue-worker.conf'
+            - './searchanddisplace-core:/var/www/core'
+        environment:
+            NGINX_ENVSUBST_TEMPLATE_SUFFIX: '${NGINX_ENVSUBST_TEMPLATE_SUFFIX:-.conf}'
+            SERVER_ROOT: '/var/www/core'
+            SERVER_NAME: 'core.sandd'
+        container_name: sandd_core
+        networks:
+            - sandd
+        expose:
+            - 9003
+        depends_on:
+            - redis
+
+    ingest:
+        build:
+            context: ./docker/ingest
+            dockerfile: Dockerfile
+        extra_hosts:
+            - 'host.docker.internal:host-gateway'
+        volumes:
+            - './docker/ingest/20-xdebug.ini:/etc/php/7.4/fpm/conf.d/20-xdebug.ini'
+            - './docker/ingest/20-supervisor.sh:/docker-entrypoint.d/20-supervisor.sh'
+            - './docker/ingest/queue-worker.conf:/etc/supervisor/conf.d/queue-worker.conf'
+            - './searchanddisplace-ingest:/var/www/ingest'
+        environment:
+            NGINX_ENVSUBST_TEMPLATE_SUFFIX: '${NGINX_ENVSUBST_TEMPLATE_SUFFIX:-.conf}'
+            SERVER_ROOT: '/var/www/ingest'
+            SERVER_NAME: 'ingest.sandd'
+        container_name: sandd_ingest
+        networks:
+            - sandd
+        expose:
+            - 9004
+        depends_on:
+            - redis
+
+    nginx:
+        image: 'nginx:alpine'
+        container_name: sandd_nginx
+        volumes:
+            - './docker/conf/nginx/nginx.conf:/etc/nginx/nginx.conf'
+            - './docker/conf/nginx/templates:/etc/nginx/templates'
+            - './docker/cert:/etc/cert'
+        environment:
+            NGINX_ENVSUBST_TEMPLATE_SUFFIX: '${NGINX_ENVSUBST_TEMPLATE_SUFFIX}'
+        ports:
+            - 80:80
+            - 443:443
+        networks:
+            - sandd
+        depends_on:
+            - core
+            - ingest
+
+    redis:
+        image: "redis:alpine"
+        container_name: sandd_redis
+        command: redis-server --requirepass sandd
+        ports:
+            - "6379:6379"
+        volumes:
+            - $PWD/docker/redis/data:/var/lib/redis
+            - $PWD/docker/redis/redis.conf:/usr/local/etc/redis/redis.conf
+        environment:
+            - REDIS_REPLICATION_MODE=master
+        networks:
+            - sandd
+
+    redis-commander:
+        image: rediscommander/redis-commander:latest
+        container_name: sandd_redis_commander
+        environment:
+            - REDIS_HOSTS=sandd:redis:6379:1:sandd
+            - HTTP_USER=root
+            - HTTP_PASSWORD=sandd
+        ports:
+            - 8081:8081
+        depends_on:
+            - redis
+        networks:
+            - sandd
+
+
+networks:
+    sandd:
+        driver: bridge

docker/conf/nginx/nginx.conf

@@ -0,0 +1,35 @@
+user  nginx;
+worker_processes  1;
+
+error_log  /var/log/nginx/error.log warn;
+pid        /var/run/nginx.pid;
+
+events {
+    worker_connections  1024;
+}
+
+http {
+    include       /etc/nginx/mime.types;
+    default_type  application/octet-stream;
+
+    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                      '$status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_x_forwarded_for"';
+
+    access_log  /var/log/nginx/access.log  main;
+
+    fastcgi_buffers 8 16k;
+    fastcgi_buffer_size 32k;
+    fastcgi_connect_timeout 90;
+    fastcgi_send_timeout 90;
+    fastcgi_read_timeout 90;
+
+    sendfile        on;
+    #tcp_nopush     on;
+
+    keepalive_timeout  65;
+
+    #gzip  on;
+
+    include /etc/nginx/conf.d/*.conf;
+}

docker/conf/nginx/templates/default.conf.conf

@@ -0,0 +1,21 @@
+server {
+    listen 80;
+    server_name core.sandd;
+
+    location / {
+        proxy_pass http://sandd_core/;
+        proxy_http_version 1.1;
+        proxy_set_header Connection "";
+    }
+}
+
+server {
+    listen 80;
+    server_name ingest.sandd;
+
+    location / {
+        proxy_pass http://sandd_ingest/;
+        proxy_http_version 1.1;
+        proxy_set_header Connection "";
+    }
+}

docker/core/20-supervisor.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+systemctl enable supervisor;
+service supervisor start;
+supervisorctl start all;

docker/core/Dockerfile

@@ -0,0 +1,39 @@
+FROM rcarjan/nginx-php:7.4
+
+LABEL maintainer="Radu Liviu Carjan"
+
+## Make queue log directory
+RUN mkdir /var/log/queue
+
+WORKDIR /var/www
+
+## Install libreoffice
+RUN apt-get install -y software-properties-common && \
+    apt-add-repository -y ppa:libreoffice/ppa && \
+    apt-get install -y \
+    libreoffice \
+    supervisor \
+    sqlite3 \
+    php7.4-sqlite3
+
+RUN mkdir /var/www/duckling
+ADD duckling /var/www/duckling
+
+## Install duckling prerequisites
+RUN  apt-get update && \
+    apt-get install -y \
+    libpcre3 \
+    libpcre3-dev \
+    pkg-config \
+    git
+
+## Change workdir to duckling folder
+WORKDIR /var/www/duckling
+
+## Install haskell stack
+RUN curl -sSL https://get.haskellstack.org/ | sh
+
+## Install duckling
+RUN stack build
+
+WORKDIR ${SERVER_ROOT}

docker/core/duckling-worker.conf

@@ -0,0 +1,10 @@
+[program:duckling-worker]
+process_name=%(program_name)s_%(process_num)02d
+directory=/var/www/duckling
+command=stack exec duckling-example-exe
+autostart=true
+autorestart=true
+user=root
+numprocs=1
+redirect_stderr=true
+stdout_logfile=/var/log/queue/duckling.log

docker/core/queue-worker.conf

@@ -0,0 +1,9 @@
+[program:queue-worker]
+process_name=%(program_name)s_%(process_num)02d
+command=php /var/www/core/artisan queue:listen --queue=sd_core,default --tries=2 --timeout=180
+autostart=true
+autorestart=true
+user=www-data
+numprocs=3
+redirect_stderr=true
+stdout_logfile=/var/log/queue/queues.log

docker/ingest/20-supervisor.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+systemctl enable supervisor;
+service supervisor start;
+supervisorctl start all;

docker/ingest/20-xdebug.ini

@@ -0,0 +1,10 @@
+zend_extension=xdebug
+
+[xdebug]
+xdebug.mode=debug
+xdebug.client_port=9004
+xdebug.start_with_request = yes
+xdebug.client_host=host.docker.internal
+xdebug.idekey = VSCODE
+xdebug.log=/tmp/xdebug.log
+xdebug.log_level=10

docker/ingest/Dockerfile

@@ -0,0 +1,44 @@
+FROM rcarjan/nginx-php:7.4
+
+LABEL maintainer="Radu Liviu Carjan"
+
+## Install libreoffice
+RUN apt-add-repository -y ppa:libreoffice/ppa && \
+    apt-get install -y \
+    libreoffice \
+    software-properties-common
+
+# Install python & popple PDF convertor
+RUN add-apt-repository -y ppa:deadsnakes/ppa && \
+    apt-get install -y \
+    supervisor \
+    python3.8 \
+    python3.8-dev \
+    python3.8-distutils \
+    libpoppler-cpp-dev \
+    poppler-utils
+
+## Install Tesseract OCR, Pandoc & other dependencies
+RUN add-apt-repository -y ppa:alex-p/tesseract-ocr-devel && \
+    apt-get install -y \
+    tesseract-ocr \
+    unpaper \
+    unoconv \
+    pandoc
+
+## Configure correct python version, install PIP
+RUN rm /usr/bin/python3 && \
+    ln -s /usr/bin/python3.8 /usr/bin/python3 && \
+    apt-get install -y python-is-python3 && \
+    curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \
+    python get-pip.py && \
+    rm -rf get-pip.py && \
+    pip install --upgrade pip
+
+## Install PIP packages
+RUN pip install \
+    pdftotext \
+    supervisor \
+    opencv-python
+
+RUN mkdir /var/log/queue

docker/ingest/queue-worker.conf

@@ -0,0 +1,9 @@
+[program:queue-worker]
+process_name=%(program_name)s_%(process_num)02d
+command=php /var/www/ingest/artisan queue:listen --queue=sd_ingest,default --tries=2 --timeout=180
+autostart=true
+autorestart=true
+user=www-data
+numprocs=3
+redirect_stderr=true
+stdout_logfile=/var/log/queue/queues.log

sandd.code-workspace

@@ -0,0 +1,42 @@
+{
+	"folders": [
+		{
+			"path": "."
+		}
+	],
+	"settings": {},
+	"launch": {
+		"version": "0.2.0",
+		"configurations": [
+			{
+				"name": "[DEBUG] Core",
+				"type": "php",
+				"request": "launch",
+				"port": 9003,
+				"log": false,
+				"xdebugSettings": {
+					"show_hidden": 0,
+					"max_depth": 3
+				},
+				"pathMappings": {
+					"/var/www/core/": "${workspaceFolder}/searchanddisplace-core",
+				}
+			},
+
+			{
+				"name": "[DEBUG] Ingest",
+				"type": "php",
+				"request": "launch",
+				"port": 9004,
+				"log": false,
+				"xdebugSettings": {
+					"show_hidden": 0,
+					"max_depth": 3
+				},
+				"pathMappings": {
+					"/var/www/ingest/": "${workspaceFolder}/searchanddisplace-ingest",
+				}
+			}
+		]
+	}
+}

scripts/artisan.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env zsh
+
+clear;
+docker compose -p sandd exec website php artisan $@

scripts/bash.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env zsh
+
+clear;
+docker compose -p sandd exec $@ /bin/bash

scripts/composer.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+clear;
+
+if tty -s; then
+    echo 'Running composer in interactive mode.'
+    docker compose -p sandd exec website composer $@
+else
+    echo 'Running composer in non-interactive mode.'
+    docker compose -p sandd exec -T website composer $@
+fi

scripts/docker.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env zsh
+
+clear;
+docker compose -p sandd $@

scripts/init.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env zsh
+
+rm -rf "$PWD/website/.env";
+# ln -s "$PWD/.env" "$PWD/api/.env";
+cp "$PWD/.env" "$PWD/website/.env";

scripts/npm.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+clear;
+
+container="website"
+dir="/var/www/website/"
+
+
+docker compose -p sandd exec -w ${dir} ${container} npm $@ -unsafe-perm;