From d2a11a5227a8ba490f6421ab75df81eb024c8259 Mon Sep 17 00:00:00 2001 From: Johann Frei Date: Mon, 12 Apr 2021 18:30:16 +0200 Subject: [PATCH 01/28] Add offline steps --- offline_01_download.sh | 69 ++++++++++++++++++++++++++++++++++++++++++ offline_02_extract.sh | 15 +++++++++ offline_03_import.sh | 12 ++++++++ 3 files changed, 96 insertions(+) create mode 100755 offline_01_download.sh create mode 100644 offline_02_extract.sh create mode 100644 offline_03_import.sh diff --git a/offline_01_download.sh b/offline_01_download.sh new file mode 100755 index 00000000..c47c6b16 --- /dev/null +++ b/offline_01_download.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +cd $DIR +unset DIR + +# FOUND LINKS: +# app/server/templates/admin.html +# https://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/styles/vs2015.min.css + +# app/server/templates/base.html +# https://use.fontawesome.com/releases/v5.0.13/css/all.css +# https://fonts.googleapis.com/css?family=Open+Sans:300,400,700 +# https://cdnjs.cloudflare.com/ajax/libs/bulma/0.7.1/css/bulma.min.css +# https://cdn.jsdelivr.net/npm/bulma-extensions@4.0.1/bulma-divider/dist/css/bulma-divider.min.css +# https://cdn.jsdelivr.net/npm/bulma-extensions@4.0.1/bulma-checkradio/dist/css/bulma-checkradio.min.css +# https://cdn.jsdelivr.net/npm/bulma-extensions@4.0.1/bulma-tooltip/dist/css/bulma-tooltip.min.css + +# app/server/templates/index.html +# https://cdnjs.cloudflare.com/ajax/libs/Swiper/4.3.3/css/swiper.min.css + +# https://source.unsplash.com/RWnpyGtY1aU +# https://source.unsplash.com/6Ticnhs1AG0 +# https://i.imgsafe.org/ba/baa924a5e3.png + +# frontend/nuxt.config.js +# https://use.fontawesome.com/releases/v5.0.6/js/all.js +# https://fonts.googleapis.com/css?family=Roboto:300,400,500,700|Material+Icons + +n_columns="2" +declare -a links=("offline/vs2015.min.css" "https://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/styles/vs2015.min.css" + "offline/all.css" "https://use.fontawesome.com/releases/v5.0.13/css/all.css" + "offline/opensans.css" "https://fonts.googleapis.com/css?family=Open+Sans:300,400,700" + "offline/bulma.min.css" "https://cdnjs.cloudflare.com/ajax/libs/bulma/0.7.1/css/bulma.min.css" + "offline/bulma-divider.min.css" "https://cdn.jsdelivr.net/npm/bulma-extensions@4.0.1/bulma-divider/dist/css/bulma-divider.min.css" + "offline/bulma-checkradio.min.css" "https://cdn.jsdelivr.net/npm/bulma-extensions@4.0.1/bulma-checkradio/dist/css/bulma-checkradio.min.css" + "offline/bulma-tooltip.min.css" "https://cdn.jsdelivr.net/npm/bulma-extensions@4.0.1/bulma-tooltip/dist/css/bulma-tooltip.min.css" + "offline/photo-1.jpg" "https://source.unsplash.com/RWnpyGtY1aU" + "offline/photo-2.jpg" "https://source.unsplash.com/6Ticnhs1AG0" + "offline/photo-3.jpg" "https://i.imgsafe.org/ba/baa924a5e3.png" + + "offline/all.js" "https://use.fontawesome.com/releases/v5.0.6/js/all.js" + "offline/google-roboto.css" "https://fonts.googleapis.com/css?family=Roboto:300,400,500,700|Material+Icons" + ) + +static_dir="app/server/static/" +mkdir -p "${static_dir}offline/" + +# root replace directories +app_dir="app/server/templates" +frontend_dir="frontend" + +for ((i = 0; i < "$(expr ${#links[@]} / $n_columns)"; ++i)); do + idx_local=$(expr $i \* $n_columns + 0) + idx_link=$(expr $i \* $n_columns + 1) + local="${links[$idx_local]}" + link="${links[$idx_link]}" + + echo "Storing file to $local: $link" + wget --content-on-error -q --show-progress -O "${static_dir}${local}" $link 2>/dev/null + if [ $? -eq 0 ]; then + # For Django: Use 'static' for template, use ^ as delimiter for sed + find $app_dir -type f -exec sed -i "s^${link}^{% static \'${local}\' %}^g" {} \; + # For Vue: Use // for same host, use ^ as delimiter for sed + find $frontend_dir -type f -exec sed -i "s^${link}^//static/${local}^g" {} \; + else + echo "Failed to transform for offline use: $link" + fi +done diff --git a/offline_02_extract.sh b/offline_02_extract.sh new file mode 100644 index 00000000..28553581 --- /dev/null +++ b/offline_02_extract.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +cd $DIR +unset DIR + +sed -i 's^dockerfile: app/Dockerfile.prod^dockerfile: app/Dockerfile.prod\n image: doccano-app:custom^g' docker-compose.prod.yml +sed -i 's^dockerfile: nginx/Dockerfile^dockerfile: nginx/Dockerfile\n image: doccano-nginx:custom^g' docker-compose.prod.yml + +docker-compose -f docker-compose.prod.yml pull +docker-compose -f docker-compose.prod.yml build + +docker image save -o doccano-app.tar doccano-app:custom +docker image save -o doccano-nginx.tar doccano-nginx:custom +docker image save -o postgres.tar postgres:13.1-alpine diff --git a/offline_03_import.sh b/offline_03_import.sh new file mode 100644 index 00000000..fd3ac8e9 --- /dev/null +++ b/offline_03_import.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +cd $DIR +unset DIR + +sed -i 's^dockerfile: app/Dockerfile.prod^dockerfile: app/Dockerfile.prod\n image: doccano-app:custom^g' docker-compose.prod.yml +sed -i 's^dockerfile: nginx/Dockerfile^dockerfile: nginx/Dockerfile\n image: doccano-nginx:custom^g' docker-compose.prod.yml + +docker image import doccano-app.tar doccano-app:custom +docker image import doccano-nginx.tar doccano-nginx:custom +docker image import postgres.tar postgres:13.1-alpine From 46f587fb186ae65bcb202ecf9d72c898bb257c8f Mon Sep 17 00:00:00 2001 From: Johann Frei Date: Mon, 12 Apr 2021 18:56:58 +0200 Subject: [PATCH 02/28] Fix for loop for bash 4.4 support --- offline_01_download.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/offline_01_download.sh b/offline_01_download.sh index c47c6b16..305c46d2 100755 --- a/offline_01_download.sh +++ b/offline_01_download.sh @@ -50,7 +50,7 @@ mkdir -p "${static_dir}offline/" app_dir="app/server/templates" frontend_dir="frontend" -for ((i = 0; i < "$(expr ${#links[@]} / $n_columns)"; ++i)); do +for ((i = 0; i < $(expr "${#links[@]}" / "$n_columns"); ++i)); do idx_local=$(expr $i \* $n_columns + 0) idx_link=$(expr $i \* $n_columns + 1) local="${links[$idx_local]}" From 47243ab4a99140ab57401fd7a43de6fedcd76896 Mon Sep 17 00:00:00 2001 From: Johann Frei Date: Mon, 12 Apr 2021 18:57:11 +0200 Subject: [PATCH 03/28] Make scripts executable --- offline_02_extract.sh | 0 offline_03_import.sh | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 offline_02_extract.sh mode change 100644 => 100755 offline_03_import.sh diff --git a/offline_02_extract.sh b/offline_02_extract.sh old mode 100644 new mode 100755 diff --git a/offline_03_import.sh b/offline_03_import.sh old mode 100644 new mode 100755 From 01100051cf1ffa1452f3220d44cf6005c5eb9e87 Mon Sep 17 00:00:00 2001 From: Johann Frei Date: Mon, 12 Apr 2021 20:25:59 +0200 Subject: [PATCH 04/28] Refactor and add offline deployment --- .../offline_01_1-download_assets.sh | 1 + .../offline_01_2-extract_Docker_images.sh | 8 +++--- ...offline_01_3-download_APT_packages copy.sh | 22 +++++++++++++++ .../offline_02_1-install_APT_packages.sh | 27 +++++++++++++++++++ .../offline_02_2-import_Docker_images.sh | 4 +-- offline_deployment/offline_03_1-runDoccano.sh | 7 +++++ 6 files changed, 63 insertions(+), 6 deletions(-) rename offline_01_download.sh => offline_deployment/offline_01_1-download_assets.sh (99%) rename offline_02_extract.sh => offline_deployment/offline_01_2-extract_Docker_images.sh (62%) create mode 100755 offline_deployment/offline_01_3-download_APT_packages copy.sh create mode 100755 offline_deployment/offline_02_1-install_APT_packages.sh rename offline_03_import.sh => offline_deployment/offline_02_2-import_Docker_images.sh (74%) create mode 100755 offline_deployment/offline_03_1-runDoccano.sh diff --git a/offline_01_download.sh b/offline_deployment/offline_01_1-download_assets.sh similarity index 99% rename from offline_01_download.sh rename to offline_deployment/offline_01_1-download_assets.sh index 305c46d2..c3f76e77 100755 --- a/offline_01_download.sh +++ b/offline_deployment/offline_01_1-download_assets.sh @@ -2,6 +2,7 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd $DIR +cd .. unset DIR # FOUND LINKS: diff --git a/offline_02_extract.sh b/offline_deployment/offline_01_2-extract_Docker_images.sh similarity index 62% rename from offline_02_extract.sh rename to offline_deployment/offline_01_2-extract_Docker_images.sh index 28553581..648000f2 100755 --- a/offline_02_extract.sh +++ b/offline_deployment/offline_01_2-extract_Docker_images.sh @@ -4,11 +4,11 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd $DIR unset DIR -sed -i 's^dockerfile: app/Dockerfile.prod^dockerfile: app/Dockerfile.prod\n image: doccano-app:custom^g' docker-compose.prod.yml -sed -i 's^dockerfile: nginx/Dockerfile^dockerfile: nginx/Dockerfile\n image: doccano-nginx:custom^g' docker-compose.prod.yml +sed -i 's^dockerfile: app/Dockerfile.prod^dockerfile: app/Dockerfile.prod\n image: doccano-app:custom^g' ../docker-compose.prod.yml +sed -i 's^dockerfile: nginx/Dockerfile^dockerfile: nginx/Dockerfile\n image: doccano-nginx:custom^g' ../docker-compose.prod.yml -docker-compose -f docker-compose.prod.yml pull -docker-compose -f docker-compose.prod.yml build +docker-compose -f ../docker-compose.prod.yml pull +docker-compose -f ../docker-compose.prod.yml build docker image save -o doccano-app.tar doccano-app:custom docker image save -o doccano-nginx.tar doccano-nginx:custom diff --git a/offline_deployment/offline_01_3-download_APT_packages copy.sh b/offline_deployment/offline_01_3-download_APT_packages copy.sh new file mode 100755 index 00000000..95342f59 --- /dev/null +++ b/offline_deployment/offline_01_3-download_APT_packages copy.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +cd $DIR +unset DIR + +# Prepare and download packages +pdir="/offline_packages" +mkdir -p "$(pwd)${pdir}" +cd "$(pwd)${pdir}" + +SELECTED_PACKAGES="wget unzip curl tar docker.io docker-compose" + +apt-get download $(apt-cache depends --recurse --no-recommends --no-suggests \ + --no-conflicts --no-breaks --no-replaces --no-enhances \ + --no-pre-depends ${SELECTED_PACKAGES} | grep "^\w") + +# Build package index +sudo apt-get install -y dpkg-dev +dpkg-scanpackages "." /dev/null | gzip -9c > Packages.gz + +echo "Packages extracted to: $(pwd)${pdir}" \ No newline at end of file diff --git a/offline_deployment/offline_02_1-install_APT_packages.sh b/offline_deployment/offline_02_1-install_APT_packages.sh new file mode 100755 index 00000000..5486ba3d --- /dev/null +++ b/offline_deployment/offline_02_1-install_APT_packages.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +cd $DIR +unset DIR + +# Import APT packages +pdir="/offline_packages" +abs_pdir="$(pwd)${pdir}" +sudo mv /etc/apt/sources.list /etc/apt/sources.list.bak +sudo sh "deb [trusted=yes] file:${abs_pdir}" + +# Install APT packages +sudo apt-get update +SELECTED_PACKAGES="wget unzip curl tar docker.io docker-compose" +sudo apt-get install -y $SELECTED_PACKAGES + +# Cleanup +sudo apt-get clean +sudo mv /etc/apt/sources.list.bak /etc/apt/sources.list + +# Setup Docker +sudo usermod -aG docker $(whoami) +sudo systemctl enable docker.service + +echo "Packages were installed. We need to reboot!" + diff --git a/offline_03_import.sh b/offline_deployment/offline_02_2-import_Docker_images.sh similarity index 74% rename from offline_03_import.sh rename to offline_deployment/offline_02_2-import_Docker_images.sh index fd3ac8e9..cafdabaa 100755 --- a/offline_03_import.sh +++ b/offline_deployment/offline_02_2-import_Docker_images.sh @@ -4,8 +4,8 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd $DIR unset DIR -sed -i 's^dockerfile: app/Dockerfile.prod^dockerfile: app/Dockerfile.prod\n image: doccano-app:custom^g' docker-compose.prod.yml -sed -i 's^dockerfile: nginx/Dockerfile^dockerfile: nginx/Dockerfile\n image: doccano-nginx:custom^g' docker-compose.prod.yml +sed -i 's^dockerfile: app/Dockerfile.prod^dockerfile: app/Dockerfile.prod\n image: doccano-app:custom^g' ../docker-compose.prod.yml +sed -i 's^dockerfile: nginx/Dockerfile^dockerfile: nginx/Dockerfile\n image: doccano-nginx:custom^g' ../docker-compose.prod.yml docker image import doccano-app.tar doccano-app:custom docker image import doccano-nginx.tar doccano-nginx:custom diff --git a/offline_deployment/offline_03_1-runDoccano.sh b/offline_deployment/offline_03_1-runDoccano.sh new file mode 100755 index 00000000..5f0f1894 --- /dev/null +++ b/offline_deployment/offline_03_1-runDoccano.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +cd $DIR +unset DIR + +docker-compose -f ../docker-compose.prod.yml up -d \ No newline at end of file From c8bdf0e35a8c03e34e36d9eb23a3834ca408d274 Mon Sep 17 00:00:00 2001 From: Johann Frei Date: Mon, 12 Apr 2021 22:36:12 +0200 Subject: [PATCH 05/28] Support SSL --- offline_deployment/README.md | 23 +++++++++++++++ ...s.sh => offline_01_1_1-download_assets.sh} | 0 .../offline_01_1_2-optional_use_https.sh | 28 +++++++++++++++++++ .../offline_01_2-extract_Docker_images.sh | 4 +++ ... => offline_01_3-download_APT_packages.sh} | 0 5 files changed, 55 insertions(+) create mode 100644 offline_deployment/README.md rename offline_deployment/{offline_01_1-download_assets.sh => offline_01_1_1-download_assets.sh} (100%) create mode 100755 offline_deployment/offline_01_1_2-optional_use_https.sh rename offline_deployment/{offline_01_3-download_APT_packages copy.sh => offline_01_3-download_APT_packages.sh} (100%) diff --git a/offline_deployment/README.md b/offline_deployment/README.md new file mode 100644 index 00000000..a30ae34f --- /dev/null +++ b/offline_deployment/README.md @@ -0,0 +1,23 @@ +# Doccano Offline Deployment + +## Use Case +These offline deployment scripts are suited for deploying Doccano on an airgaped Ubuntu 18.04/20.04 virtual machine (VM 2) with no internet connectivity. + +The preparation requires another machine (VM 1) with internet access and `docker`/`docker-compose` preinstalled and running the same Ubuntu distribution as VM 2. + +The focus is primarily on the `docker-compose`-based production deployment. + +## Setup Steps + +Run the following steps on VM 1: +1. Clone this repository +2. Run the scripts `offline_01_*.sh` in ascending order + Skip OR modify and run the script `offline_01_1_2-optional_use_https` + +Now, move over to VM 2 + +3. Copy the repository folder to VM 2 +4. Run the scripts `offline_02_*.sh` in ascending order +5. Make minor changes on `docker-compose.prod.yml` to change the admin credentials +6. Run `docker-compose -f docker-compose.prod.yml up` or use the script `offline_03_*.sh` + diff --git a/offline_deployment/offline_01_1-download_assets.sh b/offline_deployment/offline_01_1_1-download_assets.sh similarity index 100% rename from offline_deployment/offline_01_1-download_assets.sh rename to offline_deployment/offline_01_1_1-download_assets.sh diff --git a/offline_deployment/offline_01_1_2-optional_use_https.sh b/offline_deployment/offline_01_1_2-optional_use_https.sh new file mode 100755 index 00000000..1099de69 --- /dev/null +++ b/offline_deployment/offline_01_1_2-optional_use_https.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +cd $DIR +cd .. +unset DIR + +# create certificate pair +sudo apt-get install openssl +openssl req -new -newkey rsa:4096 -sha256 -nodes -x509 -keyout ./nginx/cert.key -out ./nginx/cert.crt \ + -subj "/C=DE/ST=Bayern/L=Augsburg/O=Universität Augsburg/OU=Misit/CN=*.informatik.uni-augsburg.de" + +# define cert paths inside container +$ssl_cert="/certs/cert.crt" +$ssl_cert_key="/certs/cert.key" + +# edit nginx.conf +sed -i "s|listen 80;|listen 443 ssl;\n ssl_certificate $ssl_cert;\n ssl_certificate_key $ssl_cert_key;|g" nginx/nginx.conf + +# edit nginx Dockerfile +echo "RUN mkdir -p /certs/" >> nginx/Dockerfile +echo "COPY ./nginx/cert.key /certs/cert.key" >> nginx/Dockerfile +echo "COPY ./nginx/cert.crt /certs/cert.crt" >> nginx/Dockerfile + +# edit published port +sed -i "s|- 80:80|- 443:443|g" docker-compose.prod.yml + +echo "Switched to HTTPS" diff --git a/offline_deployment/offline_01_2-extract_Docker_images.sh b/offline_deployment/offline_01_2-extract_Docker_images.sh index 648000f2..955d60a0 100755 --- a/offline_deployment/offline_01_2-extract_Docker_images.sh +++ b/offline_deployment/offline_01_2-extract_Docker_images.sh @@ -4,6 +4,10 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd $DIR unset DIR +# WORKAROUND: Downgrade docker-compose version to match Ubuntu 18.04 default compose package +echo "Patching docker-compose to match Ubuntu 18.04 compose package" +sed -i 's|version: "3.7"|version: "3.3"|g' ../docker-compose.prod.yml + sed -i 's^dockerfile: app/Dockerfile.prod^dockerfile: app/Dockerfile.prod\n image: doccano-app:custom^g' ../docker-compose.prod.yml sed -i 's^dockerfile: nginx/Dockerfile^dockerfile: nginx/Dockerfile\n image: doccano-nginx:custom^g' ../docker-compose.prod.yml diff --git a/offline_deployment/offline_01_3-download_APT_packages copy.sh b/offline_deployment/offline_01_3-download_APT_packages.sh similarity index 100% rename from offline_deployment/offline_01_3-download_APT_packages copy.sh rename to offline_deployment/offline_01_3-download_APT_packages.sh From f9767602d72d6c053ee1a8c3aee013b9dff25e84 Mon Sep 17 00:00:00 2001 From: j-frei <27365662+j-frei@users.noreply.github.com> Date: Mon, 12 Apr 2021 22:49:11 +0200 Subject: [PATCH 06/28] Update README.md --- offline_deployment/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/offline_deployment/README.md b/offline_deployment/README.md index a30ae34f..cc7128b4 100644 --- a/offline_deployment/README.md +++ b/offline_deployment/README.md @@ -1,7 +1,7 @@ # Doccano Offline Deployment ## Use Case -These offline deployment scripts are suited for deploying Doccano on an airgaped Ubuntu 18.04/20.04 virtual machine (VM 2) with no internet connectivity. +These offline deployment scripts are suited for deploying Doccano on an air gapped Ubuntu 18.04/20.04 virtual machine (VM 2) with no internet connectivity. The preparation requires another machine (VM 1) with internet access and `docker`/`docker-compose` preinstalled and running the same Ubuntu distribution as VM 2. @@ -16,7 +16,7 @@ Run the following steps on VM 1: Now, move over to VM 2 -3. Copy the repository folder to VM 2 +3. Copy the repository folder from VM 1 to VM 2 4. Run the scripts `offline_02_*.sh` in ascending order 5. Make minor changes on `docker-compose.prod.yml` to change the admin credentials 6. Run `docker-compose -f docker-compose.prod.yml up` or use the script `offline_03_*.sh` From c94c5504c3792f784f9112f71702bb8bf87c6166 Mon Sep 17 00:00:00 2001 From: Johann Frei Date: Tue, 13 Apr 2021 13:12:43 +0200 Subject: [PATCH 07/28] Minor cleanups and fixes --- .../offline_01_1_2-optional_use_https.sh | 12 ++++++------ .../offline_02_1-install_APT_packages.sh | 5 ++++- .../offline_02_2-import_Docker_images.sh | 8 +++++--- offline_deployment/offline_03_1-runDoccano.sh | 2 +- 4 files changed, 16 insertions(+), 11 deletions(-) diff --git a/offline_deployment/offline_01_1_2-optional_use_https.sh b/offline_deployment/offline_01_1_2-optional_use_https.sh index 1099de69..b15c8e7b 100755 --- a/offline_deployment/offline_01_1_2-optional_use_https.sh +++ b/offline_deployment/offline_01_1_2-optional_use_https.sh @@ -6,21 +6,21 @@ cd .. unset DIR # create certificate pair -sudo apt-get install openssl +sudo apt-get install -y openssl openssl req -new -newkey rsa:4096 -sha256 -nodes -x509 -keyout ./nginx/cert.key -out ./nginx/cert.crt \ -subj "/C=DE/ST=Bayern/L=Augsburg/O=Universität Augsburg/OU=Misit/CN=*.informatik.uni-augsburg.de" # define cert paths inside container -$ssl_cert="/certs/cert.crt" -$ssl_cert_key="/certs/cert.key" +ssl_cert="/certs/cert.crt" +ssl_cert_key="/certs/cert.key" # edit nginx.conf sed -i "s|listen 80;|listen 443 ssl;\n ssl_certificate $ssl_cert;\n ssl_certificate_key $ssl_cert_key;|g" nginx/nginx.conf # edit nginx Dockerfile -echo "RUN mkdir -p /certs/" >> nginx/Dockerfile -echo "COPY ./nginx/cert.key /certs/cert.key" >> nginx/Dockerfile -echo "COPY ./nginx/cert.crt /certs/cert.crt" >> nginx/Dockerfile +echo "RUN mkdir -p /certs/" >> nginx/Dockerfile +echo "COPY nginx/cert.key /certs/cert.key" >> nginx/Dockerfile +echo "COPY nginx/cert.crt /certs/cert.crt" >> nginx/Dockerfile # edit published port sed -i "s|- 80:80|- 443:443|g" docker-compose.prod.yml diff --git a/offline_deployment/offline_02_1-install_APT_packages.sh b/offline_deployment/offline_02_1-install_APT_packages.sh index 5486ba3d..6f23f226 100755 --- a/offline_deployment/offline_02_1-install_APT_packages.sh +++ b/offline_deployment/offline_02_1-install_APT_packages.sh @@ -8,7 +8,10 @@ unset DIR pdir="/offline_packages" abs_pdir="$(pwd)${pdir}" sudo mv /etc/apt/sources.list /etc/apt/sources.list.bak -sudo sh "deb [trusted=yes] file:${abs_pdir}" +cat < sources.list +deb [trusted=yes] file://${abs_pdir} ./ +EOF +sudo mv sources.list /etc/apt/sources.list # Install APT packages sudo apt-get update diff --git a/offline_deployment/offline_02_2-import_Docker_images.sh b/offline_deployment/offline_02_2-import_Docker_images.sh index cafdabaa..58c0fb9c 100755 --- a/offline_deployment/offline_02_2-import_Docker_images.sh +++ b/offline_deployment/offline_02_2-import_Docker_images.sh @@ -4,9 +4,11 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd $DIR unset DIR +# Set image tag in Compose to avoid image build sed -i 's^dockerfile: app/Dockerfile.prod^dockerfile: app/Dockerfile.prod\n image: doccano-app:custom^g' ../docker-compose.prod.yml sed -i 's^dockerfile: nginx/Dockerfile^dockerfile: nginx/Dockerfile\n image: doccano-nginx:custom^g' ../docker-compose.prod.yml -docker image import doccano-app.tar doccano-app:custom -docker image import doccano-nginx.tar doccano-nginx:custom -docker image import postgres.tar postgres:13.1-alpine +# Load docker images +docker image load -i doccano-app.tar +docker image load -i doccano-nginx.tar +docker image load -i postgres.tar diff --git a/offline_deployment/offline_03_1-runDoccano.sh b/offline_deployment/offline_03_1-runDoccano.sh index 5f0f1894..dec8e02e 100755 --- a/offline_deployment/offline_03_1-runDoccano.sh +++ b/offline_deployment/offline_03_1-runDoccano.sh @@ -4,4 +4,4 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd $DIR unset DIR -docker-compose -f ../docker-compose.prod.yml up -d \ No newline at end of file +docker-compose -f ../docker-compose.prod.yml up -d From aa50f662dadd274e0b1739eec1d70aca0cd0b539 Mon Sep 17 00:00:00 2001 From: j-frei <27365662+j-frei@users.noreply.github.com> Date: Tue, 13 Apr 2021 13:53:00 +0200 Subject: [PATCH 08/28] Update README.md --- offline_deployment/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/offline_deployment/README.md b/offline_deployment/README.md index cc7128b4..16521b78 100644 --- a/offline_deployment/README.md +++ b/offline_deployment/README.md @@ -3,7 +3,7 @@ ## Use Case These offline deployment scripts are suited for deploying Doccano on an air gapped Ubuntu 18.04/20.04 virtual machine (VM 2) with no internet connectivity. -The preparation requires another machine (VM 1) with internet access and `docker`/`docker-compose` preinstalled and running the same Ubuntu distribution as VM 2. +The preparation requires another machine (VM 1) with internet access and `docker`/`docker-compose` preinstalled (with $USER in `docker` group) and running the same Ubuntu distribution as VM 2. The focus is primarily on the `docker-compose`-based production deployment. From 7916f7c1344f054e10574e3d09e6860a96de51da Mon Sep 17 00:00:00 2001 From: Johann Frei Date: Tue, 13 Apr 2021 16:30:50 +0200 Subject: [PATCH 09/28] Store assets in frontend static store --- offline_deployment/offline_01_1_1-download_assets.sh | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/offline_deployment/offline_01_1_1-download_assets.sh b/offline_deployment/offline_01_1_1-download_assets.sh index c3f76e77..a280b54a 100755 --- a/offline_deployment/offline_01_1_1-download_assets.sh +++ b/offline_deployment/offline_01_1_1-download_assets.sh @@ -44,8 +44,10 @@ declare -a links=("offline/vs2015.min.css" "https://cdnjs.cloudflare.c "offline/google-roboto.css" "https://fonts.googleapis.com/css?family=Roboto:300,400,500,700|Material+Icons" ) -static_dir="app/server/static/" -mkdir -p "${static_dir}offline/" +static_dir_app="app/server/static/" +static_dir_frontend="frontend/static/" +mkdir -p "${static_dir_app}offline/" +mkdir -p "${static_dir_frontend}offline/" # root replace directories app_dir="app/server/templates" @@ -58,12 +60,14 @@ for ((i = 0; i < $(expr "${#links[@]}" / "$n_columns"); ++i)); do link="${links[$idx_link]}" echo "Storing file to $local: $link" - wget --content-on-error -q --show-progress -O "${static_dir}${local}" $link 2>/dev/null + wget --content-on-error -q --show-progress -O "${static_dir_app}${local}" $link 2>/dev/null if [ $? -eq 0 ]; then + # Copy to frontend static dir + cp "${static_dir_app}${local}" "${static_dir_frontend}${local}" # For Django: Use 'static' for template, use ^ as delimiter for sed find $app_dir -type f -exec sed -i "s^${link}^{% static \'${local}\' %}^g" {} \; # For Vue: Use // for same host, use ^ as delimiter for sed - find $frontend_dir -type f -exec sed -i "s^${link}^//static/${local}^g" {} \; + find $frontend_dir -type f -exec sed -i "s^${link}^/${local}^g" {} \; else echo "Failed to transform for offline use: $link" fi From 8d32cad2529e8d34367fb157f1e7d1a02d1b51bf Mon Sep 17 00:00:00 2001 From: Johann Frei Date: Wed, 14 Apr 2021 19:47:58 +0200 Subject: [PATCH 10/28] Add offline patcher script --- .../offline_02_2-import_Docker_images.sh | 7 + offline_deployment/offline_patcher.py | 135 ++++++++++++++++++ 2 files changed, 142 insertions(+) create mode 100644 offline_deployment/offline_patcher.py diff --git a/offline_deployment/offline_02_2-import_Docker_images.sh b/offline_deployment/offline_02_2-import_Docker_images.sh index 58c0fb9c..fdbcda64 100755 --- a/offline_deployment/offline_02_2-import_Docker_images.sh +++ b/offline_deployment/offline_02_2-import_Docker_images.sh @@ -8,6 +8,13 @@ unset DIR sed -i 's^dockerfile: app/Dockerfile.prod^dockerfile: app/Dockerfile.prod\n image: doccano-app:custom^g' ../docker-compose.prod.yml sed -i 's^dockerfile: nginx/Dockerfile^dockerfile: nginx/Dockerfile\n image: doccano-nginx:custom^g' ../docker-compose.prod.yml +# Modify Dockerfile for nginx to add python3 and offline patch +sed -i 's|FROM nginx|COPY offline_deployment/offline_patcher.py /patch.py\ +RUN apk add -U --no-cache py3-requests \\\ + \&\& mkdir -p /app/dist/static/offline \&\& python3 /patch.py /app/dist /app/dist/static/offline\ +\ +FROM nginx|' ../nginx/Dockerfile + # Load docker images docker image load -i doccano-app.tar docker image load -i doccano-nginx.tar diff --git a/offline_deployment/offline_patcher.py b/offline_deployment/offline_patcher.py new file mode 100644 index 00000000..cf49b127 --- /dev/null +++ b/offline_deployment/offline_patcher.py @@ -0,0 +1,135 @@ +import sys, os, re +from glob import glob +import uuid +import requests + +def main(): + # root folder to scan for URLs + root_folder = sys.argv[1] + # offline folder to store static offline files + offline_folder = sys.argv[2] + + + offline_file = os.path.join(offline_folder, "offline_{}.{}") + offline_link = "offline/offline_{}.{}" + + mime_ptn = re.compile(r"(?P(?P[\w^\/]+)\/(?P[\S\.^\;]+))(\;|$)", re.IGNORECASE) + #link_ptn = re.compile(r"(?P[\S\"\'])(?Phttps?:\/\/(?P[\S^:\/)]+)(?P\:[0-9]+)?\/((?!(?P=encl)).)+)(?P=encl)", re.IGNORECASE) + link_ptn = re.compile(r"[\(\'\"\ ](?Phttps?:\/\/(?P[\S^:\/)]+)(?P\:[0-9]+)?\/[^\(\)\'\"\ ]+)(?P[\(\)\'\"\ ])") + + # Block special hosts + forbidden_hosts = [ + re.compile(r"^.*registry\.npmjs\.org$"), # No yarnpkg repository + re.compile(r"^.*yarnpkg\.com$"), # No yarnpkg repository + re.compile(r"^[0-9\.]+$"), # avoid IP addresses + re.compile(r"^[^\.]+$"), # needs a dot in host + ] + + # only support certain content types + supported_mime_types = [ + # (filter function -> bool, file extension -> str) + (lambda m: m["t2"] == "javascript", lambda m: "js"), + (lambda m: m["t2"] == "css", lambda m: "css"), + (lambda m: m["t1"] == "font", lambda m: m["t2"]), + ] + + + # load all initial files + files_to_check = [] + for cur_dir, n_dir, n_files in os.walk(root_folder): + files_to_check += [ os.path.join(cur_dir, f) for f in n_files ] + + cached_urls = {} + valid_urls = {} + + i = 0 + while i < len(files_to_check): + file_i = files_to_check[i] + try: + print("Inspect", file_i) + with open(file_i, "r", encoding="utf-8") as f: + t = f.read() + + for match in link_ptn.finditer(t): + found_link = match.group("link") + found_host = match.group("host") + + if found_link not in valid_urls: + # check link + if True in [ True for fh in forbidden_hosts if fh.match(found_link) is not None ]: + # host is forbidden + valid_urls[found_link] = False + else: + # host is not forbidden + # check mime type + response = requests.head(found_link, allow_redirects=True) + mime = response.headers.get("Content-Type", None) + if mime is None: + valid_urls[found_link] = False + else: + mime_match = mime_ptn.match(mime) + if mime_match is None: + valid_urls[found_link] = False + else: + final_fext = None + for smt, get_fext in supported_mime_types: + if smt(mime_match): + final_fext = get_fext(mime_match) + break + if final_fext is None: + # mime not supported + valid_urls[found_link] = False + else: + # mime is supported -> store and remember file + valid_urls[found_link] = True + file_unique = uuid.uuid4() + target_link = offline_link.format(file_unique, final_fext) + target_file = offline_file.format(file_unique, final_fext) + + # download file + try: + file_response = requests.get(found_link, allow_redirects=True) + file_response.raise_for_status() + with open(target_file, 'wb') as download_file: + for chunk in file_response.iter_content(100000): + download_file.write(chunk) + # also check downloaded file + files_to_check.append(target_file) + + print("Downloaded file:", found_link) + except: + print("Link could not been downloaded:", found_link) + + # register downloaded file + cached_urls[found_link] = { + "input_link": found_link, + "target_link": target_link, + "file": target_file, + "fext": final_fext, + "found": [ file_i ] + } + + if valid_urls[found_link]: + # add to cached urls entries + cached_urls[found_link]["found"].append(file_i) + + print("Checked file:", file_i) + except: + print("Skip file:", file_i) + + # look at next file + i+= 1 + + # replace files with offline link + for _, cached in cached_urls.items(): + for edit_file in cached["found"]: + with open(edit_file, "r", encoding="utf-8") as f: + file_content = f.read() + with open(edit_file, "w", encoding="utf-8") as f: + f.write(file_content.replace(cached["input_link"], cached["target_link"])) + print("Patched to", len(cached["found"]), "file with link:", cached["target_link"]) + + print("Done") + +if __name__ == "__main__": + main() \ No newline at end of file From b46777bbbb80924bd9a1b8c61a168ddaf9fe2017 Mon Sep 17 00:00:00 2001 From: Johann Frei Date: Wed, 14 Apr 2021 19:49:43 +0200 Subject: [PATCH 11/28] Add link prefix as parameter --- offline_deployment/offline_02_2-import_Docker_images.sh | 2 +- offline_deployment/offline_patcher.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/offline_deployment/offline_02_2-import_Docker_images.sh b/offline_deployment/offline_02_2-import_Docker_images.sh index fdbcda64..b7b4f7a2 100755 --- a/offline_deployment/offline_02_2-import_Docker_images.sh +++ b/offline_deployment/offline_02_2-import_Docker_images.sh @@ -11,7 +11,7 @@ sed -i 's^dockerfile: nginx/Dockerfile^dockerfile: nginx/Dockerfile\n image: # Modify Dockerfile for nginx to add python3 and offline patch sed -i 's|FROM nginx|COPY offline_deployment/offline_patcher.py /patch.py\ RUN apk add -U --no-cache py3-requests \\\ - \&\& mkdir -p /app/dist/static/offline \&\& python3 /patch.py /app/dist /app/dist/static/offline\ + \&\& mkdir -p /app/dist/static/offline \&\& python3 /patch.py /app/dist /app/dist/static/offline /offline\ \ FROM nginx|' ../nginx/Dockerfile diff --git a/offline_deployment/offline_patcher.py b/offline_deployment/offline_patcher.py index cf49b127..47b27b44 100644 --- a/offline_deployment/offline_patcher.py +++ b/offline_deployment/offline_patcher.py @@ -8,10 +8,12 @@ def main(): root_folder = sys.argv[1] # offline folder to store static offline files offline_folder = sys.argv[2] + # offline link prefix + offline_prefix = sys.argv[3] offline_file = os.path.join(offline_folder, "offline_{}.{}") - offline_link = "offline/offline_{}.{}" + offline_link = offline_prefix + "/offline_{}.{}" mime_ptn = re.compile(r"(?P(?P[\w^\/]+)\/(?P[\S\.^\;]+))(\;|$)", re.IGNORECASE) #link_ptn = re.compile(r"(?P[\S\"\'])(?Phttps?:\/\/(?P[\S^:\/)]+)(?P\:[0-9]+)?\/((?!(?P=encl)).)+)(?P=encl)", re.IGNORECASE) From 9e3de6eeaf1987273cefd7364b47948393389839 Mon Sep 17 00:00:00 2001 From: Johann Frei Date: Wed, 14 Apr 2021 20:09:53 +0200 Subject: [PATCH 12/28] Move commands to other script --- offline_deployment/offline_01_2-extract_Docker_images.sh | 7 +++++++ offline_deployment/offline_02_2-import_Docker_images.sh | 7 ------- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/offline_deployment/offline_01_2-extract_Docker_images.sh b/offline_deployment/offline_01_2-extract_Docker_images.sh index 955d60a0..2b41725c 100755 --- a/offline_deployment/offline_01_2-extract_Docker_images.sh +++ b/offline_deployment/offline_01_2-extract_Docker_images.sh @@ -11,6 +11,13 @@ sed -i 's|version: "3.7"|version: "3.3"|g' ../docker-compose.prod.yml sed -i 's^dockerfile: app/Dockerfile.prod^dockerfile: app/Dockerfile.prod\n image: doccano-app:custom^g' ../docker-compose.prod.yml sed -i 's^dockerfile: nginx/Dockerfile^dockerfile: nginx/Dockerfile\n image: doccano-nginx:custom^g' ../docker-compose.prod.yml +# Modify Dockerfile for nginx to add python3 and offline patch +sed -i 's|FROM nginx|COPY offline_deployment/offline_patcher.py /patch.py\ +RUN apk add -U --no-cache py3-requests \\\ + \&\& mkdir -p /app/dist/static/offline \&\& python3 /patch.py /app/dist /app/dist/static/offline /offline\ +\ +FROM nginx|' ../nginx/Dockerfile + docker-compose -f ../docker-compose.prod.yml pull docker-compose -f ../docker-compose.prod.yml build diff --git a/offline_deployment/offline_02_2-import_Docker_images.sh b/offline_deployment/offline_02_2-import_Docker_images.sh index b7b4f7a2..58c0fb9c 100755 --- a/offline_deployment/offline_02_2-import_Docker_images.sh +++ b/offline_deployment/offline_02_2-import_Docker_images.sh @@ -8,13 +8,6 @@ unset DIR sed -i 's^dockerfile: app/Dockerfile.prod^dockerfile: app/Dockerfile.prod\n image: doccano-app:custom^g' ../docker-compose.prod.yml sed -i 's^dockerfile: nginx/Dockerfile^dockerfile: nginx/Dockerfile\n image: doccano-nginx:custom^g' ../docker-compose.prod.yml -# Modify Dockerfile for nginx to add python3 and offline patch -sed -i 's|FROM nginx|COPY offline_deployment/offline_patcher.py /patch.py\ -RUN apk add -U --no-cache py3-requests \\\ - \&\& mkdir -p /app/dist/static/offline \&\& python3 /patch.py /app/dist /app/dist/static/offline /offline\ -\ -FROM nginx|' ../nginx/Dockerfile - # Load docker images docker image load -i doccano-app.tar docker image load -i doccano-nginx.tar From 419b863654b05e31551c6d3d43e113f77b3e10ce Mon Sep 17 00:00:00 2001 From: Johann Frei Date: Wed, 14 Apr 2021 20:35:01 +0200 Subject: [PATCH 13/28] Add offline deployment to dockerignore --- .dockerignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.dockerignore b/.dockerignore index cb81fd1b..a8f60b80 100644 --- a/.dockerignore +++ b/.dockerignore @@ -4,6 +4,7 @@ !nginx/ !tests/ !tools/ +!offline_deployment/ !.coveragerc !.flake8 !Dockerfile From 39b9dc3a2399f3ef0a1a321dcb58fb11b0871ca8 Mon Sep 17 00:00:00 2001 From: Johann Frei Date: Wed, 14 Apr 2021 20:35:14 +0200 Subject: [PATCH 14/28] Improve regex pattern --- offline_deployment/offline_patcher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/offline_deployment/offline_patcher.py b/offline_deployment/offline_patcher.py index 47b27b44..21bcaa9a 100644 --- a/offline_deployment/offline_patcher.py +++ b/offline_deployment/offline_patcher.py @@ -17,7 +17,7 @@ def main(): mime_ptn = re.compile(r"(?P(?P[\w^\/]+)\/(?P[\S\.^\;]+))(\;|$)", re.IGNORECASE) #link_ptn = re.compile(r"(?P[\S\"\'])(?Phttps?:\/\/(?P[\S^:\/)]+)(?P\:[0-9]+)?\/((?!(?P=encl)).)+)(?P=encl)", re.IGNORECASE) - link_ptn = re.compile(r"[\(\'\"\ ](?Phttps?:\/\/(?P[\S^:\/)]+)(?P\:[0-9]+)?\/[^\(\)\'\"\ ]+)(?P[\(\)\'\"\ ])") + link_ptn = re.compile(r"[\(\'\"\ ](?Phttps?:\/\/(?P(?P((?=[^\:\/])(?=[\S]).)+))(?P\:[0-9]+)?\/[^\(\)\'\"\ ]+)(?P[\(\)\'\"\ ])") # Block special hosts forbidden_hosts = [ From b04f7ec8dffefc749d1cac1b254c5eb166bd96c3 Mon Sep 17 00:00:00 2001 From: Johann Frei Date: Wed, 14 Apr 2021 23:53:41 +0200 Subject: [PATCH 15/28] Support local url references in css --- offline_deployment/offline_patcher.py | 84 ++++++++++++++++++--------- 1 file changed, 57 insertions(+), 27 deletions(-) diff --git a/offline_deployment/offline_patcher.py b/offline_deployment/offline_patcher.py index 21bcaa9a..9d9b18ae 100644 --- a/offline_deployment/offline_patcher.py +++ b/offline_deployment/offline_patcher.py @@ -1,6 +1,7 @@ import sys, os, re from glob import glob import uuid +from urllib.parse import urljoin import requests def main(): @@ -17,7 +18,11 @@ def main(): mime_ptn = re.compile(r"(?P(?P[\w^\/]+)\/(?P[\S\.^\;]+))(\;|$)", re.IGNORECASE) #link_ptn = re.compile(r"(?P[\S\"\'])(?Phttps?:\/\/(?P[\S^:\/)]+)(?P\:[0-9]+)?\/((?!(?P=encl)).)+)(?P=encl)", re.IGNORECASE) - link_ptn = re.compile(r"[\(\'\"\ ](?Phttps?:\/\/(?P(?P((?=[^\:\/])(?=[\S]).)+))(?P\:[0-9]+)?\/[^\(\)\'\"\ ]+)(?P[\(\)\'\"\ ])") + + # Regex to find matches like: "https://[:]/a/link/location.html" + link_ptn = re.compile(r"[\(\'\"\ ](?Phttps?:\/\/(?P(?P((?=[^\(\)\'\"\ \:\/])(?=[\S]).)+))(?P\:[0-9]+)?\/[^\(\)\'\"\ ]+)(?P[\(\)\'\"\ ])") + # Regex to find matches like: url(../relative/parent_directory/links/without/quotes/are/hard) + link_ptn_url = re.compile(r"url\([\"\']?(?P((?=[^\)\"\'])(?=[\S]).)+)[\"\']?\)") # Block special hosts forbidden_hosts = [ @@ -43,6 +48,7 @@ def main(): cached_urls = {} valid_urls = {} + file_origins = {} i = 0 while i < len(files_to_check): @@ -52,72 +58,96 @@ def main(): with open(file_i, "r", encoding="utf-8") as f: t = f.read() - for match in link_ptn.finditer(t): - found_link = match.group("link") - found_host = match.group("host") - - if found_link not in valid_urls: + link_findings_default = [ { + "abs": match.group("link"), + "found": match.group("link"), + "host": match.group("host") + } for match in link_ptn.finditer(t) ] + + # extract relative urls and convert them to absolute http urls + link_findings_url_prefix = [] + for match in link_ptn_url.finditer(t): + if os.path.abspath(file_i) in file_origins and not match.group("link").startswith("http"): + link_abs = urljoin(file_origins[os.path.abspath(file_i)], match.group("link")) + item = { + "abs": link_abs, + "found": match.group("link"), + "host": link_ptn.match( "\"" + link_abs + "\"").group("host") + } + link_findings_url_prefix.append(item) + + for spot in link_findings_default + link_findings_url_prefix: + absolute_link = spot["abs"] + found_link = spot["found"] + found_host = spot["host"] + + if absolute_link not in valid_urls: # check link - if True in [ True for fh in forbidden_hosts if fh.match(found_link) is not None ]: + if True in [ True for fh in forbidden_hosts if fh.match(absolute_link) is not None ]: # host is forbidden - valid_urls[found_link] = False + valid_urls[absolute_link] = False else: # host is not forbidden # check mime type - response = requests.head(found_link, allow_redirects=True) + response = requests.head(absolute_link, allow_redirects=True) mime = response.headers.get("Content-Type", None) if mime is None: - valid_urls[found_link] = False + valid_urls[absolute_link] = False else: mime_match = mime_ptn.match(mime) if mime_match is None: - valid_urls[found_link] = False + valid_urls[absolute_link] = False else: final_fext = None + # try supported content types for smt, get_fext in supported_mime_types: if smt(mime_match): final_fext = get_fext(mime_match) break if final_fext is None: # mime not supported - valid_urls[found_link] = False + valid_urls[absolute_link] = False else: # mime is supported -> store and remember file - valid_urls[found_link] = True + valid_urls[absolute_link] = True file_unique = uuid.uuid4() target_link = offline_link.format(file_unique, final_fext) target_file = offline_file.format(file_unique, final_fext) # download file try: - file_response = requests.get(found_link, allow_redirects=True) + file_response = requests.get(absolute_link, allow_redirects=True) file_response.raise_for_status() with open(target_file, 'wb') as download_file: for chunk in file_response.iter_content(100000): download_file.write(chunk) - # also check downloaded file + # also check downloaded file for links later files_to_check.append(target_file) - print("Downloaded file:", found_link) + print("Downloaded file:", absolute_link) except: - print("Link could not been downloaded:", found_link) + print("Link could not been downloaded:", absolute_link) # register downloaded file - cached_urls[found_link] = { - "input_link": found_link, + cached_urls[absolute_link] = { + "input_link": absolute_link, "target_link": target_link, "file": target_file, "fext": final_fext, - "found": [ file_i ] + "found": [ {"file": file_i, "found_link": found_link} ] } + # store reverse lookup for recursive url("../rel/link") patterns + file_origins[os.path.abspath(target_file)] = absolute_link - if valid_urls[found_link]: + if valid_urls[absolute_link]: # add to cached urls entries - cached_urls[found_link]["found"].append(file_i) + cached_urls[absolute_link]["found"].append({"file": file_i, "found_link": found_link}) print("Checked file:", file_i) + except UnicodeDecodeError: + print("Skip file (No unicode):", file_i) except: - print("Skip file:", file_i) + print("Unknown error... Skip file:", file_i) # look at next file i+= 1 @@ -125,13 +155,13 @@ def main(): # replace files with offline link for _, cached in cached_urls.items(): for edit_file in cached["found"]: - with open(edit_file, "r", encoding="utf-8") as f: + with open(edit_file["file"], "r", encoding="utf-8") as f: file_content = f.read() - with open(edit_file, "w", encoding="utf-8") as f: - f.write(file_content.replace(cached["input_link"], cached["target_link"])) + with open(edit_file["file"], "w", encoding="utf-8") as f: + f.write(file_content.replace(edit_file["found_link"], cached["target_link"])) print("Patched to", len(cached["found"]), "file with link:", cached["target_link"]) print("Done") if __name__ == "__main__": - main() \ No newline at end of file + main() From 7204236fe8e23dcaeeb2faf4665c94fc43f14798 Mon Sep 17 00:00:00 2001 From: Johann Frei Date: Thu, 15 Apr 2021 00:22:21 +0200 Subject: [PATCH 16/28] Add offline patch into scripts --- .../offline_01_2-extract_Docker_images.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/offline_deployment/offline_01_2-extract_Docker_images.sh b/offline_deployment/offline_01_2-extract_Docker_images.sh index 2b41725c..052e3c7e 100755 --- a/offline_deployment/offline_01_2-extract_Docker_images.sh +++ b/offline_deployment/offline_01_2-extract_Docker_images.sh @@ -18,6 +18,19 @@ RUN apk add -U --no-cache py3-requests \\\ \ FROM nginx|' ../nginx/Dockerfile +# Modify Dockerfile for app to add python3 and offline patch +# TODO: Remark: Not needed due to SPA frontend +#sed -i 's|COPY ./Pipfile\* /app/|COPY ./Pipfile\* /app/\ +#COPY offline_deployment/offline_patcher.py /patch.py\ +#RUN apt-get update \ +# \&\& apt-get install -y --no-install-recommends \ +# python3 python3-requests \ +# \&\& apt-get clean \\\ +# \&\& rm -rf /var/lib/apt/lists/\*\ +# \&\& mkdir -p /app/server/static/offline \&\& python3 /patch.py /app/server /server/static/offline\ +#\ +#|' ../app/Dockerfile.prod + docker-compose -f ../docker-compose.prod.yml pull docker-compose -f ../docker-compose.prod.yml build From 29459e284767f146863d041efe5504d7b10ff4a7 Mon Sep 17 00:00:00 2001 From: Johann Frei Date: Thu, 15 Apr 2021 00:25:47 +0200 Subject: [PATCH 17/28] Remove old offline patch script --- .../offline_01_1_1-download_assets.sh | 74 ------------------- 1 file changed, 74 deletions(-) delete mode 100755 offline_deployment/offline_01_1_1-download_assets.sh diff --git a/offline_deployment/offline_01_1_1-download_assets.sh b/offline_deployment/offline_01_1_1-download_assets.sh deleted file mode 100755 index a280b54a..00000000 --- a/offline_deployment/offline_01_1_1-download_assets.sh +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/env bash - -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -cd $DIR -cd .. -unset DIR - -# FOUND LINKS: -# app/server/templates/admin.html -# https://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/styles/vs2015.min.css - -# app/server/templates/base.html -# https://use.fontawesome.com/releases/v5.0.13/css/all.css -# https://fonts.googleapis.com/css?family=Open+Sans:300,400,700 -# https://cdnjs.cloudflare.com/ajax/libs/bulma/0.7.1/css/bulma.min.css -# https://cdn.jsdelivr.net/npm/bulma-extensions@4.0.1/bulma-divider/dist/css/bulma-divider.min.css -# https://cdn.jsdelivr.net/npm/bulma-extensions@4.0.1/bulma-checkradio/dist/css/bulma-checkradio.min.css -# https://cdn.jsdelivr.net/npm/bulma-extensions@4.0.1/bulma-tooltip/dist/css/bulma-tooltip.min.css - -# app/server/templates/index.html -# https://cdnjs.cloudflare.com/ajax/libs/Swiper/4.3.3/css/swiper.min.css - -# https://source.unsplash.com/RWnpyGtY1aU -# https://source.unsplash.com/6Ticnhs1AG0 -# https://i.imgsafe.org/ba/baa924a5e3.png - -# frontend/nuxt.config.js -# https://use.fontawesome.com/releases/v5.0.6/js/all.js -# https://fonts.googleapis.com/css?family=Roboto:300,400,500,700|Material+Icons - -n_columns="2" -declare -a links=("offline/vs2015.min.css" "https://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/styles/vs2015.min.css" - "offline/all.css" "https://use.fontawesome.com/releases/v5.0.13/css/all.css" - "offline/opensans.css" "https://fonts.googleapis.com/css?family=Open+Sans:300,400,700" - "offline/bulma.min.css" "https://cdnjs.cloudflare.com/ajax/libs/bulma/0.7.1/css/bulma.min.css" - "offline/bulma-divider.min.css" "https://cdn.jsdelivr.net/npm/bulma-extensions@4.0.1/bulma-divider/dist/css/bulma-divider.min.css" - "offline/bulma-checkradio.min.css" "https://cdn.jsdelivr.net/npm/bulma-extensions@4.0.1/bulma-checkradio/dist/css/bulma-checkradio.min.css" - "offline/bulma-tooltip.min.css" "https://cdn.jsdelivr.net/npm/bulma-extensions@4.0.1/bulma-tooltip/dist/css/bulma-tooltip.min.css" - "offline/photo-1.jpg" "https://source.unsplash.com/RWnpyGtY1aU" - "offline/photo-2.jpg" "https://source.unsplash.com/6Ticnhs1AG0" - "offline/photo-3.jpg" "https://i.imgsafe.org/ba/baa924a5e3.png" - - "offline/all.js" "https://use.fontawesome.com/releases/v5.0.6/js/all.js" - "offline/google-roboto.css" "https://fonts.googleapis.com/css?family=Roboto:300,400,500,700|Material+Icons" - ) - -static_dir_app="app/server/static/" -static_dir_frontend="frontend/static/" -mkdir -p "${static_dir_app}offline/" -mkdir -p "${static_dir_frontend}offline/" - -# root replace directories -app_dir="app/server/templates" -frontend_dir="frontend" - -for ((i = 0; i < $(expr "${#links[@]}" / "$n_columns"); ++i)); do - idx_local=$(expr $i \* $n_columns + 0) - idx_link=$(expr $i \* $n_columns + 1) - local="${links[$idx_local]}" - link="${links[$idx_link]}" - - echo "Storing file to $local: $link" - wget --content-on-error -q --show-progress -O "${static_dir_app}${local}" $link 2>/dev/null - if [ $? -eq 0 ]; then - # Copy to frontend static dir - cp "${static_dir_app}${local}" "${static_dir_frontend}${local}" - # For Django: Use 'static' for template, use ^ as delimiter for sed - find $app_dir -type f -exec sed -i "s^${link}^{% static \'${local}\' %}^g" {} \; - # For Vue: Use // for same host, use ^ as delimiter for sed - find $frontend_dir -type f -exec sed -i "s^${link}^/${local}^g" {} \; - else - echo "Failed to transform for offline use: $link" - fi -done From cdb5579f22b8dd06de78b24a9c33526a6922d6bf Mon Sep 17 00:00:00 2001 From: Johann Frei Date: Thu, 15 Apr 2021 00:26:00 +0200 Subject: [PATCH 18/28] Rename bash scripts --- ...2-optional_use_https.sh => offline_01_1-optional_use_https.sh} | 0 ..._images.sh => offline_01_2-patch_and_extract_Docker_images.sh} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename offline_deployment/{offline_01_1_2-optional_use_https.sh => offline_01_1-optional_use_https.sh} (100%) rename offline_deployment/{offline_01_2-extract_Docker_images.sh => offline_01_2-patch_and_extract_Docker_images.sh} (100%) diff --git a/offline_deployment/offline_01_1_2-optional_use_https.sh b/offline_deployment/offline_01_1-optional_use_https.sh similarity index 100% rename from offline_deployment/offline_01_1_2-optional_use_https.sh rename to offline_deployment/offline_01_1-optional_use_https.sh diff --git a/offline_deployment/offline_01_2-extract_Docker_images.sh b/offline_deployment/offline_01_2-patch_and_extract_Docker_images.sh similarity index 100% rename from offline_deployment/offline_01_2-extract_Docker_images.sh rename to offline_deployment/offline_01_2-patch_and_extract_Docker_images.sh From b0ab7b29b73ca6b9675f148042617db756412252 Mon Sep 17 00:00:00 2001 From: Johann Frei Date: Thu, 15 Apr 2021 00:52:05 +0200 Subject: [PATCH 19/28] Fix path for offline files --- .../offline_01_2-patch_and_extract_Docker_images.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/offline_deployment/offline_01_2-patch_and_extract_Docker_images.sh b/offline_deployment/offline_01_2-patch_and_extract_Docker_images.sh index 052e3c7e..0b0a3819 100755 --- a/offline_deployment/offline_01_2-patch_and_extract_Docker_images.sh +++ b/offline_deployment/offline_01_2-patch_and_extract_Docker_images.sh @@ -14,7 +14,7 @@ sed -i 's^dockerfile: nginx/Dockerfile^dockerfile: nginx/Dockerfile\n image: # Modify Dockerfile for nginx to add python3 and offline patch sed -i 's|FROM nginx|COPY offline_deployment/offline_patcher.py /patch.py\ RUN apk add -U --no-cache py3-requests \\\ - \&\& mkdir -p /app/dist/static/offline \&\& python3 /patch.py /app/dist /app/dist/static/offline /offline\ + \&\& mkdir -p /app/dist/offline \&\& python3 /patch.py /app/dist /app/dist/offline /offline\ \ FROM nginx|' ../nginx/Dockerfile From 4c192ca8c7cb393711bd79947d8a955f07ad2b80 Mon Sep 17 00:00:00 2001 From: Johann Frei Date: Wed, 21 Apr 2021 21:14:18 +0200 Subject: [PATCH 20/28] Refactor paths in scripts --- ...ine_01_2-patch_and_extract_Docker_images.sh | 18 +++++++++--------- .../offline_02_2-import_Docker_images.sh | 4 ++-- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/offline_deployment/offline_01_2-patch_and_extract_Docker_images.sh b/offline_deployment/offline_01_2-patch_and_extract_Docker_images.sh index 0b0a3819..09f5e7c4 100755 --- a/offline_deployment/offline_01_2-patch_and_extract_Docker_images.sh +++ b/offline_deployment/offline_01_2-patch_and_extract_Docker_images.sh @@ -8,32 +8,32 @@ unset DIR echo "Patching docker-compose to match Ubuntu 18.04 compose package" sed -i 's|version: "3.7"|version: "3.3"|g' ../docker-compose.prod.yml -sed -i 's^dockerfile: app/Dockerfile.prod^dockerfile: app/Dockerfile.prod\n image: doccano-app:custom^g' ../docker-compose.prod.yml +sed -i 's^dockerfile: backend/Dockerfile.prod^dockerfile: backend/Dockerfile.prod\n image: doccano-backend:custom^g' ../docker-compose.prod.yml sed -i 's^dockerfile: nginx/Dockerfile^dockerfile: nginx/Dockerfile\n image: doccano-nginx:custom^g' ../docker-compose.prod.yml # Modify Dockerfile for nginx to add python3 and offline patch sed -i 's|FROM nginx|COPY offline_deployment/offline_patcher.py /patch.py\ RUN apk add -U --no-cache py3-requests \\\ - \&\& mkdir -p /app/dist/offline \&\& python3 /patch.py /app/dist /app/dist/offline /offline\ + \&\& mkdir -p /backend/dist/offline \&\& python3 /patch.py /backend/dist /backend/dist/offline /offline\ \ FROM nginx|' ../nginx/Dockerfile -# Modify Dockerfile for app to add python3 and offline patch +# Modify Dockerfile for backend to add python3 and offline patch # TODO: Remark: Not needed due to SPA frontend -#sed -i 's|COPY ./Pipfile\* /app/|COPY ./Pipfile\* /app/\ +#sed -i 's|COPY ./Pipfile\* /backend/|COPY ./Pipfile\* /backend/\ #COPY offline_deployment/offline_patcher.py /patch.py\ #RUN apt-get update \ # \&\& apt-get install -y --no-install-recommends \ # python3 python3-requests \ # \&\& apt-get clean \\\ # \&\& rm -rf /var/lib/apt/lists/\*\ -# \&\& mkdir -p /app/server/static/offline \&\& python3 /patch.py /app/server /server/static/offline\ +# \&\& mkdir -p /backend/server/static/offline \&\& python3 /patch.py /backend/server /server/static/offline\ #\ -#|' ../app/Dockerfile.prod +#|' ../backend/Dockerfile.prod docker-compose -f ../docker-compose.prod.yml pull docker-compose -f ../docker-compose.prod.yml build -docker image save -o doccano-app.tar doccano-app:custom -docker image save -o doccano-nginx.tar doccano-nginx:custom -docker image save -o postgres.tar postgres:13.1-alpine +docker image save -o doccano-backend.tar doccano-backend:custom +docker image save -o doccano-nginx.tar doccano-nginx:custom +docker image save -o postgres.tar postgres:13.1-alpine diff --git a/offline_deployment/offline_02_2-import_Docker_images.sh b/offline_deployment/offline_02_2-import_Docker_images.sh index 58c0fb9c..01c759ac 100755 --- a/offline_deployment/offline_02_2-import_Docker_images.sh +++ b/offline_deployment/offline_02_2-import_Docker_images.sh @@ -5,10 +5,10 @@ cd $DIR unset DIR # Set image tag in Compose to avoid image build -sed -i 's^dockerfile: app/Dockerfile.prod^dockerfile: app/Dockerfile.prod\n image: doccano-app:custom^g' ../docker-compose.prod.yml +sed -i 's^dockerfile: backend/Dockerfile.prod^dockerfile: backend/Dockerfile.prod\n image: doccano-backend:custom^g' ../docker-compose.prod.yml sed -i 's^dockerfile: nginx/Dockerfile^dockerfile: nginx/Dockerfile\n image: doccano-nginx:custom^g' ../docker-compose.prod.yml # Load docker images -docker image load -i doccano-app.tar +docker image load -i doccano-backend.tar docker image load -i doccano-nginx.tar docker image load -i postgres.tar From 19248a97413aeebff564ae7af0a164385e75d861 Mon Sep 17 00:00:00 2001 From: Johann Frei Date: Wed, 21 Apr 2021 21:14:49 +0200 Subject: [PATCH 21/28] Add info to offline patcher script --- offline_deployment/offline_patcher.py | 48 +++++++++++++++++++++++---- 1 file changed, 42 insertions(+), 6 deletions(-) diff --git a/offline_deployment/offline_patcher.py b/offline_deployment/offline_patcher.py index 9d9b18ae..720dddd7 100644 --- a/offline_deployment/offline_patcher.py +++ b/offline_deployment/offline_patcher.py @@ -1,9 +1,47 @@ +#!/usr/bin/env python3 import sys, os, re -from glob import glob import uuid from urllib.parse import urljoin import requests +"""Script Information + +This script scans all files of a given directory [1] for URL addresses and +hyperlink references. +All found URLs are requested for Content-Type. +For certain Content-Types (like js, css, or fonts), the file is downloaded and +stored locally into a given directory [2] and the existing URLs are altered +to a local URL location (with a given URL prefix [3]). + +Downloaded files are scanned for URLs recursively. +Relative references in CSS files are an edge case that is +handled separately by a specific regex pattern. + +Arguments: + 1. + 2. + 3. + +Example: + - Given: + - File ./webspace/index.html, containing URL: https://example.com/library.js + - Directory ./webspace/static, containing static files, + serving content on HTTP location: /staticfiles + + - Call: + $> python3 offline_patcher.py webspace/ webspace/static /staticfiles + + - Result: + - Library from https://example.com/library.js is stored as file: + webspace/static/offline_.js + - Link in file webspace/index.html is replaced to: + /staticfiles/offline_.js + - File webspace/static/offline_.js is scanned recursively for URLs + +Author: Johann Frei +""" + + def main(): # root folder to scan for URLs root_folder = sys.argv[1] @@ -12,19 +50,17 @@ def main(): # offline link prefix offline_prefix = sys.argv[3] - offline_file = os.path.join(offline_folder, "offline_{}.{}") offline_link = offline_prefix + "/offline_{}.{}" mime_ptn = re.compile(r"(?P(?P[\w^\/]+)\/(?P[\S\.^\;]+))(\;|$)", re.IGNORECASE) - #link_ptn = re.compile(r"(?P[\S\"\'])(?Phttps?:\/\/(?P[\S^:\/)]+)(?P\:[0-9]+)?\/((?!(?P=encl)).)+)(?P=encl)", re.IGNORECASE) - # Regex to find matches like: "https://[:]/a/link/location.html" + # regex to find matches like: "https://[:]/a/link/location.html" link_ptn = re.compile(r"[\(\'\"\ ](?Phttps?:\/\/(?P(?P((?=[^\(\)\'\"\ \:\/])(?=[\S]).)+))(?P\:[0-9]+)?\/[^\(\)\'\"\ ]+)(?P[\(\)\'\"\ ])") - # Regex to find matches like: url(../relative/parent_directory/links/without/quotes/are/hard) + # regex to find matches like: url(../relative/parent_directory/links/without/quotes/are/hard) link_ptn_url = re.compile(r"url\([\"\']?(?P((?=[^\)\"\'])(?=[\S]).)+)[\"\']?\)") - # Block special hosts + # block special hosts forbidden_hosts = [ re.compile(r"^.*registry\.npmjs\.org$"), # No yarnpkg repository re.compile(r"^.*yarnpkg\.com$"), # No yarnpkg repository From af060a5dd9c5261f7bacad24201d532e55b4c809 Mon Sep 17 00:00:00 2001 From: Johann Frei Date: Wed, 21 Apr 2021 21:15:02 +0200 Subject: [PATCH 22/28] Update offline deployment README --- offline_deployment/README.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/offline_deployment/README.md b/offline_deployment/README.md index 16521b78..b1168575 100644 --- a/offline_deployment/README.md +++ b/offline_deployment/README.md @@ -1,7 +1,7 @@ # Doccano Offline Deployment ## Use Case -These offline deployment scripts are suited for deploying Doccano on an air gapped Ubuntu 18.04/20.04 virtual machine (VM 2) with no internet connectivity. +These offline deployment scripts are suited for deploying Doccano on an air gapped Ubuntu 18.04/20.04 virtual machine (VM 2) with no internet connectivity (such as in clinical environments). The preparation requires another machine (VM 1) with internet access and `docker`/`docker-compose` preinstalled (with $USER in `docker` group) and running the same Ubuntu distribution as VM 2. @@ -12,12 +12,17 @@ The focus is primarily on the `docker-compose`-based production deployment. Run the following steps on VM 1: 1. Clone this repository 2. Run the scripts `offline_01_*.sh` in ascending order - Skip OR modify and run the script `offline_01_1_2-optional_use_https` + Skip OR modify and run the script `offline_01_1-optional_use_https` + Do NOT run these scripts as `sudo`! The scripts will ask for sudo-permissions when it is needed. Now, move over to VM 2 3. Copy the repository folder from VM 1 to VM 2 4. Run the scripts `offline_02_*.sh` in ascending order + Do NOT run these scripts as `sudo`! The scripts will ask for sudo-permissions when it is needed. 5. Make minor changes on `docker-compose.prod.yml` to change the admin credentials 6. Run `docker-compose -f docker-compose.prod.yml up` or use the script `offline_03_*.sh` +## Remarks + +The setup was tested on Ubuntu 18.04 machines. \ No newline at end of file From f8be085e020e721880a14682dbb042942a732efd Mon Sep 17 00:00:00 2001 From: Johann Frei Date: Wed, 21 Apr 2021 21:38:40 +0200 Subject: [PATCH 23/28] Add rabbitmq docker image --- .../offline_01_2-patch_and_extract_Docker_images.sh | 1 + offline_deployment/offline_02_2-import_Docker_images.sh | 1 + 2 files changed, 2 insertions(+) diff --git a/offline_deployment/offline_01_2-patch_and_extract_Docker_images.sh b/offline_deployment/offline_01_2-patch_and_extract_Docker_images.sh index 09f5e7c4..70933f05 100755 --- a/offline_deployment/offline_01_2-patch_and_extract_Docker_images.sh +++ b/offline_deployment/offline_01_2-patch_and_extract_Docker_images.sh @@ -37,3 +37,4 @@ docker-compose -f ../docker-compose.prod.yml build docker image save -o doccano-backend.tar doccano-backend:custom docker image save -o doccano-nginx.tar doccano-nginx:custom docker image save -o postgres.tar postgres:13.1-alpine +docker image save -o rabbitmq.tar rabbitmq:3.8 diff --git a/offline_deployment/offline_02_2-import_Docker_images.sh b/offline_deployment/offline_02_2-import_Docker_images.sh index 01c759ac..38bebefc 100755 --- a/offline_deployment/offline_02_2-import_Docker_images.sh +++ b/offline_deployment/offline_02_2-import_Docker_images.sh @@ -12,3 +12,4 @@ sed -i 's^dockerfile: nginx/Dockerfile^dockerfile: nginx/Dockerfile\n image: docker image load -i doccano-backend.tar docker image load -i doccano-nginx.tar docker image load -i postgres.tar +docker image load -i rabbitmq.tar From b076d6bbf1e3a0f8336d29e1c4d7d74bf4a35229 Mon Sep 17 00:00:00 2001 From: Johann Frei Date: Wed, 21 Apr 2021 21:43:04 +0200 Subject: [PATCH 24/28] Fix wrong path for frontend --- .../offline_01_2-patch_and_extract_Docker_images.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/offline_deployment/offline_01_2-patch_and_extract_Docker_images.sh b/offline_deployment/offline_01_2-patch_and_extract_Docker_images.sh index 70933f05..29aa4d8c 100755 --- a/offline_deployment/offline_01_2-patch_and_extract_Docker_images.sh +++ b/offline_deployment/offline_01_2-patch_and_extract_Docker_images.sh @@ -14,7 +14,7 @@ sed -i 's^dockerfile: nginx/Dockerfile^dockerfile: nginx/Dockerfile\n image: # Modify Dockerfile for nginx to add python3 and offline patch sed -i 's|FROM nginx|COPY offline_deployment/offline_patcher.py /patch.py\ RUN apk add -U --no-cache py3-requests \\\ - \&\& mkdir -p /backend/dist/offline \&\& python3 /patch.py /backend/dist /backend/dist/offline /offline\ + \&\& mkdir -p /app/dist/offline \&\& python3 /patch.py /app/dist /app/dist/offline /offline\ \ FROM nginx|' ../nginx/Dockerfile From 784cea583cb62cdc0db10bec9f773c9489bddb4f Mon Sep 17 00:00:00 2001 From: Johann Frei Date: Thu, 22 Apr 2021 00:21:10 +0200 Subject: [PATCH 25/28] Patch compose file only once --- .../offline_01_2-patch_and_extract_Docker_images.sh | 2 +- offline_deployment/offline_02_2-import_Docker_images.sh | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/offline_deployment/offline_01_2-patch_and_extract_Docker_images.sh b/offline_deployment/offline_01_2-patch_and_extract_Docker_images.sh index 29aa4d8c..ce2f127f 100755 --- a/offline_deployment/offline_01_2-patch_and_extract_Docker_images.sh +++ b/offline_deployment/offline_01_2-patch_and_extract_Docker_images.sh @@ -9,7 +9,7 @@ echo "Patching docker-compose to match Ubuntu 18.04 compose package" sed -i 's|version: "3.7"|version: "3.3"|g' ../docker-compose.prod.yml sed -i 's^dockerfile: backend/Dockerfile.prod^dockerfile: backend/Dockerfile.prod\n image: doccano-backend:custom^g' ../docker-compose.prod.yml -sed -i 's^dockerfile: nginx/Dockerfile^dockerfile: nginx/Dockerfile\n image: doccano-nginx:custom^g' ../docker-compose.prod.yml +sed -i 's^dockerfile: nginx/Dockerfile^dockerfile: nginx/Dockerfile\n image: doccano-nginx:custom^g' ../docker-compose.prod.yml # Modify Dockerfile for nginx to add python3 and offline patch sed -i 's|FROM nginx|COPY offline_deployment/offline_patcher.py /patch.py\ diff --git a/offline_deployment/offline_02_2-import_Docker_images.sh b/offline_deployment/offline_02_2-import_Docker_images.sh index 38bebefc..a124ec9d 100755 --- a/offline_deployment/offline_02_2-import_Docker_images.sh +++ b/offline_deployment/offline_02_2-import_Docker_images.sh @@ -4,9 +4,10 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd $DIR unset DIR -# Set image tag in Compose to avoid image build -sed -i 's^dockerfile: backend/Dockerfile.prod^dockerfile: backend/Dockerfile.prod\n image: doccano-backend:custom^g' ../docker-compose.prod.yml -sed -i 's^dockerfile: nginx/Dockerfile^dockerfile: nginx/Dockerfile\n image: doccano-nginx:custom^g' ../docker-compose.prod.yml +# Info: Docker image name is already set in previous scripts +## Set image tag in Compose to avoid image build +#sed -i 's^dockerfile: backend/Dockerfile.prod^dockerfile: backend/Dockerfile.prod\n image: doccano-backend:custom^g' ../docker-compose.prod.yml +#sed -i 's^dockerfile: nginx/Dockerfile^dockerfile: nginx/Dockerfile\n image: doccano-nginx:custom^g' ../docker-compose.prod.yml # Load docker images docker image load -i doccano-backend.tar From 5c89606f40c71b471936fde8562be01cb749d5fe Mon Sep 17 00:00:00 2001 From: Johann Frei Date: Thu, 22 Apr 2021 00:27:19 +0200 Subject: [PATCH 26/28] Use neutral x509 self-signed cert --- offline_deployment/offline_01_1-optional_use_https.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/offline_deployment/offline_01_1-optional_use_https.sh b/offline_deployment/offline_01_1-optional_use_https.sh index b15c8e7b..990ec3fc 100755 --- a/offline_deployment/offline_01_1-optional_use_https.sh +++ b/offline_deployment/offline_01_1-optional_use_https.sh @@ -8,7 +8,7 @@ unset DIR # create certificate pair sudo apt-get install -y openssl openssl req -new -newkey rsa:4096 -sha256 -nodes -x509 -keyout ./nginx/cert.key -out ./nginx/cert.crt \ - -subj "/C=DE/ST=Bayern/L=Augsburg/O=Universität Augsburg/OU=Misit/CN=*.informatik.uni-augsburg.de" + -subj "/C=US/ST=StateCode/L=LocationName/O=OrganizationName/OU=OrganizationUnit/CN=doccano.herokuapp.com" # define cert paths inside container ssl_cert="/certs/cert.crt" From f353170a0b09ee2e9b5e9c09f33ca63e401c8deb Mon Sep 17 00:00:00 2001 From: Johann Frei Date: Mon, 26 Apr 2021 14:46:23 +0200 Subject: [PATCH 27/28] Move and adapt deployment files --- .../advanced/offline_deployment.md | 3 ++- .../offline_01_1-optional_use_https.sh | 2 +- ...ine_01_2-patch_and_extract_Docker_images.sh | 18 +++++++++--------- .../offline_01_3-download_APT_packages.sh | 0 .../offline_02_1-install_APT_packages.sh | 0 .../offline_02_2-import_Docker_images.sh | 4 ++-- .../offline_03_1-runDoccano.sh | 2 +- .../offline_deployment}/offline_patcher.py | 0 8 files changed, 15 insertions(+), 14 deletions(-) rename offline_deployment/README.md => docs/advanced/offline_deployment.md (81%) rename {offline_deployment => tools/offline_deployment}/offline_01_1-optional_use_https.sh (99%) rename {offline_deployment => tools/offline_deployment}/offline_01_2-patch_and_extract_Docker_images.sh (69%) rename {offline_deployment => tools/offline_deployment}/offline_01_3-download_APT_packages.sh (100%) rename {offline_deployment => tools/offline_deployment}/offline_02_1-install_APT_packages.sh (100%) rename {offline_deployment => tools/offline_deployment}/offline_02_2-import_Docker_images.sh (74%) rename {offline_deployment => tools/offline_deployment}/offline_03_1-runDoccano.sh (64%) rename {offline_deployment => tools/offline_deployment}/offline_patcher.py (100%) diff --git a/offline_deployment/README.md b/docs/advanced/offline_deployment.md similarity index 81% rename from offline_deployment/README.md rename to docs/advanced/offline_deployment.md index b1168575..906161e2 100644 --- a/offline_deployment/README.md +++ b/docs/advanced/offline_deployment.md @@ -6,6 +6,7 @@ These offline deployment scripts are suited for deploying Doccano on an air gapp The preparation requires another machine (VM 1) with internet access and `docker`/`docker-compose` preinstalled (with $USER in `docker` group) and running the same Ubuntu distribution as VM 2. The focus is primarily on the `docker-compose`-based production deployment. +The files mentioned in this document are located in the `tools/offline_deployment/` directory. ## Setup Steps @@ -21,7 +22,7 @@ Now, move over to VM 2 4. Run the scripts `offline_02_*.sh` in ascending order Do NOT run these scripts as `sudo`! The scripts will ask for sudo-permissions when it is needed. 5. Make minor changes on `docker-compose.prod.yml` to change the admin credentials -6. Run `docker-compose -f docker-compose.prod.yml up` or use the script `offline_03_*.sh` +6. Run `docker-compose -f docker-compose.prod.yml up` in the repository root directory or use the script `offline_03_*.sh` ## Remarks diff --git a/offline_deployment/offline_01_1-optional_use_https.sh b/tools/offline_deployment/offline_01_1-optional_use_https.sh similarity index 99% rename from offline_deployment/offline_01_1-optional_use_https.sh rename to tools/offline_deployment/offline_01_1-optional_use_https.sh index 990ec3fc..468b0edf 100755 --- a/offline_deployment/offline_01_1-optional_use_https.sh +++ b/tools/offline_deployment/offline_01_1-optional_use_https.sh @@ -2,7 +2,7 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd $DIR -cd .. +cd ../.. unset DIR # create certificate pair diff --git a/offline_deployment/offline_01_2-patch_and_extract_Docker_images.sh b/tools/offline_deployment/offline_01_2-patch_and_extract_Docker_images.sh similarity index 69% rename from offline_deployment/offline_01_2-patch_and_extract_Docker_images.sh rename to tools/offline_deployment/offline_01_2-patch_and_extract_Docker_images.sh index ce2f127f..1024af24 100755 --- a/offline_deployment/offline_01_2-patch_and_extract_Docker_images.sh +++ b/tools/offline_deployment/offline_01_2-patch_and_extract_Docker_images.sh @@ -6,22 +6,22 @@ unset DIR # WORKAROUND: Downgrade docker-compose version to match Ubuntu 18.04 default compose package echo "Patching docker-compose to match Ubuntu 18.04 compose package" -sed -i 's|version: "3.7"|version: "3.3"|g' ../docker-compose.prod.yml +sed -i 's|version: "3.7"|version: "3.3"|g' ../../docker-compose.prod.yml -sed -i 's^dockerfile: backend/Dockerfile.prod^dockerfile: backend/Dockerfile.prod\n image: doccano-backend:custom^g' ../docker-compose.prod.yml -sed -i 's^dockerfile: nginx/Dockerfile^dockerfile: nginx/Dockerfile\n image: doccano-nginx:custom^g' ../docker-compose.prod.yml +sed -i 's^dockerfile: backend/Dockerfile.prod^dockerfile: backend/Dockerfile.prod\n image: doccano-backend:custom^g' ../../docker-compose.prod.yml +sed -i 's^dockerfile: nginx/Dockerfile^dockerfile: nginx/Dockerfile\n image: doccano-nginx:custom^g' ../../docker-compose.prod.yml # Modify Dockerfile for nginx to add python3 and offline patch -sed -i 's|FROM nginx|COPY offline_deployment/offline_patcher.py /patch.py\ +sed -i 's|FROM nginx|COPY tools/offline_deployment/offline_patcher.py /patch.py\ RUN apk add -U --no-cache py3-requests \\\ \&\& mkdir -p /app/dist/offline \&\& python3 /patch.py /app/dist /app/dist/offline /offline\ \ -FROM nginx|' ../nginx/Dockerfile +FROM nginx|' ../../nginx/Dockerfile # Modify Dockerfile for backend to add python3 and offline patch # TODO: Remark: Not needed due to SPA frontend #sed -i 's|COPY ./Pipfile\* /backend/|COPY ./Pipfile\* /backend/\ -#COPY offline_deployment/offline_patcher.py /patch.py\ +#COPY tools/offline_deployment/offline_patcher.py /patch.py\ #RUN apt-get update \ # \&\& apt-get install -y --no-install-recommends \ # python3 python3-requests \ @@ -29,10 +29,10 @@ FROM nginx|' ../nginx/Dockerfile # \&\& rm -rf /var/lib/apt/lists/\*\ # \&\& mkdir -p /backend/server/static/offline \&\& python3 /patch.py /backend/server /server/static/offline\ #\ -#|' ../backend/Dockerfile.prod +#|' ../../backend/Dockerfile.prod -docker-compose -f ../docker-compose.prod.yml pull -docker-compose -f ../docker-compose.prod.yml build +docker-compose -f ../../docker-compose.prod.yml pull +docker-compose -f ../../docker-compose.prod.yml build docker image save -o doccano-backend.tar doccano-backend:custom docker image save -o doccano-nginx.tar doccano-nginx:custom diff --git a/offline_deployment/offline_01_3-download_APT_packages.sh b/tools/offline_deployment/offline_01_3-download_APT_packages.sh similarity index 100% rename from offline_deployment/offline_01_3-download_APT_packages.sh rename to tools/offline_deployment/offline_01_3-download_APT_packages.sh diff --git a/offline_deployment/offline_02_1-install_APT_packages.sh b/tools/offline_deployment/offline_02_1-install_APT_packages.sh similarity index 100% rename from offline_deployment/offline_02_1-install_APT_packages.sh rename to tools/offline_deployment/offline_02_1-install_APT_packages.sh diff --git a/offline_deployment/offline_02_2-import_Docker_images.sh b/tools/offline_deployment/offline_02_2-import_Docker_images.sh similarity index 74% rename from offline_deployment/offline_02_2-import_Docker_images.sh rename to tools/offline_deployment/offline_02_2-import_Docker_images.sh index a124ec9d..e2509e7f 100755 --- a/offline_deployment/offline_02_2-import_Docker_images.sh +++ b/tools/offline_deployment/offline_02_2-import_Docker_images.sh @@ -6,8 +6,8 @@ unset DIR # Info: Docker image name is already set in previous scripts ## Set image tag in Compose to avoid image build -#sed -i 's^dockerfile: backend/Dockerfile.prod^dockerfile: backend/Dockerfile.prod\n image: doccano-backend:custom^g' ../docker-compose.prod.yml -#sed -i 's^dockerfile: nginx/Dockerfile^dockerfile: nginx/Dockerfile\n image: doccano-nginx:custom^g' ../docker-compose.prod.yml +#sed -i 's^dockerfile: backend/Dockerfile.prod^dockerfile: backend/Dockerfile.prod\n image: doccano-backend:custom^g' ../../docker-compose.prod.yml +#sed -i 's^dockerfile: nginx/Dockerfile^dockerfile: nginx/Dockerfile\n image: doccano-nginx:custom^g' ../../docker-compose.prod.yml # Load docker images docker image load -i doccano-backend.tar diff --git a/offline_deployment/offline_03_1-runDoccano.sh b/tools/offline_deployment/offline_03_1-runDoccano.sh similarity index 64% rename from offline_deployment/offline_03_1-runDoccano.sh rename to tools/offline_deployment/offline_03_1-runDoccano.sh index dec8e02e..95ac7ec2 100755 --- a/offline_deployment/offline_03_1-runDoccano.sh +++ b/tools/offline_deployment/offline_03_1-runDoccano.sh @@ -4,4 +4,4 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd $DIR unset DIR -docker-compose -f ../docker-compose.prod.yml up -d +docker-compose -f ../../docker-compose.prod.yml up -d diff --git a/offline_deployment/offline_patcher.py b/tools/offline_deployment/offline_patcher.py similarity index 100% rename from offline_deployment/offline_patcher.py rename to tools/offline_deployment/offline_patcher.py From 195664b22a643e918a1c49325d34f0e6ca05fb08 Mon Sep 17 00:00:00 2001 From: Johann Frei Date: Mon, 26 Apr 2021 14:47:22 +0200 Subject: [PATCH 28/28] Remove empty directory from dockerignore --- .dockerignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.dockerignore b/.dockerignore index 4eeef9ef..8c91f960 100644 --- a/.dockerignore +++ b/.dockerignore @@ -4,7 +4,6 @@ !nginx/ !tests/ !tools/ -!offline_deployment/ !.coveragerc !.flake8 !Dockerfile