From a6e0ddd95947e35d68d75da13601bb14a71c87e6 Mon Sep 17 00:00:00 2001 From: Clemens Wolff Date: Thu, 16 May 2019 18:24:12 -0400 Subject: [PATCH 1/6] Add option to publish to non DockerHub registry --- tools/cd.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tools/cd.sh b/tools/cd.sh index 1b6b46df..591408a5 100755 --- a/tools/cd.sh +++ b/tools/cd.sh @@ -6,10 +6,15 @@ if [[ -z "$1" ]]; then echo "Usage: $0 " >&2; exit 1; fi set -o errexit +if [[ -z "${DOCKER_REGISTRY}" ]]; then + echo "${DOCKER_PASSWORD}" | docker login --username "${DOCKER_USERNAME}" --password-stdin +else + echo "${DOCKER_PASSWORD}" | docker login --username "${DOCKER_USERNAME}" --password-stdin "${DOCKER_REGISTRY}" + DOCKER_USERNAME="${DOCKER_REGISTRY}" +fi + docker build -t "${DOCKER_USERNAME}/doccano:latest" . docker build -t "${DOCKER_USERNAME}/doccano:$1" . -echo "${DOCKER_PASSWORD}" | docker login --username "${DOCKER_USERNAME}" --password-stdin - docker push "${DOCKER_USERNAME}/doccano:latest" docker push "${DOCKER_USERNAME}/doccano:$1" From e249f3b0effd76ecb5fd62bbefdbc6fd01526f59 Mon Sep 17 00:00:00 2001 From: Clemens Wolff Date: Thu, 16 May 2019 20:13:39 -0400 Subject: [PATCH 2/6] Add support for private docker registry --- azuredeploy.json | 41 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/azuredeploy.json b/azuredeploy.json index e9067d50..fe286a6b 100644 --- a/azuredeploy.json +++ b/azuredeploy.json @@ -90,6 +90,27 @@ "metadata": { "description": "The Docker image to deploy." } + }, + "dockerRegistry": { + "type": "string", + "defaultValue": "", + "metadata": { + "description": "The registry of the Docker image." + } + }, + "dockerRegistryUserName": { + "type": "string", + "defaultValue": "", + "metadata": { + "description": "The user name for the Docker registry." + } + }, + "dockerRegistryPassword": { + "type": "securestring", + "defaultValue": "", + "metadata": { + "description": "The password for the Docker registry." + } } }, "variables": { @@ -102,7 +123,12 @@ "databaseServerName": "[concat(parameters('appName'),'-state')]", "setupScriptName": "[concat(parameters('appName'),'-setup')]", "appServicePlanName": "[concat(parameters('appName'),'-hosting')]", - "analyticsName": "[concat(parameters('appName'),'-analytics')]" + "analyticsName": "[concat(parameters('appName'),'-analytics')]", + "dockerRegistryCredential": { + "password": "[parameters('dockerRegistryPassword')]", + "username": "[parameters('dockerRegistryUserName')]", + "server": "[parameters('dockerRegistry')]" + } }, "resources": [ { @@ -182,6 +208,7 @@ "apiVersion": "2018-10-01", "location": "[variables('location')]", "properties": { + "imageRegistryCredentials": "[if(equals(parameters('dockerRegistry'), ''), json('null'), array(variables('dockerRegistryCredential')))]", "containers": [ { "name": "createadmin", @@ -244,6 +271,18 @@ "name": "WEBSITES_ENABLE_APP_SERVICE_STORAGE", "value": "false" }, + { + "name": "DOCKER_REGISTRY_SERVER_URL", + "value": "[parameters('dockerRegistry')]" + }, + { + "name": "DOCKER_REGISTRY_SERVER_USERNAME", + "value": "[parameters('dockerRegistryUserName')]" + }, + { + "name": "DOCKER_REGISTRY_SERVER_PASSWORD", + "value": "[parameters('dockerRegistryPassword')]" + }, { "name": "AZURE_APPINSIGHTS_IKEY", "value": "[reference(resourceId('Microsoft.Insights/components', variables('analyticsName')), '2014-04-01').InstrumentationKey]" From 0965baa55f7033df9ee49659edb8c5198b8da1e0 Mon Sep 17 00:00:00 2001 From: Clemens Wolff Date: Thu, 16 May 2019 18:25:13 -0400 Subject: [PATCH 3/6] Add Azure Pipelines CI --- azure-pipelines.yaml | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 azure-pipelines.yaml diff --git a/azure-pipelines.yaml b/azure-pipelines.yaml new file mode 100644 index 00000000..4b5d714f --- /dev/null +++ b/azure-pipelines.yaml @@ -0,0 +1,13 @@ +trigger: +- master + +pool: + vmImage: 'ubuntu-latest' + +steps: +- script: docker build --target=builder . + displayName: 'Run tests' + +- script: DOCKER_PASSWORD="$(docker_password)" tools/cd.sh "azdo-$(Build.BuildId)" + displayName: 'Push docker image' + condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master'), ne(variables['docker_password'], '')) From dd57112ff618848d4ebee844859e9fc089600e0a Mon Sep 17 00:00:00 2001 From: Clemens Wolff Date: Thu, 16 May 2019 20:56:42 -0400 Subject: [PATCH 4/6] Add Azure Pipelines CD --- azure-pipelines.yaml | 10 ++++++++++ tools/azure.sh | 31 +++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+) create mode 100755 tools/azure.sh diff --git a/azure-pipelines.yaml b/azure-pipelines.yaml index 4b5d714f..42b0ca7c 100644 --- a/azure-pipelines.yaml +++ b/azure-pipelines.yaml @@ -11,3 +11,13 @@ steps: - script: DOCKER_PASSWORD="$(docker_password)" tools/cd.sh "azdo-$(Build.BuildId)" displayName: 'Push docker image' condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master'), ne(variables['docker_password'], '')) + +- script: | + az login --service-principal --password "$(auth_password)" --tenant "$(auth_tenant)" --username "$(auth_username)" + + DOCCANO_ADMIN_PASSWORD="$(doccano_admin_password)" \ + DOCCANO_SECRET_KEY="$(doccano_secret_key)" \ + DOCKER_PASSWORD="$(docker_password)" \ + tools/azure.sh "azdo-$(Build.BuildId)" + displayName: 'Deploy to Azure' + condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master'), ne(variables['auth_password'], '')) diff --git a/tools/azure.sh b/tools/azure.sh new file mode 100755 index 00000000..5ed5839b --- /dev/null +++ b/tools/azure.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +set -o errexit + +if [[ -z "${DOCCANO_LOCATION}" ]]; then echo "Missing DOCCANO_LOCATION environment variable" >&2; exit 1; fi +if [[ -z "${DOCCANO_RESOURCE_GROUP}" ]]; then echo "Missing DOCCANO_LOCATION environment variable" >&2; exit 1; fi +if [[ -z "${DOCCANO_APP_NAME}" ]]; then echo "Missing DOCCANO_APP_NAME environment variable" >&2; exit 1; fi +if [[ -z "${DOCCANO_SECRET_KEY}" ]]; then echo "Missing DOCCANO_SECRET_KEY environment variable" >&2; exit 1; fi +if [[ -z "${DOCCANO_ADMIN_USERNAME}" ]]; then echo "Missing DOCCANO_ADMIN_USERNAME environment variable" >&2; exit 1; fi +if [[ -z "${DOCCANO_ADMIN_CONTACT_EMAIL}" ]]; then echo "Missing DOCCANO_ADMIN_CONTACT_EMAIL environment variable" >&2; exit 1; fi +if [[ -z "${DOCCANO_ADMIN_PASSWORD}" ]]; then echo "Missing DOCCANO_ADMIN_PASSWORD environment variable" >&2; exit 1; fi +if ! az account show >/dev/null; then echo "Must be logged into Azure" >&2; exit 2; fi + +az group create \ + --location "${DOCCANO_LOCATION}" \ + --name "${DOCCANO_RESOURCE_GROUP}" + +az group deployment create \ + --resource-group "${DOCCANO_RESOURCE_GROUP}" \ + --name "azuredeploy$1" \ + --parameters \ + appName="${DOCCANO_APP_NAME}" \ + secretKey="${DOCCANO_SECRET_KEY}" \ + adminUserName="${DOCCANO_ADMIN_USERNAME}" \ + adminContactEmail="${DOCCANO_ADMIN_CONTACT_EMAIL}" \ + adminPassword="${DOCCANO_ADMIN_PASSWORD}" \ + dockerImageName="${DOCKER_REGISTRY:-${DOCKER_USERNAME:-chakkiworks}}/doccano:${1:-latest}" \ + dockerRegistry="${DOCKER_REGISTRY}" \ + dockerRegistryUserName="${DOCKER_USERNAME}" \ + dockerRegistryPassword="${DOCKER_PASSWORD}" \ + --template-file azuredeploy.json From 495fe566057393cc0af8ab4f90c2a0b514a619b5 Mon Sep 17 00:00:00 2001 From: Clemens Wolff Date: Fri, 17 May 2019 00:54:10 -0400 Subject: [PATCH 5/6] Publish test results --- .gitignore | 1 + app/app/settings.py | 2 ++ azure-pipelines.yaml | 14 +++++++++++++- requirements.txt | 1 + 4 files changed, 17 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 9077a274..1a9bd283 100644 --- a/.gitignore +++ b/.gitignore @@ -74,6 +74,7 @@ nosetests.xml coverage.xml *.cover .hypothesis/ +junitxml/ # Translations *.mo diff --git a/app/app/settings.py b/app/app/settings.py index 2bd86126..7963b5b6 100644 --- a/app/app/settings.py +++ b/app/app/settings.py @@ -185,6 +185,8 @@ USE_L10N = True USE_TZ = True +TEST_RUNNER = 'xmlrunner.extra.djangotestrunner.XMLTestRunner' +TEST_OUTPUT_DIR = path.join(BASE_DIR, 'junitxml') # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/2.0/howto/static-files/ diff --git a/azure-pipelines.yaml b/azure-pipelines.yaml index 42b0ca7c..8cda230a 100644 --- a/azure-pipelines.yaml +++ b/azure-pipelines.yaml @@ -5,9 +5,21 @@ pool: vmImage: 'ubuntu-latest' steps: -- script: docker build --target=builder . +- script: docker build --tag=doccano --target=builder . displayName: 'Run tests' +- script: docker run doccano tar Ccf /doccano/app - junitxml | tar Cxf "$(Build.ArtifactStagingDirectory)" - + displayName: 'Export test results' + +- task: PublishTestResults@2 + inputs: + testResultsFormat: 'JUnit' + testResultsFiles: 'TEST-*.xml' + searchFolder: '$(Build.ArtifactStagingDirectory)/junitxml' + mergeTestResults: true + testRunTitle: 'server.tests' + displayName: 'Publish test results' + - script: DOCKER_PASSWORD="$(docker_password)" tools/cd.sh "azdo-$(Build.BuildId)" displayName: 'Push docker image' condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master'), ne(variables['docker_password'], '')) diff --git a/requirements.txt b/requirements.txt index 2c16a05c..177033a4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -27,4 +27,5 @@ social-auth-app-django==3.1.0 social-auth-core[azuread]==3.0.0 text-unidecode==1.2 tornado==5.0.2 +unittest-xml-reporting==2.5.1 whitenoise[brotli]==4.1.2 From 236886a6bd3267e25b4e81ae448634880289c048 Mon Sep 17 00:00:00 2001 From: Clemens Wolff Date: Fri, 17 May 2019 02:10:12 -0400 Subject: [PATCH 6/6] Add documentation for pipeline variables --- azure-pipelines.yaml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/azure-pipelines.yaml b/azure-pipelines.yaml index 8cda230a..12da7d86 100644 --- a/azure-pipelines.yaml +++ b/azure-pipelines.yaml @@ -20,10 +20,29 @@ steps: testRunTitle: 'server.tests' displayName: 'Publish test results' +# To publish docker images to a container registry, set the following pipeline variables: +# - docker_password +# - docker_username +# - docker_registry (optional, set this to publish to a registry other than Docker Hub) +# - script: DOCKER_PASSWORD="$(docker_password)" tools/cd.sh "azdo-$(Build.BuildId)" displayName: 'Push docker image' condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master'), ne(variables['docker_password'], '')) +# To automatically deploy to Azure, create a service principal and set the following pipeline variables: +# - auth_username (app ID) +# - auth_tenant (tenant ID +# - auth_password (secret) +# +# Additionally, to configure the deployment, set the following pipeline variables: +# - doccano_admin_username +# - doccano_admin_password +# - doccano_admin_contact_email +# - doccano_app_name (globally unique name for the app) +# - doccano_secret_key (pass-through secret for Django) +# - doccano_resource_group (group for all resources, will be created if it doesn't yet exist) +# - doccano_location (name of the Azure region to which to deploy all resources) +# - script: | az login --service-principal --password "$(auth_password)" --tenant "$(auth_tenant)" --username "$(auth_username)"