From 5202eadebb2085aa608c380d2807223072a0e596 Mon Sep 17 00:00:00 2001 From: Andrew Sim Date: Mon, 9 Sep 2019 02:11:25 +0100 Subject: [PATCH] feat: AWS S3 + Digitalocean Spaces storage modules (#1015) * Provide basic implementation of AWS S3 storage module * Abstract S3 Compatible Storage Module logic * Refactor `getFileExtension()` into the `page` object * Add implementation for Digitalocean storage module * Remove accidental `async`/`await` in S3 Storage Module * Remove argument from the call to `page.getFileExtension()` https://github.com/Requarks/wiki/pull/1015#discussion_r321990073 --- server/models/pages.js | 14 ++++ .../storage/digitalocean/definition.yml | 24 +++---- .../modules/storage/digitalocean/storage.js | 24 +------ server/modules/storage/disk/storage.js | 26 ++------ server/modules/storage/git/storage.js | 26 ++------ server/modules/storage/s3/common.js | 64 +++++++++++++++++++ server/modules/storage/s3/definition.yml | 31 +++++++-- server/modules/storage/s3/storage.js | 24 +------ 8 files changed, 132 insertions(+), 101 deletions(-) create mode 100644 server/modules/storage/s3/common.js diff --git a/server/models/pages.js b/server/models/pages.js index 45a162cd..27cd8bd6 100644 --- a/server/models/pages.js +++ b/server/models/pages.js @@ -147,6 +147,20 @@ module.exports = class Page extends Model { return pageHelper.injectPageMetadata(this) } + /** + * Get the page's file extension based on content type + */ + getFileExtension() { + switch (this.contentType) { + case 'markdown': + return 'md' + case 'html': + return 'html' + default: + return 'txt' + } + } + /** * Parse injected page metadata from raw content * diff --git a/server/modules/storage/digitalocean/definition.yml b/server/modules/storage/digitalocean/definition.yml index 7d77d5c5..d43c9d3e 100644 --- a/server/modules/storage/digitalocean/definition.yml +++ b/server/modules/storage/digitalocean/definition.yml @@ -1,28 +1,28 @@ key: digitalocean title: DigitalOcean Spaces description: DigitalOcean provides developers and businesses a reliable, easy-to-use cloud computing platform of virtual servers (Droplets), object storage (Spaces) and more. -author: requarks.io +author: andrewsim logo: https://static.requarks.io/logo/digitalocean.svg website: https://www.digitalocean.com/products/spaces/ -isAvailable: false +isAvailable: true supportedModes: - push defaultMode: push schedule: false props: - region: + endpoint: type: String - title: Region - hint: The DigitalOcean datacenter region where the Space will be created. - default: nyc3 + title: Endpoint + hint: The DigitalOcean spaces endpoint that has the form ${REGION}.digitaloceanspaces.com + default: nyc3.digitaloceanspaces.com enum: - - ams3 - - fra1 - - nyc3 - - sfo2 - - sgp1 + - ams3.digitaloceanspaces.com + - fra1.digitaloceanspaces.com + - nyc3.digitaloceanspaces.com + - sfo2.digitaloceanspaces.com + - sgp1.digitaloceanspaces.com order: 1 - spaceId: + bucket: type: String title: Space Unique Name hint: The unique space name to create (e.g. wiki-johndoe) diff --git a/server/modules/storage/digitalocean/storage.js b/server/modules/storage/digitalocean/storage.js index ab25ce97..672fb6bc 100644 --- a/server/modules/storage/digitalocean/storage.js +++ b/server/modules/storage/digitalocean/storage.js @@ -1,23 +1,3 @@ -module.exports = { - async activated() { +const S3CompatibleStorage = require('../s3/common') - }, - async deactivated() { - - }, - async init() { - - }, - async created() { - - }, - async updated() { - - }, - async deleted() { - - }, - async renamed() { - - } -} +module.exports = new S3CompatibleStorage('Digitalocean') diff --git a/server/modules/storage/disk/storage.js b/server/modules/storage/disk/storage.js index 914d6370..b6ab75ea 100644 --- a/server/modules/storage/disk/storage.js +++ b/server/modules/storage/disk/storage.js @@ -10,20 +10,6 @@ const moment = require('moment') /* global WIKI */ -/** - * Get file extension based on content type - */ -const getFileExtension = (contentType) => { - switch (contentType) { - case 'markdown': - return 'md' - case 'html': - return 'html' - default: - return 'txt' - } -} - module.exports = { async activated() { // not used @@ -58,7 +44,7 @@ module.exports = { }, async created(page) { WIKI.logger.info(`(STORAGE/DISK) Creating file ${page.path}...`) - let fileName = `${page.path}.${getFileExtension(page.contentType)}` + let fileName = `${page.path}.${page.getFileExtension()}` if (WIKI.config.lang.code !== page.localeCode) { fileName = `${page.localeCode}/${fileName}` } @@ -67,7 +53,7 @@ module.exports = { }, async updated(page) { WIKI.logger.info(`(STORAGE/DISK) Updating file ${page.path}...`) - let fileName = `${page.path}.${getFileExtension(page.contentType)}` + let fileName = `${page.path}.${page.getFileExtension()}` if (WIKI.config.lang.code !== page.localeCode) { fileName = `${page.localeCode}/${fileName}` } @@ -76,7 +62,7 @@ module.exports = { }, async deleted(page) { WIKI.logger.info(`(STORAGE/DISK) Deleting file ${page.path}...`) - let fileName = `${page.path}.${getFileExtension(page.contentType)}` + let fileName = `${page.path}.${page.getFileExtension()}` if (WIKI.config.lang.code !== page.localeCode) { fileName = `${page.localeCode}/${fileName}` } @@ -85,8 +71,8 @@ module.exports = { }, async renamed(page) { WIKI.logger.info(`(STORAGE/DISK) Renaming file ${page.sourcePath} to ${page.destinationPath}...`) - let sourceFilePath = `${page.sourcePath}.${getFileExtension(page.contentType)}` - let destinationFilePath = `${page.destinationPath}.${getFileExtension(page.contentType)}` + let sourceFilePath = `${page.sourcePath}.${page.getFileExtension()}` + let destinationFilePath = `${page.destinationPath}.${page.getFileExtension()}` if (WIKI.config.lang.code !== page.localeCode) { sourceFilePath = `${page.localeCode}/${sourceFilePath}` @@ -107,7 +93,7 @@ module.exports = { new stream.Transform({ objectMode: true, transform: async (page, enc, cb) => { - let fileName = `${page.path}.${getFileExtension(page.contentType)}` + let fileName = `${page.path}.${page.getFileExtension()}` if (WIKI.config.lang.code !== page.localeCode) { fileName = `${page.localeCode}/${fileName}` } diff --git a/server/modules/storage/git/storage.js b/server/modules/storage/git/storage.js index d00f44b9..9e3d44a3 100644 --- a/server/modules/storage/git/storage.js +++ b/server/modules/storage/git/storage.js @@ -12,20 +12,6 @@ const localeFolderRegex = /^([a-z]{2}(?:-[a-z]{2})?\/)?(.*)/i /* global WIKI */ -/** - * Get file extension based on content type - */ -const getFileExtension = (contentType) => { - switch (contentType) { - case 'markdown': - return 'md' - case 'html': - return 'html' - default: - return 'txt' - } -} - const getContenType = (filePath) => { const ext = _.last(filePath.split('.')) switch (ext) { @@ -256,7 +242,7 @@ module.exports = { */ async created(page) { WIKI.logger.info(`(STORAGE/GIT) Committing new file ${page.path}...`) - let fileName = `${page.path}.${getFileExtension(page.contentType)}` + let fileName = `${page.path}.${page.getFileExtension()}` if (WIKI.config.lang.namespacing && WIKI.config.lang.code !== page.localeCode) { fileName = `${page.localeCode}/${fileName}` } @@ -275,7 +261,7 @@ module.exports = { */ async updated(page) { WIKI.logger.info(`(STORAGE/GIT) Committing updated file ${page.path}...`) - let fileName = `${page.path}.${getFileExtension(page.contentType)}` + let fileName = `${page.path}.${page.getFileExtension()}` if (WIKI.config.lang.namespacing && WIKI.config.lang.code !== page.localeCode) { fileName = `${page.localeCode}/${fileName}` } @@ -294,7 +280,7 @@ module.exports = { */ async deleted(page) { WIKI.logger.info(`(STORAGE/GIT) Committing removed file ${page.path}...`) - let fileName = `${page.path}.${getFileExtension(page.contentType)}` + let fileName = `${page.path}.${page.getFileExtension()}` if (WIKI.config.lang.namespacing && WIKI.config.lang.code !== page.localeCode) { fileName = `${page.localeCode}/${fileName}` } @@ -311,8 +297,8 @@ module.exports = { */ async renamed(page) { WIKI.logger.info(`(STORAGE/GIT) Committing file move from ${page.sourcePath} to ${page.destinationPath}...`) - let sourceFilePath = `${page.sourcePath}.${getFileExtension(page.contentType)}` - let destinationFilePath = `${page.destinationPath}.${getFileExtension(page.contentType)}` + let sourceFilePath = `${page.sourcePath}.${page.getFileExtension()}` + let destinationFilePath = `${page.destinationPath}.${page.getFileExtension()}` if (WIKI.config.lang.namespacing && WIKI.config.lang.code !== page.localeCode) { sourceFilePath = `${page.localeCode}/${sourceFilePath}` @@ -363,7 +349,7 @@ module.exports = { new stream.Transform({ objectMode: true, transform: async (page, enc, cb) => { - let fileName = `${page.path}.${getFileExtension(page.contentType)}` + let fileName = `${page.path}.${page.getFileExtension()}` if (WIKI.config.lang.namespacing && WIKI.config.lang.code !== page.localeCode) { fileName = `${page.localeCode}/${fileName}` } diff --git a/server/modules/storage/s3/common.js b/server/modules/storage/s3/common.js new file mode 100644 index 00000000..89deccb7 --- /dev/null +++ b/server/modules/storage/s3/common.js @@ -0,0 +1,64 @@ +const S3 = require('aws-sdk/clients/s3') + +/* global WIKI */ + +/** + * Deduce the file path given the `page` object and the object's key to the page's path. + */ +const getFilePath = (page, pathKey) => { + const fileName = `${page[pathKey]}.${page.getFileExtension()}` + const withLocaleCode = WIKI.config.lang.namespacing && WIKI.config.lang.code !== page.localeCode + return withLocaleCode ? `${page.localeCode}/${fileName}` : fileName +} + +/** + * Can be used with S3 compatible storage. + */ +module.exports = class S3CompatibleStorage { + constructor(storageName) { + this.storageName = storageName + } + async activated() { + // not used + } + async deactivated() { + // not used + } + async init() { + WIKI.logger.info(`(STORAGE/${this.storageName}) Initializing...`) + const { accessKeyId, secretAccessKey, region, bucket, endpoint } = this.config + this.s3 = new S3({ + accessKeyId, + secretAccessKey, + region, + endpoint, + params: { Bucket: bucket }, + apiVersions: '2006-03-01' + }) + // determine if a bucket exists and you have permission to access it + await this.s3.headBucket().promise() + WIKI.logger.info(`(STORAGE/${this.storageName}) Initialization completed.`) + } + async created(page) { + WIKI.logger.info(`(STORAGE/${this.storageName}) Creating file ${page.path}...`) + const filePath = getFilePath(page, 'path') + await this.s3.putObject({ Key: filePath, Body: page.injectMetadata() }).promise() + } + async updated(page) { + WIKI.logger.info(`(STORAGE/${this.storageName}) Updating file ${page.path}...`) + const filePath = getFilePath(page, 'path') + await this.s3.putObject({ Key: filePath, Body: page.injectMetadata() }).promise() + } + async deleted(page) { + WIKI.logger.info(`(STORAGE/${this.storageName}) Deleting file ${page.path}...`) + const filePath = getFilePath(page, 'path') + await this.s3.deleteObject({ Key: filePath }).promise() + } + async renamed(page) { + WIKI.logger.info(`(STORAGE/${this.storageName}) Renaming file ${page.sourcePath} to ${page.destinationPath}...`) + const sourceFilePath = getFilePath(page, 'sourcePath') + const destinationFilePath = getFilePath(page, 'destinationPath') + await this.s3.copyObject({ CopySource: sourceFilePath, Key: destinationFilePath }).promise() + await this.s3.deleteObject({ Key: sourceFilePath }).promise() + } +} diff --git a/server/modules/storage/s3/definition.yml b/server/modules/storage/s3/definition.yml index e9681231..f18adea6 100644 --- a/server/modules/storage/s3/definition.yml +++ b/server/modules/storage/s3/definition.yml @@ -1,11 +1,32 @@ key: s3 title: Amazon S3 description: Amazon S3 is a cloud computing web service offered by Amazon Web Services which provides object storage. -author: requarks.io +author: andrewsim logo: https://static.requarks.io/logo/aws-s3.svg website: https://aws.amazon.com/s3/ +isAvailable: true +supportedModes: + - push +defaultMode: push +schedule: false props: - accessKeyId: String - accessSecret: String - region: String - bucket: String + region: + type: String + title: Region + hint: The AWS datacenter region where the bucket will be created. + order: 1 + bucket: + type: String + title: Unique bucket name + hint: The unique bucket name to create (e.g. wiki-johndoe). + order: 2 + accessKeyId: + type: String + title: Access Key ID + hint: The Access Key. + order: 3 + secretAccessKey: + type: String + title: Secret Access Key + hint: The Secret Access Key for the Access Key ID you created above. + order: 4 diff --git a/server/modules/storage/s3/storage.js b/server/modules/storage/s3/storage.js index ab25ce97..ea428b79 100644 --- a/server/modules/storage/s3/storage.js +++ b/server/modules/storage/s3/storage.js @@ -1,23 +1,3 @@ -module.exports = { - async activated() { +const S3CompatibleStorage = require('./common') - }, - async deactivated() { - - }, - async init() { - - }, - async created() { - - }, - async updated() { - - }, - async deleted() { - - }, - async renamed() { - - } -} +module.exports = new S3CompatibleStorage('S3')