From e2518c7a8dd2392a06863354f4af117c386b82f9 Mon Sep 17 00:00:00 2001 From: Nick Date: Sun, 7 Apr 2019 17:00:42 -0400 Subject: [PATCH] feat: storage disk - actions + daily backup --- package.json | 1 + server/models/storage.js | 10 ++++ server/modules/storage/disk/definition.yml | 8 +++ server/modules/storage/disk/storage.js | 57 +++++++++++++++++++++- yarn.lock | 5 ++ 5 files changed, 79 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 2420d98d..b991af2e 100644 --- a/package.json +++ b/package.json @@ -157,6 +157,7 @@ "sqlite3": "4.0.6", "striptags": "3.1.1", "subscriptions-transport-ws": "0.9.15", + "tar-fs": "2.0.0", "twemoji": "11.3.0", "uslug": "1.0.4", "uuid": "3.3.2", diff --git a/server/models/storage.js b/server/models/storage.js index 91cfd91e..31e876bb 100644 --- a/server/models/storage.js +++ b/server/models/storage.js @@ -142,6 +142,16 @@ module.exports = class Storage extends Model { repeat: true }, target.key) } + + // -> Set internal recurring sync job + if (targetDef.intervalSchedule && targetDef.intervalSchedule !== `P0D`) { + WIKI.scheduler.registerJob({ + name: `sync-storage`, + immediate: false, + schedule: target.intervalSchedule, + repeat: true + }, target.key) + } } catch (err) { // -> Save initialization error await WIKI.models.storage.query().patch({ diff --git a/server/modules/storage/disk/definition.yml b/server/modules/storage/disk/definition.yml index 09466115..1b326fdf 100644 --- a/server/modules/storage/disk/definition.yml +++ b/server/modules/storage/disk/definition.yml @@ -9,6 +9,7 @@ supportedModes: - push defaultMode: push schedule: false +internalSchedule: P1D props: path: type: String @@ -21,3 +22,10 @@ props: title: Create Daily Backups hint: A tar.gz archive containing all content will be created daily in subfolder named _daily. Archives are kept for a month. order: 2 +actions: + - handler: dump + label: Dump all content to disk + hint: Output all content from the DB to the local disk. If you enabled this module after content was created or you temporarily disabled this module, you'll want to execute this action to add the missing files. + - handler: backup + label: Create Backup + hint: Will create a manual backup archive at this point in time, in a subfolder named _manual, from the contents currently on disk. diff --git a/server/modules/storage/disk/storage.js b/server/modules/storage/disk/storage.js index 9f983019..1a2e9548 100644 --- a/server/modules/storage/disk/storage.js +++ b/server/modules/storage/disk/storage.js @@ -1,5 +1,15 @@ const fs = require('fs-extra') +const _ = require('lodash') const path = require('path') +const tar = require('tar-fs') +const zlib = require('zlib') +const stream = require('stream') +const Promise = require('bluebird') +const pipeline = Promise.promisify(stream.pipeline) +const pageHelper = require('../../../helpers/page.js') +const moment = require('moment') + +/* global WIKI */ /** * Get file extension based on content type @@ -27,8 +37,25 @@ module.exports = { await fs.ensureDir(this.config.path) WIKI.logger.info('(STORAGE/DISK) Initialization completed.') }, - async sync() { - // not used + async sync({ manual } = { manual: false }) { + if (this.config.createDailyBackups || manual) { + const dirPath = path.join(this.config.path, manual ? '_manual' : '_daily') + await fs.ensureDir(dirPath) + + const dateFilename = moment().format(manual ? 'YYYYMMDD-HHmmss' : 'DD') + + WIKI.logger.info(`(STORAGE/DISK) Creating backup archive...`) + await pipeline( + tar.pack(this.config.path, { + ignore: (filePath) => { + return filePath.indexOf('_daily') >= 0 || filePath.indexOf('_manual') >= 0 + } + }), + zlib.createGzip(), + fs.createWriteStream(path.join(dirPath, `wiki-${dateFilename}.tar.gz`)) + ) + WIKI.logger.info('(STORAGE/DISK) Backup archive created successfully.') + } }, async created(page) { WIKI.logger.info(`(STORAGE/DISK) Creating file ${page.path}...`) @@ -50,5 +77,31 @@ module.exports = { const sourceFilePath = path.join(this.config.path, `${page.sourcePath}.${getFileExtension(page.contentType)}`) const destinationFilePath = path.join(this.config.path, `${page.destinationPath}.${getFileExtension(page.contentType)}`) await fs.move(sourceFilePath, destinationFilePath, { overwrite: true }) + }, + + /** + * HANDLERS + */ + async dump() { + WIKI.logger.info(`(STORAGE/DISK) Dumping all content to disk...`) + await pipeline( + WIKI.models.knex.column('path', 'localeCode', 'title', 'description', 'contentType', 'content', 'isPublished', 'updatedAt').select().from('pages').where({ + isPrivate: false + }).stream(), + new stream.Transform({ + objectMode: true, + transform: async (page, enc, cb) => { + const fileName = `${page.path}.${getFileExtension(page.contentType)}` + WIKI.logger.info(`(STORAGE/DISK) Dumping ${fileName}...`) + const filePath = path.join(this.config.path, fileName) + await fs.outputFile(filePath, pageHelper.injectPageMetadata(page), 'utf8') + cb() + } + }) + ) + WIKI.logger.info('(STORAGE/DISK) All content was dumped to disk successfully.') + }, + async backup() { + return this.sync({ manual: true }) } } diff --git a/yarn.lock b/yarn.lock index 4d2a1635..737320e2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -12438,6 +12438,11 @@ tapable@^1.0.0, tapable@^1.1.0: resolved "https://registry.yarnpkg.com/tapable/-/tapable-1.1.1.tgz#4d297923c5a72a42360de2ab52dadfaaec00018e" integrity sha512-9I2ydhj8Z9veORCw5PRm4u9uebCn0mcCa6scWoNcbZ6dAtoo2618u9UUzxgmsCOreJpqDDuv61LvwofW7hLcBA== +tar-js@0.3.0: + version "0.3.0" + resolved "https://registry.yarnpkg.com/tar-js/-/tar-js-0.3.0.tgz#6949aabfb0ba18bb1562ae51a439fd0f30183a17" + integrity sha1-aUmqv7C6GLsVYq5RpDn9DzAYOhc= + tar@^2.0.0: version "2.2.1" resolved "https://registry.yarnpkg.com/tar/-/tar-2.2.1.tgz#8e4d2a256c0e2185c6b18ad694aec968b83cb1d1"