From 59feacd8462d71de4f5eabfdd4f46096c1af9d5c Mon Sep 17 00:00:00 2001 From: NGPixel Date: Wed, 31 Aug 2016 22:45:28 -0400 Subject: [PATCH] Added Background Agent (git resync + cache purge) --- README.md | 4 ++- agent.js | 91 +++++++++++++++++++++++++++++++++++++++++++++++ models/entries.js | 43 +++++++++++++++++++++- models/git.js | 4 ++- package.json | 1 + server.js | 17 ++++++++- 6 files changed, 156 insertions(+), 4 deletions(-) create mode 100644 agent.js diff --git a/README.md b/README.md index c1309db4..507abd5a 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ ##### Milestones - [ ] Assets Management - [ ] Authentication +- [x] Background Agent (git sync, cache purge, etc.) - [x] Caching - [x] Create Entry - [x] Edit Entry @@ -27,4 +28,5 @@ - [x] Navigation - [x] Parsing / Tree / Metadata - [ ] Search -- [x] UI \ No newline at end of file +- [x] UI +- [x] View Entry Source \ No newline at end of file diff --git a/agent.js b/agent.js new file mode 100644 index 00000000..067d6d26 --- /dev/null +++ b/agent.js @@ -0,0 +1,91 @@ +// =========================================== +// REQUARKS WIKI - Background Agent +// 1.0.0 +// Licensed under AGPLv3 +// =========================================== + +global.ROOTPATH = __dirname; + +// ---------------------------------------- +// Load modules +// ---------------------------------------- + +global.winston = require('winston'); +winston.info('[AGENT] Requarks Wiki BgAgent is initializing...'); + +var appconfig = require('./models/config')('./config.yml'); + +global.git = require('./models/git').init(appconfig, true); +global.entries = require('./models/entries').init(appconfig); +global.mark = require('./models/markdown'); + +var _ = require('lodash'); +var moment = require('moment'); +var Promise = require('bluebird'); +var cron = require('cron').CronJob; + +// ---------------------------------------- +// Start Cron +// ---------------------------------------- + +var jobIsBusy = false; +var job = new cron({ + cronTime: '0 */5 * * * *', + onTick: () => { + + // Make sure we don't start two concurrent jobs + + if(jobIsBusy) { + winston.warn('[AGENT] Previous job has not completed gracefully or is still running! Skipping for now. (This is not normal, you should investigate)'); + return; + } + jobIsBusy = true; + + // Prepare async job collector + + let jobs = []; + + // ---------------------------------------- + // Compile Jobs + // ---------------------------------------- + + //-> Resync with Git remote + + jobs.push(git.resync().then(() => { + + //-> Purge outdated cache + + return entries.purgeStaleCache(); + + })); + + // ---------------------------------------- + // Run + // ---------------------------------------- + + Promise.all(jobs).then(() => { + winston.info('[AGENT] All jobs completed successfully! Going to sleep for now... [' + moment().toISOString() + ']'); + }).catch((err) => { + winston.error('[AGENT] One or more jobs have failed [' + moment().toISOString() + ']: ', err); + }).finally(() => { + jobIsBusy = false; + }); + + }, + start: true, + timeZone: 'UTC' +}); + +// ---------------------------------------- +// Shutdown gracefully +// ---------------------------------------- + +process.on('disconnect', () => { + winston.warn('[AGENT] Lost connection to main server. Exiting... [' + moment().toISOString() + ']'); + job.stop(); + process.exit(); +}); + +process.on('exit', () => { + job.stop(); +}); \ No newline at end of file diff --git a/models/entries.js b/models/entries.js index 8b5ecf97..31325925 100644 --- a/models/entries.js +++ b/models/entries.js @@ -6,7 +6,8 @@ var Promise = require('bluebird'), _ = require('lodash'), farmhash = require('farmhash'), BSONModule = require('bson'), - BSON = new BSONModule.BSONPure.BSON(); + BSON = new BSONModule.BSONPure.BSON(), + moment = require('moment'); /** * Entries Model @@ -259,6 +260,17 @@ module.exports = { return path.join(this._cachePath, farmhash.fingerprint32(entryPath) + '.bson'); }, + /** + * Gets the entry path from full path. + * + * @param {String} fullPath The full path + * @return {String} The entry path + */ + getEntryPathFromFullPath(fullPath) { + let absRepoPath = path.resolve(ROOTPATH, this._repoPath); + return _.chain(fullPath).replace(absRepoPath, '').replace('.md', '').replace(new RegExp('\\\\', 'g'),'/').value(); + }, + /** * Update an existing document * @@ -344,6 +356,35 @@ module.exports = { return _.replace(contents, new RegExp('{TITLE}', 'g'), formattedTitle); }); + }, + + purgeStaleCache() { + + let self = this; + + let cacheJobs = []; + + fs.walk(self._repoPath) + .on('data', function (item) { + if(path.extname(item.path) === '.md') { + + let entryPath = self.parsePath(self.getEntryPathFromFullPath(item.path)); + let cachePath = self.getCachePath(entryPath); + + cacheJobs.push(fs.statAsync(cachePath).then((st) => { + if(moment(st.mtime).isBefore(item.stats.mtime)) { + return fs.unlinkAsync(cachePath); + } else { + return true; + } + }).catch((err) => { + return (err.code !== 'EEXIST') ? err : true; + })); + } + }); + + return Promise.all(cacheJobs); + } }; \ No newline at end of file diff --git a/models/git.js b/models/git.js index cdb2f43d..ac54a923 100644 --- a/models/git.js +++ b/models/git.js @@ -37,10 +37,12 @@ module.exports = { * @param {Object} appconfig The application config * @return {Object} Git model instance */ - init(appconfig) { + init(appconfig, sync) { let self = this; + self._repo.sync = sync; + //-> Build repository path if(_.isEmpty(appconfig.datadir.repo)) { diff --git a/package.json b/package.json index 20c003ac..b6279658 100644 --- a/package.json +++ b/package.json @@ -42,6 +42,7 @@ "connect-loki": "^1.0.6", "connect-redis": "^3.1.0", "cookie-parser": "^1.4.3", + "cron": "^1.1.0", "express": "^4.14.0", "express-brute": "^1.0.0", "express-brute-loki": "^1.0.0", diff --git a/server.js b/server.js index b18c7df4..d6c5e554 100644 --- a/server.js +++ b/server.js @@ -17,7 +17,7 @@ var appconfig = require('./models/config')('./config.yml'); let lcdata = require('./models/localdata'); global.db = require('./models/db')(appconfig); -global.git = require('./models/git').init(appconfig); +global.git = require('./models/git').init(appconfig, false); global.entries = require('./models/entries').init(appconfig); global.mark = require('./models/markdown'); @@ -192,4 +192,19 @@ server.on('error', (error) => { server.on('listening', () => { winston.info('[SERVER] HTTP server started successfully! [RUNNING]'); +}); + +// ---------------------------------------- +// Start Background Agent +// ---------------------------------------- + +var fork = require('child_process').fork; +var bgAgent = fork('agent.js'); + +bgAgent.on('message', (m) => { + +}); + +process.on('exit', (code) => { + bgAgent.disconnect(); }); \ No newline at end of file