richard
/
wiki
mirror of https://github.com/Requarks/wiki.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Uploads model + watcher

pull/2/head
NGPixel 8 years ago
parent
819d4ad346
commit
99a07d342c
12 changed files with 367 additions and 119 deletions
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 84
      agent.js
  2. 2
      assets/js/app.js
  3. 67
      client/js/components/editor-image.js
  4. 10
      controllers/pages.js
  5. 15
      controllers/uploads.js
  6. 2
      models/entries.js
  7. 17
      models/git.js
  8. 64
      models/localdata.js
  9. 176
      models/uploads.js
  10. 1
      package.json
  11. 32
      views/error-notexist.pug
  12. 16
      ws-server.js

84
agent.js
View File

@ -31,7 +31,8 @@ global.WSInternalKey = process.argv[2];
winston.info('[AGENT] Background Agent is initializing...');
var appconfig = require('./models/config')('./config.yml');
let lcdata = require('./models/localdata').init(appconfig, 'agent');
global.lcdata = require('./models/localdata').init(appconfig, 'agent');
var upl = require('./models/uploads').init(appconfig);
global.git = require('./models/git').init(appconfig);
global.entries = require('./models/entries').init(appconfig);
@ -43,9 +44,6 @@ var Promise = require('bluebird');
var fs = Promise.promisifyAll(require("fs-extra"));
var path = require('path');
var cron = require('cron').CronJob;
var readChunk = require('read-chunk');
var fileType = require('file-type');
var farmhash = require('farmhash');
global.ws = require('socket.io-client')('http://localhost:' + appconfig.wsPort, { reconnectionAttempts: 10 });
@ -56,6 +54,8 @@ const mimeImgTypes = ['image/png', 'image/jpg']
// ----------------------------------------
var jobIsBusy = false;
var jobUplWatchStarted = false;
var job = new cron({
cronTime: '0 */5 * * * *',
onTick: () => {
@ -168,71 +168,11 @@ var job = new cron({
let fldPath = path.join(uploadsPath, fldName);
return fs.readdirAsync(fldPath).then((fList) => {
return Promise.map(fList, (f) => {
let fPath = path.join(fldPath, f);
let fPathObj = path.parse(fPath);
let fUid = farmhash.fingerprint32(fldName + '/' + f);
return fs.statAsync(fPath)
.then((s) => {
if(!s.isFile()) { return false; }
// Get MIME info
let mimeInfo = fileType(readChunk.sync(fPath, 0, 262));
// Images
if(s.size < 3145728) { // ignore files larger than 3MB
if(_.includes(['image/png', 'image/jpeg', 'image/gif', 'image/webp'], mimeInfo.mime)) {
return lcdata.getImageMetadata(fPath).then((mData) => {
let cacheThumbnailPath = path.parse(path.join(dataPath, 'thumbs', fUid + '.png'));
let cacheThumbnailPathStr = path.format(cacheThumbnailPath);
mData = _.pick(mData, ['format', 'width', 'height', 'density', 'hasAlpha', 'orientation']);
mData.uid = fUid;
mData.category = 'image';
mData.mime = mimeInfo.mime;
mData.folder = fldName;
mData.filename = f;
mData.basename = fPathObj.name;
mData.filesize = s.size;
mData.uploadedOn = moment().utc();
allFiles.push(mData);
// Generate thumbnail
return fs.statAsync(cacheThumbnailPathStr).then((st) => {
return st.isFile();
}).catch((err) => {
return false;
}).then((thumbExists) => {
return (thumbExists) ? true : fs.ensureDirAsync(cacheThumbnailPath.dir).then(() => {
return lcdata.generateThumbnail(fPath, cacheThumbnailPathStr);
});
});
})
}
}
// Other Files
allFiles.push({
uid: fUid,
category: 'file',
mime: mimeInfo.mime,
folder: fldName,
filename: f,
basename: fPathObj.name,
filesize: s.size,
uploadedOn: moment().utc()
});
});
return upl.processFile(fldName, f).then((mData) => {
if(mData) {
allFiles.push(mData);
}
});
}, {concurrency: 3});
});
}, {concurrency: 1}).finally(() => {
@ -255,6 +195,12 @@ var job = new cron({
Promise.all(jobs).then(() => {
winston.info('[AGENT] All jobs completed successfully! Going to sleep for now.');
if(!jobUplWatchStarted) {
jobUplWatchStarted = true;
upl.watch();
}
}).catch((err) => {
winston.error('[AGENT] One or more jobs have failed: ', err);
}).finally(() => {

2
assets/js/app.js
File diff suppressed because it is too large
View File

67
client/js/components/editor-image.js
View File

@ -13,7 +13,9 @@ let vueImage = new Vue({
currentFolder: '',
currentImage: '',
currentAlign: 'left',
images: []
images: [],
uploadSucceeded: false,
postUploadChecks: 0
},
methods: {
open: () => {
@ -126,13 +128,17 @@ let vueImage = new Vue({
*
* @return {Void} Void
*/
loadImages: () => {
vueImage.isLoading = true;
vueImage.isLoadingText = 'Fetching images...';
loadImages: (silent) => {
if(!silent) {
vueImage.isLoading = true;
vueImage.isLoadingText = 'Fetching images...';
}
Vue.nextTick(() => {
socket.emit('uploadsGetImages', { folder: vueImage.currentFolder }, (data) => {
vueImage.images = data;
vueImage.isLoading = false;
if(!silent) {
vueImage.isLoading = false;
}
vueImage.attachContextMenus();
});
});
@ -209,6 +215,31 @@ let vueImage = new Vue({
}
}
});
},
waitUploadComplete: () => {
vueImage.postUploadChecks++;
vueImage.isLoadingText = 'Processing uploads...';
let currentUplAmount = vueImage.images.length;
vueImage.loadImages(true);
Vue.nextTick(() => {
_.delay(() => {
if(currentUplAmount !== vueImage.images.length) {
vueImage.postUploadChecks = 0;
vueImage.isLoading = false;
} else if(vueImage.postUploadChecks > 5) {
vueImage.postUploadChecks = 0;
vueImage.isLoading = false;
alerts.pushError('Unable to fetch new uploads', 'Try again later');
} else {
vueImage.waitUploadComplete();
}
}, 2000);
});
}
}
@ -228,7 +259,8 @@ $('#btn-editor-uploadimage input').on('change', (ev) => {
allowedTypes: ['image/png', 'image/jpeg', 'image/gif', 'image/webp'],
maxFileSize: 3145728, // max 3 MB
init: () => {
init: (totalUploads) => {
vueImage.uploadSucceeded = false;
vueImage.isLoading = true;
vueImage.isLoadingText = 'Preparing to upload...';
},
@ -240,18 +272,37 @@ $('#btn-editor-uploadimage input').on('change', (ev) => {
success: (data) => {
if(data.ok) {
let failedUpls = _.filter(data.results, ['ok', false]);
if(failedUpls.length) {
_.forEach(failedUpls, (u) => {
alerts.pushError('Upload error', u.msg);
});
if(failedUpls.length < data.results.length) {
alerts.push({
title: 'Some uploads succeeded',
message: 'Files that are not mentionned in the errors above were uploaded successfully.'
});
vueImage.uploadSucceeded = true;
}
} else {
vueImage.uploadSucceeded = true;
}
} else {
alerts.pushError('Upload error', data.msg);
}
},
error: function(error) {
vueImage.isLoading = false;
alerts.pushError(error.message, this.upload.file.name);
},
finish: () => {
vueImage.isLoading = false;
if(vueImage.uploadSucceeded) {
vueImage.waitUploadComplete();
} else {
vueImage.isLoading = false;
}
}
});

10
controllers/pages.js
View File

@ -160,11 +160,15 @@ router.get('/*', (req, res, next) => {
if(pageData) {
return res.render('pages/view', { pageData });
} else {
res.render('error', {
message: err.message,
error: {}
res.render('error-notexist', {
newpath: safePath
});
}
}).error((err) => {
res.render('error-notexist', {
message: err.message,
newpath: safePath
});
}).catch((err) => {
res.render('error', {
message: err.message,

15
controllers/uploads.js
View File

@ -72,13 +72,24 @@ router.post('/img', lcdata.uploadImgHandler, (req, res, next) => {
}).then(() => {
return {
ok: true,
filename: destFilename,
filesize: f.size
};
});
}).reflect();
}, {concurrency: 3}).then((results) => {
res.json({ ok: true, results });
let uplResults = _.map(results, (r) => {
if(r.isFulfilled()) {
return r.value();
} else {
return {
ok: false,
msg: r.reason().message
}
}
});
res.json({ ok: true, results: uplResults });
}).catch((err) => {
res.json({ ok: false, msg: err.message });
});

2
models/entries.js
View File

@ -170,7 +170,7 @@ module.exports = {
return false;
}
}).catch((err) => {
return Promise.reject(new Error('Entry ' + entryPath + ' does not exist!'));
return Promise.reject(new Promise.OperationalError('Entry ' + entryPath + ' does not exist!'));
});
},

17
models/git.js
View File

@ -235,6 +235,23 @@ module.exports = {
return true;
})
},
/**
* Commits uploads changes.
*
* @return {Promise} Resolve on commit success
*/
commitUploads() {
let self = this;
return self._git.add('uploads').then(() => {
return self._git.commit("Uploads repository sync").catch((err) => {
if(_.includes(err.stdout, 'nothing to commit')) { return true; }
});
});
}
};

64
models/localdata.js
View File

@ -267,6 +267,22 @@ module.exports = {
},
/**
* Parse relative Uploads path
*
* @param {String} f Relative Uploads path
* @return {Object} Parsed path (folder and filename)
*/
parseUploadsRelPath(f) {
let fObj = path.parse(f);
return {
folder: fObj.dir,
filename: fObj.base
};
},
/**
* Sets the uploads files.
*
@ -288,6 +304,19 @@ module.exports = {
},
/**
* Adds one or more uploads files.
*
* @param {Array<Object>} arrFiles The uploads files
* @return {Void} Void
*/
addUploadsFiles(arrFiles) {
if(_.isArray(arrFiles) || _.isPlainObject(arrFiles)) {
this._uploadsDb.Files.insert(arrFiles);
}
return;
},
/**
* Gets the uploads files.
*
@ -301,41 +330,6 @@ module.exports = {
'$and': [{ 'category' : cat },{ 'folder' : fld }]
}).simplesort('filename').data();
},
/**
* Generate thumbnail of image
*
* @param {String} sourcePath The source path
* @return {Promise<Object>} Promise returning the resized image info
*/
generateThumbnail(sourcePath, destPath) {
let sharp = require('sharp');
return sharp(sourcePath)
.withoutEnlargement()
.resize(150,150)
.background('white')
.embed()
.flatten()
.toFormat('png')
.toFile(destPath);
},
/**
* Gets the image metadata.
*
* @param {String} sourcePath The source path
* @return {Object} The image metadata.
*/
getImageMetadata(sourcePath) {
let sharp = require('sharp');
return sharp(sourcePath).metadata();
}
};

176
models/uploads.js
View File

@ -0,0 +1,176 @@
"use strict";
var path = require('path'),
Promise = require('bluebird'),
fs = Promise.promisifyAll(require('fs-extra')),
readChunk = require('read-chunk'),
fileType = require('file-type'),
farmhash = require('farmhash'),
moment = require('moment'),
chokidar = require('chokidar'),
_ = require('lodash');
/**
* Uploads
*
* @param {Object} appconfig The application configuration
*/
module.exports = {
_uploadsPath: './repo/uploads',
_uploadsThumbsPath: './data/thumbs',
_watcher: null,
/**
* Initialize Uploads model
*
* @param {Object} appconfig The application config
* @return {Object} Uploads model instance
*/
init(appconfig) {
let self = this;
self._uploadsPath = path.resolve(ROOTPATH, appconfig.datadir.repo, 'uploads');
self._uploadsThumbsPath = path.resolve(ROOTPATH, appconfig.datadir.db, 'thumbs');
return self;
},
watch() {
let self = this;
self._watcher = chokidar.watch(self._uploadsPath, {
persistent: true,
ignoreInitial: true,
cwd: self._uploadsPath,
depth: 1,
awaitWriteFinish: true
});
self._watcher.on('add', (p) => {
let pInfo = lcdata.parseUploadsRelPath(p);
return self.processFile(pInfo.folder, pInfo.filename).then((mData) => {
ws.emit('uploadsAddFiles', {
auth: WSInternalKey,
content: mData
});
}).then(() => {
return git.commitUploads();
});
});
},
processFile(fldName, f) {
let self = this;
let fldPath = path.join(self._uploadsPath, fldName);
let fPath = path.join(fldPath, f);
let fPathObj = path.parse(fPath);
let fUid = farmhash.fingerprint32(fldName + '/' + f);
return fs.statAsync(fPath).then((s) => {
if(!s.isFile()) { return false; }
// Get MIME info
let mimeInfo = fileType(readChunk.sync(fPath, 0, 262));
// Images
if(s.size < 3145728) { // ignore files larger than 3MB
if(_.includes(['image/png', 'image/jpeg', 'image/gif', 'image/webp'], mimeInfo.mime)) {
return self.getImageMetadata(fPath).then((mData) => {
let cacheThumbnailPath = path.parse(path.join(self._uploadsThumbsPath, fUid + '.png'));
let cacheThumbnailPathStr = path.format(cacheThumbnailPath);
mData = _.pick(mData, ['format', 'width', 'height', 'density', 'hasAlpha', 'orientation']);
mData.uid = fUid;
mData.category = 'image';
mData.mime = mimeInfo.mime;
mData.folder = fldName;
mData.filename = f;
mData.basename = fPathObj.name;
mData.filesize = s.size;
mData.uploadedOn = moment().utc();
// Generate thumbnail
return fs.statAsync(cacheThumbnailPathStr).then((st) => {
return st.isFile();
}).catch((err) => {
return false;
}).then((thumbExists) => {
return (thumbExists) ? mData : fs.ensureDirAsync(cacheThumbnailPath.dir).then(() => {
return self.generateThumbnail(fPath, cacheThumbnailPathStr);
}).return(mData);
});
})
}
}
// Other Files
return {
uid: fUid,
category: 'file',
mime: mimeInfo.mime,
folder: fldName,
filename: f,
basename: fPathObj.name,
filesize: s.size,
uploadedOn: moment().utc()
};
});
},
/**
* Generate thumbnail of image
*
* @param {String} sourcePath The source path
* @return {Promise<Object>} Promise returning the resized image info
*/
generateThumbnail(sourcePath, destPath) {
let sharp = require('sharp');
return sharp(sourcePath)
.withoutEnlargement()
.resize(150,150)
.background('white')
.embed()
.flatten()
.toFormat('png')
.toFile(destPath);
},
/**
* Gets the image metadata.
*
* @param {String} sourcePath The source path
* @return {Object} The image metadata.
*/
getImageMetadata(sourcePath) {
let sharp = require('sharp');
return sharp(sourcePath).metadata();
}
};

1
package.json
View File

@ -39,6 +39,7 @@
"bson": "^0.5.5",
"cheerio": "^0.22.0",
"child-process-promise": "^2.1.3",
"chokidar": "^1.6.0",
"compression": "^1.6.2",
"connect-flash": "^0.1.1",
"connect-loki": "^1.0.6",

32
views/error-notexist.pug
View File

@ -0,0 +1,32 @@
doctype html
html
head
meta(http-equiv='X-UA-Compatible', content='IE=edge')
meta(charset='UTF-8')
meta(name='viewport', content='width=device-width, initial-scale=1')
meta(name='theme-color', content='#009688')
meta(name='msapplication-TileColor', content='#009688')
meta(name='msapplication-TileImage', content='/favicons/ms-icon-144x144.png')
title= appconfig.title
// Favicon
each favsize in [57, 60, 72, 76, 114, 120, 144, 152, 180]
link(rel='apple-touch-icon', sizes=favsize + 'x' + favsize, href='/favicons/apple-icon-' + favsize + 'x' + favsize + '.png')
link(rel='icon', type='image/png', sizes='192x192', href='/favicons/android-icon-192x192.png')
each favsize in [32, 96, 16]
link(rel='icon', type='image/png', sizes=favsize + 'x' + favsize, href='/favicons/favicon-' + favsize + 'x' + favsize + '.png')
link(rel='manifest', href='/manifest.json')
// CSS
link(type='text/css', rel='stylesheet', href='/css/libs.css')
link(type='text/css', rel='stylesheet', href='/css/app.css')
body(class='server-error')
section.hero.is-dark.is-fullheight
.hero-body
.container
a(href='/'): img(src='/favicons/android-icon-96x96.png')
h1.title(style={ 'margin-top': '30px'})= message
h2.subtitle(style={ 'margin-bottom': '50px'}) Would you like to create this entry?
a.button.is-dark.is-inverted(href='/create/' + newpath, style={'margin-right': '5px'}) Create
a.button.is-dark.is-inverted(href='/') Go Home

16
ws-server.js
View File

@ -108,12 +108,14 @@ io.on('connection', (socket) => {
});
socket.on('searchDel', (data, cb) => {
cb = cb || _.noop
if(internalAuth.validateKey(data.auth)) {
search.delete(data.entryPath);
}
});
socket.on('search', (data, cb) => {
cb = cb || _.noop
search.find(data.terms).then((results) => {
cb(results);
});
@ -124,28 +126,42 @@ io.on('connection', (socket) => {
//-----------------------------------------
socket.on('uploadsSetFolders', (data, cb) => {
cb = cb || _.noop
if(internalAuth.validateKey(data.auth)) {
lcdata.setUploadsFolders(data.content);
}
});
socket.on('uploadsGetFolders', (data, cb) => {
cb = cb || _.noop
cb(lcdata.getUploadsFolders());
});
socket.on('uploadsCreateFolder', (data, cb) => {
cb = cb || _.noop
lcdata.createUploadsFolder(data.foldername).then((fldList) => {
cb(fldList);
});
});
socket.on('uploadsSetFiles', (data, cb) => {
cb = cb || _.noop;
if(internalAuth.validateKey(data.auth)) {
lcdata.setUploadsFiles(data.content);
cb(true);
}
});
socket.on('uploadsAddFiles', (data, cb) => {
cb = cb || _.noop
if(internalAuth.validateKey(data.auth)) {
lcdata.addUploadsFiles(data.content);
cb(true);
}
});
socket.on('uploadsGetImages', (data, cb) => {
cb = cb || _.noop
cb(lcdata.getUploadsFiles('image', data.folder));
});

Write Preview
Loading…
Cancel
Save