Search results picker + create/update index update

agent.js

@@ -68,6 +68,7 @@ var job = new cron({
 			winston.warn('[AGENT] Previous job has not completed gracefully or is still running! Skipping for now. (This is not normal, you should investigate)');
 			return;
 		}
+		winston.info('[AGENT] Running all jobs...');
 		jobIsBusy = true;
 
 		// Prepare async job collector
@@ -87,6 +88,10 @@ var job = new cron({
 				//-> Stream all documents
 
 				let cacheJobs = [];
+				let jobCbStreamDocs_resolve = null,
+						jobCbStreamDocs = new Promise((resolve, reject) => {
+							jobCbStreamDocs_resolve = resolve;
+						});
 
 				fs.walk(repoPath).on('data', function (item) {
 					if(path.extname(item.path) === '.md') {
@@ -113,15 +118,10 @@ var job = new cron({
 
 							}).then((fileStatus) => {
 
-								//-> Update search index
+								//-> Update cache and search index
 
 								if(fileStatus !== 'active') {
-									return entries.fetchIndexableVersion(entryPath).then((content) => {
+									return entries.updateCache(entryPath);
-										ws.emit('searchAdd', {
-											auth: WSInternalKey,
-											content
-										});
-									});
 								}
 
 								return true;
@@ -131,9 +131,11 @@ var job = new cron({
 						);
 
 					}
+				}).on('end', () => {
+					jobCbStreamDocs_resolve(Promise.all(cacheJobs));
 				});
 
-				return Promise.all(cacheJobs);
+				return jobCbStreamDocs;
 
 			});
 		}));
@@ -143,7 +145,7 @@ var job = new cron({
 		// ----------------------------------------
 
 		Promise.all(jobs).then(() => {
-			winston.info('[AGENT] All jobs completed successfully! Going to sleep for now...');
+			winston.info('[AGENT] All jobs completed successfully! Going to sleep for now.');
 		}).catch((err) => {
 			winston.error('[AGENT] One or more jobs have failed: ', err);
 		}).finally(() => {
@@ -161,8 +163,8 @@ var job = new cron({
 // ----------------------------------------
 
 ws.on('connect', function () {
-	job.start();
 	winston.info('[AGENT] Background Agent started successfully! [RUNNING]');
+	job.start();
 });
 
 ws.on('connect_error', function () {

client/js/components/search.js

@@ -27,24 +27,32 @@ jQuery( document ).ready(function( $ ) {
 			},
 			watch: {
 				searchq: (val, oldVal) => {
-					searchmoveidx: 0;
+					vueHeader.searchmoveidx = 0;
 					if(val.length >= 3) {
 						vueHeader.searchactive = true;
 						vueHeader.searchload++;
 						socket.emit('search', { terms: val }, (data) => {
 							vueHeader.searchres = data.match;
 							vueHeader.searchsuggest = data.suggest;
+							vueHeader.searchmovearr = _.concat([], vueHeader.searchres, vueHeader.searchsuggest);
 							if(vueHeader.searchload > 0) { vueHeader.searchload--; }
 						});
 					} else {
 						vueHeader.searchactive = false;
 						vueHeader.searchres = [];
 						vueHeader.searchsuggest = [];
+						vueHeader.searchmovearr = [];
 						vueHeader.searchload = 0;
 					}
 				},
 				searchmoveidx: (val, oldVal) => {
-
+					if(val > 0) {
+						vueHeader.searchmovekey = (vueHeader.searchmovearr[val - 1].document) ?
+																				'res.' + vueHeader.searchmovearr[val - 1].document.entryPath :
+																				'sug.' + vueHeader.searchmovearr[val - 1];
+					} else {
+						vueHeader.searchmovekey = '';
+					}
 				}
 			},
 			methods: {
@@ -53,13 +61,20 @@ jQuery( document ).ready(function( $ ) {
 				},
 				closeSearch: () => {
 					vueHeader.searchq = '';
-					vueHeader.searchactive = false;
 				},
 				moveSelectSearch: () => {
+					if(vueHeader.searchmoveidx < 1) { return; }
+					let i = vueHeader.searchmoveidx - 1;
+
+					if(vueHeader.searchmovearr[i].document) {
+						window.location.assign('/' + vueHeader.searchmovearr[i].document.entryPath);
+					} else {
+						vueHeader.searchq = vueHeader.searchmovearr[i];
+					}
 
 				},
 				moveDownSearch: () => {
-					if(vueHeader.searchmoveidx < vueHeader.searchmovearr) {
+					if(vueHeader.searchmoveidx < vueHeader.searchmovearr.length) {
 						vueHeader.searchmoveidx++;
 					}
 				},

client/scss/components/_alerts.scss

@@ -3,7 +3,7 @@
 	top: 60px;
 	right: 10px;
 	width: 350px;
-	z-index: 2;
+	z-index: 10;
 	text-shadow: 1px 1px 0 rgba(0,0,0,0.1);
 
 	.notification {

controllers/pages.js

@@ -71,7 +71,7 @@ router.get('/create/*', (req, res, next) => {
 
 	entries.exists(safePath).then((docExists) => {
 		if(!docExists) {
-			entries.getStarter(safePath).then((contents) => {
+			return entries.getStarter(safePath).then((contents) => {
 
 				let pageData = {
 					markdown: contents,
@@ -80,7 +80,9 @@ router.get('/create/*', (req, res, next) => {
 						path: safePath
 					}
 				};
-				return res.render('pages/create', { pageData });
+				res.render('pages/create', { pageData });
+
+				return true;
 
 			}).catch((err) => {
 				throw new Error('Could not load starter content!');

models/entries.js

@@ -156,7 +156,7 @@ module.exports = {
 						// Cache to disk
 
 						if(options.cache) {
-							let cacheData = BSON.serialize(pageData, false, false, false);
+							let cacheData = BSON.serialize(_.pick(pageData, ['html', 'meta', 'tree', 'parent']), false, false, false);
 							return fs.writeFileAsync(cpath, cacheData).catch((err) => {
 								winston.error('Unable to write to cache! Performance may be affected.');
 								return true;
@@ -177,34 +177,6 @@ module.exports = {
 
 	},
 
-	/**
-	 * Fetches a text version of a Markdown-formatted document
-	 *
-	 * @param      {String}  entryPath  The entry path
-	 * @return     {String}  Text-only version
-	 */
-	fetchIndexableVersion(entryPath) {
-
-		let self = this;
-
-		return self.fetchOriginal(entryPath, {
-			parseMarkdown: false,
-			parseMeta: true,
-			parseTree: false,
-			includeMarkdown: true,
-			includeParentInfo: true,
-			cache: false
-		}).then((pageData) => {
-			return {
-				entryPath,
-				meta: pageData.meta,
-				parent: pageData.parent || {},
-				text: mark.removeMarkdown(pageData.markdown)
-			};
-		});
-
-	},
-
 	/**
 	 * Parse raw url path and make it safe
 	 *
@@ -314,13 +286,48 @@ module.exports = {
 		return fs.statAsync(fpath).then((st) => {
 			if(st.isFile()) {
 				return self.makePersistent(entryPath, contents).then(() => {
-					return self.fetchOriginal(entryPath, {});
+					return self.updateCache(entryPath);
 				});
 			} else {
 				return Promise.reject(new Error('Entry does not exist!'));
 			}
 		}).catch((err) => {
-			return Promise.reject(new Error('Entry does not exist!'));
+			winston.error(err);
+			return Promise.reject(new Error('Failed to save document.'));
+		});
+
+	},
+
+	/**
+	 * Update local cache and search index
+	 *
+	 * @param      {String}   entryPath  The entry path
+	 * @return     {Promise}  Promise of the operation
+	 */
+	updateCache(entryPath) {
+
+		let self = this;
+
+		return self.fetchOriginal(entryPath, {
+			parseMarkdown: true,
+			parseMeta: true,
+			parseTree: true,
+			includeMarkdown: true,
+			includeParentInfo: true,
+			cache: true
+		}).then((pageData) => {
+			return {
+				entryPath,
+				meta: pageData.meta,
+				parent: pageData.parent || {},
+				text: mark.removeMarkdown(pageData.markdown)
+			};
+		}).then((content) => {
+			ws.emit('searchAdd', {
+				auth: WSInternalKey,
+				content
+			});
+			return true;
 		});
 
 	},
@@ -339,7 +346,7 @@ module.exports = {
 		return self.exists(entryPath).then((docExists) => {
 			if(!docExists) {
 				return self.makePersistent(entryPath, contents).then(() => {
-					return self.fetchOriginal(entryPath, {});
+					return self.updateCache(entryPath);
 				});
 			} else {
 				return Promise.reject(new Error('Entry already exists!'));

models/git.js

@@ -195,7 +195,9 @@ module.exports = {
 			commitMsg = (isTracked) ? 'Updated ' + gitFilePath : 'Added ' + gitFilePath;
 			return self._git.add(gitFilePath);
 		}).then(() => {
-			return self._git.commit(commitMsg);
+			return self._git.commit(commitMsg).catch((err) => {
+			  if(_.includes(err.stdout, 'nothing to commit')) { return true; }
+			});
 		});
 
 	}

models/search.js

@@ -100,42 +100,62 @@ module.exports = {
 
 		let self = this;
 
-		return self._si.addAsync({
+		return self._si.searchAsync({
-			entryPath: content.entryPath,
+			query: {
-			title: content.meta.title,
+				AND: [{ 'entryPath': [content.entryPath] }]
-			subtitle: content.meta.subtitle || '',
+			}
-			parent: content.parent.title || '',
+		}).then((results) => {
-			content: content.text || ''
+			if(results.totalHits > 0) {
-		}, {
+				let delIds = _.map(results.hits, 'id');
-			fieldOptions: [{
+				return self._si.delAsync(delIds);
-				fieldName: 'entryPath',
+			} else {
-				searchable: true,
+				return true;
-				weight: 2
+			}
-			},
+
-			{
-				fieldName: 'title',
-				nGramLength: [1, 2],
-				searchable: true,
-				weight: 3
-			},
-			{
-				fieldName: 'subtitle',
-				searchable: true,
-				weight: 1,
-				store: false
-			},
-			{
-				fieldName: 'subtitle',
-				searchable: false,
-			},
-			{
-				fieldName: 'content',
-				searchable: true,
-				weight: 0,
-				store: false
-			}]
 		}).then(() => {
-			winston.info('Entry ' + content.entryPath + ' added to index.');
+			return self._si.addAsync({
+				entryPath: content.entryPath,
+				title: content.meta.title,
+				subtitle: content.meta.subtitle || '',
+				parent: content.parent.title || '',
+				content: content.text || ''
+			}, {
+				fieldOptions: [{
+					fieldName: 'entryPath',
+					searchable: true,
+					weight: 2
+				},
+				{
+					fieldName: 'title',
+					nGramLength: [1, 2],
+					searchable: true,
+					weight: 3
+				},
+				{
+					fieldName: 'subtitle',
+					searchable: true,
+					weight: 1,
+					store: false
+				},
+				{
+					fieldName: 'parent',
+					searchable: false,
+				},
+				{
+					fieldName: 'content',
+					searchable: true,
+					weight: 0,
+					store: false
+				}]
+			}).then(() => {
+				winston.info('Entry ' + content.entryPath + ' added/updated to index.');
+				return true;
+			}).catch((err) => {
+				winston.error(err);
+			});
+
 		}).catch((err) => {
 			winston.error(err);
 		});

server.js

@@ -206,13 +206,32 @@ server.on('listening', () => {
 // ----------------------------------------
 
 var fork = require('child_process').fork,
-    libInternalAuth = require('./lib/internalAuth'),
+    libInternalAuth = require('./lib/internalAuth');
-    internalAuthKey = libInternalAuth.generateKey();
 
-var wsSrv = fork('ws-server.js', [internalAuthKey]),
+global.WSInternalKey = libInternalAuth.generateKey();
-    bgAgent = fork('agent.js', [internalAuthKey]);
+var wsSrv = fork('ws-server.js', [WSInternalKey]),
+    bgAgent = fork('agent.js', [WSInternalKey]);
 
 process.on('exit', (code) => {
   wsSrv.disconnect();
   bgAgent.disconnect();
+});
+
+// ----------------------------------------
+// Connect to local WebSocket server
+// ----------------------------------------
+
+var wsClient = require('socket.io-client');
+global.ws = wsClient('http://localhost:' + appconfig.wsPort, { reconnectionAttempts: 10 });
+
+ws.on('connect', function () {
+  winston.info('[SERVER] Connected to WebSocket server successfully!');
+});
+ws.on('connect_error', function () {
+  winston.warn('[SERVER] Unable to connect to WebSocket server! Retrying...');
+});
+ws.on('reconnect_failed', function () {
+  winston.error('[SERVER] Failed to reconnect to WebSocket server too many times! Stopping...');
+  process.exit(1);
 });

views/common/header.pug

@@ -11,7 +11,7 @@
 			block rootNavCenter
 				.nav-item
 					p.control(v-bind:class="{ 'is-loading': searchload > 0 }")
-						input.input#search-input(type='text', v-model='searchq', @keyup.esc='closeSearch', @keyup.down='moveDownSearch', @keyup.up='moveUpSearch', debounce='400', placeholder='Search...')
+						input.input#search-input(type='text', v-model='searchq', @keyup.esc='closeSearch', @keyup.down='moveDownSearch', @keyup.up='moveUpSearch', @keyup.enter='moveSelectSearch', debounce='400', placeholder='Search...')
 		span.nav-toggle
 			span
 			span
@@ -46,6 +46,6 @@
 				| Did you mean...?
 			ul.menu-list(v-if='searchsuggest.length > 0')
 				li(v-for='sug in searchsuggest')
-					a(v-on:click="useSuggestion(sug)") {{ sug }}
+					a(v-on:click="useSuggestion(sug)", v-bind:class="{ 'is-active': searchmovekey === 'sug.' + sug }") {{ sug }}
 
 

views/layout.pug

@@ -31,10 +31,10 @@ html
 
 	body
 		#root
-			include ./common/header
+			include ./common/header.pug
-			include ./common/alerts
+			include ./common/alerts.pug
 			main
 				block content
-			include ./common/footer
+			include ./common/footer.pug
 
 		block outside

views/pages/create.pug

@@ -1,4 +1,4 @@
-extends ../layout
+extends ../layout.pug
 
 block rootNavCenter
 	h2.nav-item Create New Document

views/pages/edit.pug

@@ -1,4 +1,4 @@
-extends ../layout
+extends ../layout.pug
 
 block rootNavCenter
 	h2.nav-item= pageData.meta.title
@@ -24,4 +24,4 @@ block content
 	#page-type-edit(data-entrypath=pageData.meta.path)
 		textarea#mk-editor= pageData.markdown
 
-	include ../modals/edit
+	include ../modals/edit.pug

views/pages/source.pug

@@ -1,4 +1,4 @@
-extends ../layout
+extends ../layout.pug
 
 block content
 

views/pages/view.pug

@@ -1,4 +1,4 @@
-extends ../layout
+extends ../layout.pug
 
 mixin tocMenu(ti)
 	each node in ti
@@ -61,4 +61,4 @@ block content
 						.content.mkcontent
 							!= pageData.html
 
-	include ../modals/create
+	include ../modals/create.pug