mirror of https://github.com/Requarks/wiki.git
Nick
6 years ago
6 changed files with 297 additions and 26 deletions
Unified View
Diff Options
-
1package.json
-
2server/modules/search/algolia/definition.yml
-
204server/modules/search/algolia/engine.js
-
13server/modules/search/aws/engine.js
-
6server/modules/search/azure/engine.js
-
97yarn.lock
@ -1,26 +1,202 @@ |
|||||
module.exports = { |
const _ = require('lodash') |
||||
activate() { |
const algoliasearch = require('algoliasearch') |
||||
|
const { pipeline, Transform } = require('stream') |
||||
|
|
||||
}, |
/* global WIKI */ |
||||
deactivate() { |
|
||||
|
|
||||
|
module.exports = { |
||||
|
async activate() { |
||||
|
// not used
|
||||
}, |
}, |
||||
query() { |
async deactivate() { |
||||
|
// not used
|
||||
}, |
}, |
||||
created() { |
/** |
||||
|
* INIT |
||||
|
*/ |
||||
|
async init() { |
||||
|
WIKI.logger.info(`(SEARCH/ALGOLIA) Initializing...`) |
||||
|
this.client = algoliasearch(this.config.appId, this.config.apiKey) |
||||
|
this.index = this.client.initIndex(this.config.indexName) |
||||
|
|
||||
|
// -> Create Search Index
|
||||
|
WIKI.logger.info(`(SEARCH/ALGOLIA) Setting index configuration...`) |
||||
|
await this.index.setSettings({ |
||||
|
searchableAttributes: [ |
||||
|
'title', |
||||
|
'description', |
||||
|
'content' |
||||
|
], |
||||
|
attributesToRetrieve: [ |
||||
|
'locale', |
||||
|
'path', |
||||
|
'title', |
||||
|
'description' |
||||
|
], |
||||
|
advancedSyntax: true |
||||
|
}) |
||||
|
WIKI.logger.info(`(SEARCH/ALGOLIA) Initialization completed.`) |
||||
}, |
}, |
||||
updated() { |
/** |
||||
|
* QUERY |
||||
|
* |
||||
|
* @param {String} q Query |
||||
|
* @param {Object} opts Additional options |
||||
|
*/ |
||||
|
async query(q, opts) { |
||||
|
try { |
||||
|
const results = await this.index.search({ |
||||
|
query: q, |
||||
|
hitsPerPage: 50 |
||||
|
}) |
||||
|
return { |
||||
|
results: _.map(results.hits, r => ({ |
||||
|
id: r.objectID, |
||||
|
locale: r.locale, |
||||
|
path: r.path, |
||||
|
title: r.title, |
||||
|
description: r.description |
||||
|
})), |
||||
|
suggestions: [], |
||||
|
totalHits: results.nbHits |
||||
|
} |
||||
|
} catch (err) { |
||||
|
WIKI.logger.warn('Search Engine Error:') |
||||
|
WIKI.logger.warn(err) |
||||
|
} |
||||
}, |
}, |
||||
deleted() { |
/** |
||||
|
* CREATE |
||||
|
* |
||||
|
* @param {Object} page Page to create |
||||
|
*/ |
||||
|
async created(page) { |
||||
|
await this.index.addObject({ |
||||
|
objectID: page.hash, |
||||
|
locale: page.localeCode, |
||||
|
path: page.path, |
||||
|
title: page.title, |
||||
|
description: page.description, |
||||
|
content: page.content |
||||
|
}) |
||||
}, |
}, |
||||
renamed() { |
/** |
||||
|
* UPDATE |
||||
|
* |
||||
|
* @param {Object} page Page to update |
||||
|
*/ |
||||
|
async updated(page) { |
||||
|
await this.index.partialUpdateObject({ |
||||
|
objectID: page.hash, |
||||
|
title: page.title, |
||||
|
description: page.description, |
||||
|
content: page.content |
||||
|
}) |
||||
|
}, |
||||
|
/** |
||||
|
* DELETE |
||||
|
* |
||||
|
* @param {Object} page Page to delete |
||||
|
*/ |
||||
|
async deleted(page) { |
||||
|
await this.index.deleteObject(page.hash) |
||||
}, |
}, |
||||
rebuild() { |
/** |
||||
|
* RENAME |
||||
|
* |
||||
|
* @param {Object} page Page to rename |
||||
|
*/ |
||||
|
async renamed(page) { |
||||
|
await this.index.deleteObject(page.sourceHash) |
||||
|
await this.index.addObject({ |
||||
|
objectID: page.destinationHash, |
||||
|
locale: page.localeCode, |
||||
|
path: page.destinationPath, |
||||
|
title: page.title, |
||||
|
description: page.description, |
||||
|
content: page.content |
||||
|
}) |
||||
|
}, |
||||
|
/** |
||||
|
* REBUILD INDEX |
||||
|
*/ |
||||
|
async rebuild() { |
||||
|
WIKI.logger.info(`(SEARCH/ALGOLIA) Rebuilding Index...`) |
||||
|
await this.index.clearIndex() |
||||
|
|
||||
|
const MAX_DOCUMENT_BYTES = 10 * Math.pow(2, 10) // 10 KB
|
||||
|
const MAX_INDEXING_BYTES = 10 * Math.pow(2, 20) - Buffer.from('[').byteLength - Buffer.from(']').byteLength // 10 MB
|
||||
|
const MAX_INDEXING_COUNT = 1000 |
||||
|
const COMMA_BYTES = Buffer.from(',').byteLength |
||||
|
|
||||
|
let chunks = [] |
||||
|
let bytes = 0 |
||||
|
|
||||
|
const processDocument = async (cb, doc) => { |
||||
|
try { |
||||
|
if (doc) { |
||||
|
const docBytes = Buffer.from(JSON.stringify(doc)).byteLength |
||||
|
// -> Document too large
|
||||
|
if (docBytes >= MAX_DOCUMENT_BYTES) { |
||||
|
throw new Error('Document exceeds maximum size allowed by Algolia.') |
||||
|
} |
||||
|
|
||||
|
// -> Current batch exceeds size hard limit, flush
|
||||
|
if (docBytes + COMMA_BYTES + bytes >= MAX_INDEXING_BYTES) { |
||||
|
await flushBuffer() |
||||
|
} |
||||
|
|
||||
|
if (chunks.length > 0) { |
||||
|
bytes += COMMA_BYTES |
||||
|
} |
||||
|
bytes += docBytes |
||||
|
chunks.push(doc) |
||||
|
|
||||
|
// -> Current batch exceeds count soft limit, flush
|
||||
|
if (chunks.length >= MAX_INDEXING_COUNT) { |
||||
|
await flushBuffer() |
||||
|
} |
||||
|
} else { |
||||
|
// -> End of stream, flush
|
||||
|
await flushBuffer() |
||||
|
} |
||||
|
cb() |
||||
|
} catch (err) { |
||||
|
cb(err) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
const flushBuffer = async () => { |
||||
|
WIKI.logger.info(`(SEARCH/ALGOLIA) Sending batch of ${chunks.length}...`) |
||||
|
try { |
||||
|
await this.index.addObjects( |
||||
|
_.map(chunks, doc => ({ |
||||
|
objectID: doc.id, |
||||
|
locale: doc.locale, |
||||
|
path: doc.path, |
||||
|
title: doc.title, |
||||
|
description: doc.description, |
||||
|
content: doc.content |
||||
|
})) |
||||
|
) |
||||
|
} catch (err) { |
||||
|
WIKI.logger.warn('(SEARCH/ALGOLIA) Failed to send batch to Algolia: ', err) |
||||
|
} |
||||
|
chunks.length = 0 |
||||
|
bytes = 0 |
||||
|
} |
||||
|
|
||||
|
await pipeline( |
||||
|
WIKI.models.knex.column({ id: 'hash' }, 'path', { locale: 'localeCode' }, 'title', 'description', 'content').select().from('pages').where({ |
||||
|
isPublished: true, |
||||
|
isPrivate: false |
||||
|
}).stream(), |
||||
|
new Transform({ |
||||
|
objectMode: true, |
||||
|
transform: async (chunk, enc, cb) => processDocument(cb, chunk), |
||||
|
flush: async (cb) => processDocument(cb) |
||||
|
}) |
||||
|
) |
||||
|
WIKI.logger.info(`(SEARCH/ALGOLIA) Index rebuilt successfully.`) |
||||
} |
} |
||||
} |
} |
||||
xxxxxxxxxx