From 3d0fb303e10f3535e13a1c8bc73902a533ec1f16 Mon Sep 17 00:00:00 2001 From: Bram van der Veen <96aa48@gmail.com> Date: Sat, 15 Aug 2015 21:48:38 +0200 Subject: [PATCH] Added localDatabase support with NeDB --- .gitignore | 1 + database.js | 27 +++++++++++++++++++++++++ lookup.js | 57 +++++++++++++++++++++++++++++++--------------------- package.json | 1 + schedule.js | 4 ++-- spider.js | 12 +++-------- 6 files changed, 68 insertions(+), 34 deletions(-) create mode 100644 database.js diff --git a/.gitignore b/.gitignore index 12b3737..ae72e15 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ node_modules settings.json +resources/databases/* diff --git a/database.js b/database.js new file mode 100644 index 0000000..15ac4fb --- /dev/null +++ b/database.js @@ -0,0 +1,27 @@ +//database.js +var config = require('./configuration'); +var fs = require('fs'); + +module.exports = function () { + if (!config().localDatabase) return require('mongoskin').db('mongodb://' + config().database); + else { + var databases = { + index: new (require('nedb'))({ filename: __dirname + '/resources/databases/index.db', autoload: true}) + }; + + return { + 'collection': function (collection) { + var database = databases[collection]; + + database.drop = function () { + fs.writeFileSync(database.filename, ''); + } + + return database; + }, + 'close': function () { + return; + } + } + } +} diff --git a/lookup.js b/lookup.js index c3bcfc5..7d19490 100644 --- a/lookup.js +++ b/lookup.js @@ -6,7 +6,7 @@ var schoolID = config().schoolID; //Getting first and third party modules var fs = require('fs'); -var database = require('mongoskin').db('mongodb://' + config().database); +var database = require('./database')(); function get(req, res, next, search) { var index = database.collection('index'); @@ -14,31 +14,42 @@ function get(req, res, next, search) { easter(search) ? search = easter(search).name : null; search = new RegExp(search, 'i'); - index.find({$or : [{id : search}, {name : search}, {first_name : search}, {last_name : search}, {username: search}]}).toArray(function (err, databaseEntry) { - if (err) console.warn(err); + if (!config().localDatabase) { + index.find({$or : [{id : search}, {name : search}, {first_name : search}, {last_name : search}, {username: search}]}).toArray(function (err, databaseEntry) { + if (err) console.warn(err); + handle(req, next, databaseEntry); + }); + } + else { + index.find({$or : [{id : search}, {name : search}, {first_name : search}, {last_name : search}, {username: search}]}, function (err, databaseEntry) { + if (err) console.warn(err); + handle(req, next, databaseEntry); + }); + } +} - if ((req.easter || {}).type == 'RIP') { - require('./auth').is(req, res, function () { - res.render('schedule', req); - }); - } - else if (databaseEntry.length == 1) { - databaseEntry[0].url = makeUrl(req, databaseEntry[0]); - req.match = databaseEntry[0]; - next(); - } - else if (databaseEntry.length == 0) { +function handle(req, next, databaseEntry) { + if ((req.easter || {}).type == 'RIP') { require('./auth').is(req, res, function () { - res.render('not_found', req); + res.render('schedule', req); }); - } - else { - req.match = databaseEntry; - require('./auth').is(req, res, function () { - res.render('list', req); - }); - } - }); + } + else if (databaseEntry.length == 1) { + databaseEntry[0].url = makeUrl(req, databaseEntry[0]); + req.match = databaseEntry[0]; + next(); + } + else if (databaseEntry.length == 0) { + require('./auth').is(req, res, function () { + res.render('not_found', req); + }); + } + else { + req.match = databaseEntry; + require('./auth').is(req, res, function () { + res.render('list', req); + }); + } } function api(req, callback) { diff --git a/package.json b/package.json index a2e23ca..6a3b3bb 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,7 @@ "jade": "^1.11.0", "mongodb": "^1.4.38", "mongoskin": "^1.4.13", + "nedb": "^1.1.3", "socks5-http-client": "^1.0.1", "socks5-https-client": "^1.1.1" } diff --git a/schedule.js b/schedule.js index 00b60ba..c134e8b 100644 --- a/schedule.js +++ b/schedule.js @@ -35,7 +35,7 @@ function getSchedule(getUrl, callback) { }); } -function types(page) { +function scheduleTypes(page) { var extract = cheerio('table tr td[valign="bottom"] table tr td b, table tr td[valign="bottom"] table tr td a', page).text().split(/\s\s/); var tab = 0; var types = []; @@ -54,7 +54,7 @@ function types(page) { //Function for converting the page into a json dataset. function toJSON(page) { var result = cheerio('td:nth-child(3) table', page); - var types = types(page); + var types = scheduleTypes(page); var isTeacher = cheerio(cheerio(page).find('tr.CoreDark').find('td')[3]).find('a').html() == null; var amountOfDays = cheerio(result).find('tr.AccentDark').find('td').length - 1; var amountOfHours = config().amountOfHours; diff --git a/spider.js b/spider.js index 7a7fea5..7d0a481 100644 --- a/spider.js +++ b/spider.js @@ -1,9 +1,9 @@ var http = require('socks5-http-client'); var cheerio = require('cheerio'); var iconv = require('iconv-lite'); -var mongodb = require('mongodb').MongoClient; var config = require('./configuration'); var url = require('url'); +var database = require('./')(); var scheduletypes = [ 'Klasrooster', @@ -12,7 +12,6 @@ var scheduletypes = [ 'Lokaalrooster' ]; var schoolID; -var database; //Function for getting pages with http requests. function get() { @@ -123,12 +122,7 @@ function rip(data) { //Function being called to access functionality from this module. function crawl() { schoolID = config().schoolID; - mongodb.connect('mongodb://' + config().database, function (error, db) { - if (error) console.warn(error); - database = db; - - get(); - }); + get(); } @@ -144,6 +138,6 @@ module.exports = { 'crawl' : crawl } -if (process.argv[2] == 'test') { +if (process.argv[2] == 'test' || process.argv[2] == 'rip') { module.exports.crawl(934); }