bram/rooster.io
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

Added socks support with configurable port.

Browse source
This commit is contained in:
Bram van der Veen 2015-06-24 17:07:52 +02:00
parent a753dcc260
commit 4a51022039
3 changed files with 18 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
configuration.js
View file

@ -9,6 +9,7 @@ module.exports = function () {
'env' : 'dev', 'env' : 'dev',
'database' : 'example.com/database', 'database' : 'example.com/database',
'web_port' : 1024, 'web_port' : 1024,
'tor_port' : 9050,
'amount_of_hours' : 7, 'amount_of_hours' : 7,
'hour_times' : [ 'hour_times' : [
'8:45 - 9:45', '8:45 - 9:45',

11
schedule.js
View file

@ -1,7 +1,8 @@
//schedule.js //schedule.js
var http = require('http'); var http = require('socks5-http-client');
var cheerio = require('cheerio'); var cheerio = require('cheerio');
var config = require('./configuration'); var config = require('./configuration');
var url = require('url');
//Wrapper function that is being called by express. //Wrapper function that is being called by express.
function schedule(req, res, next) { function schedule(req, res, next) {
@ -13,7 +14,13 @@ function schedule(req, res, next) {
//Function for getting the page via http. //Function for getting the page via http.
function get(url, callback) { function get(url, callback) {
http.get(url, function (res) {
var options = url.parse(url);
options.socksPort = config().tor_port;
console.log(options);
http.get(options, function (res) {
var _download = ''; var _download = '';
res.on('data', function (data) { res.on('data', function (data) {

12
spider.js
View file

@ -1,8 +1,9 @@
var http = require('http'); var http = require('socsk5-http-client');
var cheerio = require('cheerio'); var cheerio = require('cheerio');
var iconv = require('iconv-lite'); var iconv = require('iconv-lite');
var mongodb = require('mongodb').MongoClient; var mongodb = require('mongodb').MongoClient;
var config = require('./configuration'); var config = require('./configuration');
var url = require('url');
var scheduletypes = [ var scheduletypes = [
'Klasrooster', 'Klasrooster',
@ -21,9 +22,10 @@ function get() {
(function (scheduletype) { (function (scheduletype) {
var link = 'http://roosters5.gepro-osi.nl/roosters/rooster.php?school=' + school_id + '&type=' + scheduletype; var options = url.parse('http://roosters5.gepro-osi.nl/roosters/rooster.php?school=' + school_id + '&type=' + scheduletype);
options.socksPort = config().tor_port;
http.get(link, function (res) { http.get(options, function (res) {
var _download = {}; var _download = {};
_download.type = scheduletype; _download.type = scheduletype;
@ -58,8 +60,10 @@ function rip(data) {
for(studentcategory of list) { for(studentcategory of list) {
(function (studentcategory) { (function (studentcategory) {
var options = url.parse('http://roosters5.gepro-osi.nl/roosters/rooster.php?school=' + school_id + '&type=' + data.type + '&afdeling=' + studentcategory);
options.socksPort = config().tor_port;
http.get('http://roosters5.gepro-osi.nl/roosters/rooster.php?school=' + school_id + '&type=' + data.type + '&afdeling=' + studentcategory, function (res) { http.get(options, function (res) {
var _download = ''; var _download = '';
res.on('data', function (data) { res.on('data', function (data) {