Added socks support with configurable port.
This commit is contained in:
parent
a753dcc260
commit
4a51022039
|
@ -9,6 +9,7 @@ module.exports = function () {
|
||||||
'env' : 'dev',
|
'env' : 'dev',
|
||||||
'database' : 'example.com/database',
|
'database' : 'example.com/database',
|
||||||
'web_port' : 1024,
|
'web_port' : 1024,
|
||||||
|
'tor_port' : 9050,
|
||||||
'amount_of_hours' : 7,
|
'amount_of_hours' : 7,
|
||||||
'hour_times' : [
|
'hour_times' : [
|
||||||
'8:45 - 9:45',
|
'8:45 - 9:45',
|
||||||
|
|
11
schedule.js
11
schedule.js
|
@ -1,7 +1,8 @@
|
||||||
//schedule.js
|
//schedule.js
|
||||||
var http = require('http');
|
var http = require('socks5-http-client');
|
||||||
var cheerio = require('cheerio');
|
var cheerio = require('cheerio');
|
||||||
var config = require('./configuration');
|
var config = require('./configuration');
|
||||||
|
var url = require('url');
|
||||||
|
|
||||||
//Wrapper function that is being called by express.
|
//Wrapper function that is being called by express.
|
||||||
function schedule(req, res, next) {
|
function schedule(req, res, next) {
|
||||||
|
@ -13,7 +14,13 @@ function schedule(req, res, next) {
|
||||||
|
|
||||||
//Function for getting the page via http.
|
//Function for getting the page via http.
|
||||||
function get(url, callback) {
|
function get(url, callback) {
|
||||||
http.get(url, function (res) {
|
|
||||||
|
var options = url.parse(url);
|
||||||
|
options.socksPort = config().tor_port;
|
||||||
|
|
||||||
|
console.log(options);
|
||||||
|
|
||||||
|
http.get(options, function (res) {
|
||||||
var _download = '';
|
var _download = '';
|
||||||
|
|
||||||
res.on('data', function (data) {
|
res.on('data', function (data) {
|
||||||
|
|
12
spider.js
12
spider.js
|
@ -1,8 +1,9 @@
|
||||||
var http = require('http');
|
var http = require('socsk5-http-client');
|
||||||
var cheerio = require('cheerio');
|
var cheerio = require('cheerio');
|
||||||
var iconv = require('iconv-lite');
|
var iconv = require('iconv-lite');
|
||||||
var mongodb = require('mongodb').MongoClient;
|
var mongodb = require('mongodb').MongoClient;
|
||||||
var config = require('./configuration');
|
var config = require('./configuration');
|
||||||
|
var url = require('url');
|
||||||
|
|
||||||
var scheduletypes = [
|
var scheduletypes = [
|
||||||
'Klasrooster',
|
'Klasrooster',
|
||||||
|
@ -21,9 +22,10 @@ function get() {
|
||||||
|
|
||||||
(function (scheduletype) {
|
(function (scheduletype) {
|
||||||
|
|
||||||
var link = 'http://roosters5.gepro-osi.nl/roosters/rooster.php?school=' + school_id + '&type=' + scheduletype;
|
var options = url.parse('http://roosters5.gepro-osi.nl/roosters/rooster.php?school=' + school_id + '&type=' + scheduletype);
|
||||||
|
options.socksPort = config().tor_port;
|
||||||
|
|
||||||
http.get(link, function (res) {
|
http.get(options, function (res) {
|
||||||
|
|
||||||
var _download = {};
|
var _download = {};
|
||||||
_download.type = scheduletype;
|
_download.type = scheduletype;
|
||||||
|
@ -58,8 +60,10 @@ function rip(data) {
|
||||||
for(studentcategory of list) {
|
for(studentcategory of list) {
|
||||||
|
|
||||||
(function (studentcategory) {
|
(function (studentcategory) {
|
||||||
|
var options = url.parse('http://roosters5.gepro-osi.nl/roosters/rooster.php?school=' + school_id + '&type=' + data.type + '&afdeling=' + studentcategory);
|
||||||
|
options.socksPort = config().tor_port;
|
||||||
|
|
||||||
http.get('http://roosters5.gepro-osi.nl/roosters/rooster.php?school=' + school_id + '&type=' + data.type + '&afdeling=' + studentcategory, function (res) {
|
http.get(options, function (res) {
|
||||||
var _download = '';
|
var _download = '';
|
||||||
|
|
||||||
res.on('data', function (data) {
|
res.on('data', function (data) {
|
||||||
|
|
Loading…
Reference in a new issue