38 lines
1.0 KiB
JavaScript
38 lines
1.0 KiB
JavaScript
const rq = require('request-promise')
|
|
const cheerio = require('cheerio')
|
|
const filters = {
|
|
breaks: require('./filter/breaks'),
|
|
spaces: require('./filter/spaces'),
|
|
tags: require('./filter/tags'),
|
|
trim: require('./filter/trim'),
|
|
}
|
|
|
|
const scraper = function(options, selector, filtersToApply, callbackSuccess, callbackError) {
|
|
filtersToApply = filtersToApply || {}
|
|
|
|
rq({
|
|
uri: options.url,
|
|
method: options.method || 'GET',
|
|
simple: !(options.acceptAllStatus || false)
|
|
})
|
|
.then(function(body) {
|
|
const $ = cheerio.load(body)
|
|
let value = $(selector).html()
|
|
|
|
for (let filter in filtersToApply) {
|
|
value = filters[filter](value, filtersToApply[filter])
|
|
}
|
|
|
|
if (callbackSuccess) {
|
|
callbackSuccess(value)
|
|
}
|
|
})
|
|
.catch(function(error) {
|
|
if (callbackError) {
|
|
callbackError(error)
|
|
}
|
|
})
|
|
}
|
|
|
|
module.exports = scraper
|