public
/
epg-grabber
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'async-functions'

This commit is contained in:
Aleksandr Statciuk 2021-08-23 14:19:05 +03:00
parent f2fb0a1b2e 9596939e68
commit 46795e6b2e
6 changed files with 244 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

62
README.md
View File

@ -32,12 +32,34 @@ module.exports = {
request: { // request options (details: https://github.com/axios/axios#request-config)
method: 'GET',
headers: {
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36 Edg/79.0.309.71',
},
timeout: 5000
timeout: 5000,
/**
* @param {object} date The 'dayjs' instance with the requested date
* @param {object} channel Data about the requested channel
*
* @return {string} The function should return headers for each request (optional)
*/
headers: function({ date, channel }) {
return {
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36 Edg/79.0.309.71'
}
},
/**
* @param {object} date The 'dayjs' instance with the requested date
* @param {object} channel Data about the requested channel
*
* @return {string} The function should return data for each request (optional)
*/
data: function({ date, channel }) {
return {
channels: [channel.site_id],
dateStart: date.format('YYYY-MM-DDT00:00:00-00:00'),
dateEnd: date.add(1, 'd').format('YYYY-MM-DDT00:00:00-00:00')
}
}
},
/**
@ -61,11 +83,12 @@ module.exports = {
},
/**
* @param {object} date The 'dayjs' instance with the requested date
* @param {string} content The response received after the request at the above url
*
* @return {array} The function should return an array of programs with their descriptions
*/
parser: function ({ content }) {
parser: function ({ date, content }) {
// content parsing...
@ -85,6 +108,33 @@ module.exports = {
}
```
Also each function can be asynchronous.
```js
module.exports = {
site: 'example.com',
output: 'example.com.guide.xml',
channels: 'example.com.channels.xml',
request: {
async headers() {
return { ... }
},
async data() {
return { ... }
}
},
async url() {
return '...'
},
async logo() {
return '...'
},
async parser() {
return [ ... ]
}
}
```
#### example.com.channels.xml
```xml

2
package.json
View File

@ -1,6 +1,6 @@
{
"name": "epg-grabber",
"version": "0.6.6",
"version": "0.7.0",
"description": "Node.js CLI tool for grabbing EPG from different sites",
"main": "src/index.js",
"preferGlobal": true,

40
src/index.js
View File

@ -13,11 +13,12 @@ program
.option('-d, --debug', 'Enable debug mode')
.parse(process.argv)
const options = program.opts()
const config = utils.loadConfig(options.config)
async function main() {
console.log('\r\nStarting...')
const options = program.opts()
const config = utils.loadConfig(options.config)
const channels = utils.parseChannels(config.channels)
const utcDate = utils.getUTCDate()
const dates = Array.from({ length: config.days }, (_, i) => utcDate.add(i, 'd'))
@ -34,35 +35,14 @@ async function main() {
for (let item of queue) {
if (options.debug) console.time(' Response Time')
await utils
.fetchData(item, config)
.then(response => {
if (options.debug) {
console.timeEnd(' Response Time')
console.time(' Parsing Time')
}
if (!item.channel.logo && config.logo) {
item.channel.logo = config.logo({
channel: item.channel,
content: response.data.toString(),
buffer: response.data
})
}
const parsed = utils.parsePrograms({ response, item, config }).map(program => {
program.lang = program.lang || item.channel.lang || undefined
return program
})
console.log(
` ${config.site} - ${item.channel.xmltv_id} - ${item.date.format('MMM D, YYYY')} (${
parsed.length
} programs)`
)
programs = programs.concat(parsed)
})
.then(() => {
.buildRequest(item, config)
.then(utils.fetchData)
.then(async response => {
if (options.debug) console.timeEnd(' Response Time')
if (options.debug) console.time(' Parsing Time')
const results = await utils.parseResponse(item, response, config)
if (options.debug) console.timeEnd(' Parsing Time')
programs = programs.concat(results)
})
.then(utils.sleep(config.delay))
.catch(err => {

126
src/utils.js
View File

@ -11,6 +11,8 @@ dayjs.extend(utc)
axiosCookieJarSupport(axios)
const utils = {}
const defaultUserAgent =
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36 Edg/79.0.309.71'
utils.loadConfig = function (file) {
if (!file) throw new Error('Path to [site].config.js is missing')
@ -39,10 +41,6 @@ utils.loadConfig = function (file) {
output: 'guide.xml',
request: {
method: 'GET',
headers: {
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36 Edg/79.0.309.71'
},
maxContentLength: 5 * 1024 * 1024,
timeout: 5000,
withCredentials: true,
@ -165,26 +163,6 @@ utils.convertToXMLTV = function ({ config, channels, programs }) {
return output
}
utils.parsePrograms = function ({ response, item, config }) {
const options = merge(item, config, {
content: response.data.toString(),
buffer: response.data
})
const programs = config.parser(options)
if (!Array.isArray(programs)) {
throw new Error('Parser should return an array')
}
return programs
.filter(i => i)
.map(p => {
p.channel = item.channel.xmltv_id
return p
})
}
utils.writeToFile = function (filename, data) {
const dir = path.resolve(path.dirname(filename))
if (!fs.existsSync(dir)) {
@ -194,17 +172,109 @@ utils.writeToFile = function (filename, data) {
fs.writeFileSync(path.resolve(filename), data)
}
utils.fetchData = function (item, config) {
utils.buildRequest = async function (item, config) {
const request = { ...config.request }
request.url = typeof config.url === 'function' ? config.url(item) : config.url
request.data =
typeof config.request.data === 'function' ? config.request.data(item) : config.request.data
const headers = await utils.getRequestHeaders(item, config)
request.headers = { 'User-Agent': defaultUserAgent, ...headers }
request.url = await utils.getRequestUrl(item, config)
request.data = await utils.getRequestData(item, config)
return request
}
utils.fetchData = function (request) {
return axios(request)
}
utils.getRequestHeaders = async function (item, config) {
if (typeof config.request.headers === 'function') {
const headers = config.request.headers(item)
if (this.isPromise(headers)) {
return await headers
}
return headers
}
return config.request.headers
}
utils.getRequestData = async function (item, config) {
if (typeof config.request.data === 'function') {
const data = config.request.data(item)
if (this.isPromise(data)) {
return await data
}
return data
}
return config.request.data
}
utils.getRequestUrl = async function (item, config) {
if (typeof config.url === 'function') {
const url = config.url(item)
if (this.isPromise(url)) {
return await url
}
return url
}
return config.url
}
utils.getUTCDate = function () {
return dayjs.utc()
}
utils.parseResponse = async (item, response, config) => {
const options = merge(item, config, {
content: response.data.toString(),
buffer: response.data
})
if (!item.channel.logo && config.logo) {
item.channel.logo = await utils.loadLogo(options, config)
}
const parsed = await utils.parsePrograms(options, config)
console.log(
` ${config.site} - ${item.channel.xmltv_id} - ${item.date.format('MMM D, YYYY')} (${
parsed.length
} programs)`
)
return parsed
}
utils.parsePrograms = async function (options, config) {
let programs = config.parser(options)
if (this.isPromise(programs)) {
programs = await programs
}
if (!Array.isArray(programs)) {
throw new Error('Parser should return an array')
}
const channel = options.channel
return programs
.filter(i => i)
.map(program => {
program.channel = channel.xmltv_id
program.lang = program.lang || channel.lang || undefined
return program
})
}
utils.loadLogo = async function (options, config) {
const logo = config.logo(options)
if (this.isPromise(logo)) {
return await logo
}
return logo
}
utils.isPromise = function (promise) {
return !!promise && typeof promise.then === 'function'
}
module.exports = utils

25
tests/input/async.config.js Normal file
View File

@ -0,0 +1,25 @@
module.exports = {
site: 'example.com',
channels: 'example.com.channels.xml',
url() {
return Promise.resolve('http://example.com/20210319/1tv.json')
},
request: {
method: 'POST',
headers() {
return Promise.resolve({
'Content-Type': 'application/json',
Cookie: 'abc=123'
})
},
data() {
return Promise.resolve({ accountID: '123' })
}
},
parser() {
return Promise.resolve([])
},
logo() {
return Promise.resolve('http://example.com/logos/1TV.png?x=шеллы&sid=777')
}
}

59
tests/utils.test.js
View File

@ -15,8 +15,6 @@ it('can load valid config.js', () => {
timeout: 5000,
headers: {
'Content-Type': 'application/json',
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36 Edg/79.0.309.71',
Cookie: 'abc=123'
}
})
@ -81,9 +79,23 @@ it('can escape url', () => {
)
})
it('can fetch data', () => {
const config = utils.loadConfig('./tests/input/example.com.config.js')
utils.fetchData({}, config).then(jest.fn).catch(jest.fn)
it('can fetch data', async () => {
const request = {
data: { accountID: '123' },
headers: {
'Content-Type': 'application/json',
Cookie: 'abc=123',
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36 Edg/79.0.309.71'
},
maxContentLength: 5242880,
method: 'POST',
responseType: 'arraybuffer',
timeout: 5000,
url: 'http://example.com/20210319/1tv.json',
withCredentials: true
}
utils.fetchData(request).then(jest.fn).catch(jest.fn)
expect(mockAxios).toHaveBeenCalledWith(
expect.objectContaining({
data: { accountID: '123' },
@ -101,3 +113,40 @@ it('can fetch data', () => {
})
)
})
it('can build request async', async () => {
const config = utils.loadConfig('./tests/input/async.config.js')
return utils.buildRequest({}, config).then(request => {
expect(request).toMatchObject({
data: { accountID: '123' },
headers: {
'Content-Type': 'application/json',
Cookie: 'abc=123',
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36 Edg/79.0.309.71'
},
maxContentLength: 5242880,
method: 'POST',
responseType: 'arraybuffer',
timeout: 5000,
url: 'http://example.com/20210319/1tv.json',
withCredentials: true
})
})
})
it('can load logo async', async () => {
const config = utils.loadConfig('./tests/input/async.config.js')
return utils.loadLogo({}, config).then(logo => {
expect(logo).toBe('http://example.com/logos/1TV.png?x=шеллы&sid=777')
})
})
it('can parse programs async', async () => {
const config = utils.loadConfig('./tests/input/async.config.js')
return utils
.parsePrograms({ channel: { xmltv_id: '1tv', lang: 'en' } }, config)
.then(programs => {
expect(programs.length).toBe(0)
})
})