Merge branch 'async-functions'

README.md

@@ -32,12 +32,34 @@ module.exports = {
   request: { // request options (details: https://github.com/axios/axios#request-config)
 
     method: 'GET',
-    headers: {
+    timeout: 5000,
-      'User-Agent':
-        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36 Edg/79.0.309.71',
-    },
-    timeout: 5000
 
+    /**
+     * @param {object} date The 'dayjs' instance with the requested date
+     * @param {object} channel Data about the requested channel
+     *
+     * @return {string} The function should return headers for each request (optional)
+     */
+    headers: function({ date, channel }) {
+      return {
+        'User-Agent':
+          'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36 Edg/79.0.309.71'
+      }
+    },
+
+    /**
+     * @param {object} date The 'dayjs' instance with the requested date
+     * @param {object} channel Data about the requested channel
+     *
+     * @return {string} The function should return data for each request (optional)
+     */
+    data: function({ date, channel }) {
+      return {
+        channels: [channel.site_id],
+        dateStart: date.format('YYYY-MM-DDT00:00:00-00:00'),
+        dateEnd: date.add(1, 'd').format('YYYY-MM-DDT00:00:00-00:00')
+      }
+    }
   },
 
   /**
@@ -61,11 +83,12 @@ module.exports = {
   },
 
   /**
+   * @param {object} date The 'dayjs' instance with the requested date
    * @param {string} content The response received after the request at the above url
    *
    * @return {array} The function should return an array of programs with their descriptions
    */
-  parser: function ({ content }) {
+  parser: function ({ date, content }) {
 
     // content parsing...
 
@@ -85,6 +108,33 @@ module.exports = {
 }
 ```
 
+Also each function can be asynchronous.
+
+```js
+module.exports = {
+  site: 'example.com',
+  output: 'example.com.guide.xml',
+  channels: 'example.com.channels.xml',
+  request: {
+    async headers() {
+      return { ... }
+    },
+    async data() {
+      return { ... }
+    }
+  },
+  async url() {
+    return '...'
+  },
+  async logo() {
+    return '...'
+  },
+  async parser() {
+    return [ ... ]
+  }
+}
+```
+
 #### example.com.channels.xml
 
 ```xml

package.json

@@ -1,6 +1,6 @@
 {
   "name": "epg-grabber",
-  "version": "0.6.6",
+  "version": "0.7.0",
   "description": "Node.js CLI tool for grabbing EPG from different sites",
   "main": "src/index.js",
   "preferGlobal": true,

src/index.js

@@ -13,11 +13,12 @@ program
   .option('-d, --debug', 'Enable debug mode')
   .parse(process.argv)
 
+const options = program.opts()
+const config = utils.loadConfig(options.config)
+
 async function main() {
   console.log('\r\nStarting...')
 
-  const options = program.opts()
-  const config = utils.loadConfig(options.config)
   const channels = utils.parseChannels(config.channels)
   const utcDate = utils.getUTCDate()
   const dates = Array.from({ length: config.days }, (_, i) => utcDate.add(i, 'd'))
@@ -34,35 +35,14 @@ async function main() {
   for (let item of queue) {
     if (options.debug) console.time('    Response Time')
     await utils
-      .fetchData(item, config)
+      .buildRequest(item, config)
-      .then(response => {
+      .then(utils.fetchData)
-        if (options.debug) {
+      .then(async response => {
-          console.timeEnd('    Response Time')
+        if (options.debug) console.timeEnd('    Response Time')
-          console.time('    Parsing Time')
+        if (options.debug) console.time('    Parsing Time')
-        }
+        const results = await utils.parseResponse(item, response, config)
-        if (!item.channel.logo && config.logo) {
-          item.channel.logo = config.logo({
-            channel: item.channel,
-            content: response.data.toString(),
-            buffer: response.data
-          })
-        }
-
-        const parsed = utils.parsePrograms({ response, item, config }).map(program => {
-          program.lang = program.lang || item.channel.lang || undefined
-          return program
-        })
-
-        console.log(
-          `  ${config.site} - ${item.channel.xmltv_id} - ${item.date.format('MMM D, YYYY')} (${
-            parsed.length
-          } programs)`
-        )
-
-        programs = programs.concat(parsed)
-      })
-      .then(() => {
         if (options.debug) console.timeEnd('    Parsing Time')
+        programs = programs.concat(results)
       })
       .then(utils.sleep(config.delay))
       .catch(err => {

src/utils.js

@@ -11,6 +11,8 @@ dayjs.extend(utc)
 axiosCookieJarSupport(axios)
 
 const utils = {}
+const defaultUserAgent =
+  'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36 Edg/79.0.309.71'
 
 utils.loadConfig = function (file) {
   if (!file) throw new Error('Path to [site].config.js is missing')
@@ -39,10 +41,6 @@ utils.loadConfig = function (file) {
     output: 'guide.xml',
     request: {
       method: 'GET',
-      headers: {
-        'User-Agent':
-          'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36 Edg/79.0.309.71'
-      },
       maxContentLength: 5 * 1024 * 1024,
       timeout: 5000,
       withCredentials: true,
@@ -165,26 +163,6 @@ utils.convertToXMLTV = function ({ config, channels, programs }) {
   return output
 }
 
-utils.parsePrograms = function ({ response, item, config }) {
-  const options = merge(item, config, {
-    content: response.data.toString(),
-    buffer: response.data
-  })
-
-  const programs = config.parser(options)
-
-  if (!Array.isArray(programs)) {
-    throw new Error('Parser should return an array')
-  }
-
-  return programs
-    .filter(i => i)
-    .map(p => {
-      p.channel = item.channel.xmltv_id
-      return p
-    })
-}
-
 utils.writeToFile = function (filename, data) {
   const dir = path.resolve(path.dirname(filename))
   if (!fs.existsSync(dir)) {
@@ -194,17 +172,109 @@ utils.writeToFile = function (filename, data) {
   fs.writeFileSync(path.resolve(filename), data)
 }
 
-utils.fetchData = function (item, config) {
+utils.buildRequest = async function (item, config) {
   const request = { ...config.request }
-  request.url = typeof config.url === 'function' ? config.url(item) : config.url
+  const headers = await utils.getRequestHeaders(item, config)
-  request.data =
+  request.headers = { 'User-Agent': defaultUserAgent, ...headers }
-    typeof config.request.data === 'function' ? config.request.data(item) : config.request.data
+  request.url = await utils.getRequestUrl(item, config)
+  request.data = await utils.getRequestData(item, config)
 
+  return request
+}
+
+utils.fetchData = function (request) {
   return axios(request)
 }
 
+utils.getRequestHeaders = async function (item, config) {
+  if (typeof config.request.headers === 'function') {
+    const headers = config.request.headers(item)
+    if (this.isPromise(headers)) {
+      return await headers
+    }
+    return headers
+  }
+  return config.request.headers
+}
+
+utils.getRequestData = async function (item, config) {
+  if (typeof config.request.data === 'function') {
+    const data = config.request.data(item)
+    if (this.isPromise(data)) {
+      return await data
+    }
+    return data
+  }
+  return config.request.data
+}
+
+utils.getRequestUrl = async function (item, config) {
+  if (typeof config.url === 'function') {
+    const url = config.url(item)
+    if (this.isPromise(url)) {
+      return await url
+    }
+    return url
+  }
+  return config.url
+}
+
 utils.getUTCDate = function () {
   return dayjs.utc()
 }
 
+utils.parseResponse = async (item, response, config) => {
+  const options = merge(item, config, {
+    content: response.data.toString(),
+    buffer: response.data
+  })
+
+  if (!item.channel.logo && config.logo) {
+    item.channel.logo = await utils.loadLogo(options, config)
+  }
+
+  const parsed = await utils.parsePrograms(options, config)
+
+  console.log(
+    `  ${config.site} - ${item.channel.xmltv_id} - ${item.date.format('MMM D, YYYY')} (${
+      parsed.length
+    } programs)`
+  )
+
+  return parsed
+}
+
+utils.parsePrograms = async function (options, config) {
+  let programs = config.parser(options)
+
+  if (this.isPromise(programs)) {
+    programs = await programs
+  }
+
+  if (!Array.isArray(programs)) {
+    throw new Error('Parser should return an array')
+  }
+
+  const channel = options.channel
+  return programs
+    .filter(i => i)
+    .map(program => {
+      program.channel = channel.xmltv_id
+      program.lang = program.lang || channel.lang || undefined
+      return program
+    })
+}
+
+utils.loadLogo = async function (options, config) {
+  const logo = config.logo(options)
+  if (this.isPromise(logo)) {
+    return await logo
+  }
+  return logo
+}
+
+utils.isPromise = function (promise) {
+  return !!promise && typeof promise.then === 'function'
+}
+
 module.exports = utils

tests/input/async.config.js

@@ -0,0 +1,25 @@
+module.exports = {
+  site: 'example.com',
+  channels: 'example.com.channels.xml',
+  url() {
+    return Promise.resolve('http://example.com/20210319/1tv.json')
+  },
+  request: {
+    method: 'POST',
+    headers() {
+      return Promise.resolve({
+        'Content-Type': 'application/json',
+        Cookie: 'abc=123'
+      })
+    },
+    data() {
+      return Promise.resolve({ accountID: '123' })
+    }
+  },
+  parser() {
+    return Promise.resolve([])
+  },
+  logo() {
+    return Promise.resolve('http://example.com/logos/1TV.png?x=шеллы&sid=777')
+  }
+}

tests/utils.test.js

@@ -15,8 +15,6 @@ it('can load valid config.js', () => {
     timeout: 5000,
     headers: {
       'Content-Type': 'application/json',
-      'User-Agent':
-        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36 Edg/79.0.309.71',
       Cookie: 'abc=123'
     }
   })
@@ -81,9 +79,23 @@ it('can escape url', () => {
   )
 })
 
-it('can fetch data', () => {
+it('can fetch data', async () => {
-  const config = utils.loadConfig('./tests/input/example.com.config.js')
+  const request = {
-  utils.fetchData({}, config).then(jest.fn).catch(jest.fn)
+    data: { accountID: '123' },
+    headers: {
+      'Content-Type': 'application/json',
+      Cookie: 'abc=123',
+      'User-Agent':
+        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36 Edg/79.0.309.71'
+    },
+    maxContentLength: 5242880,
+    method: 'POST',
+    responseType: 'arraybuffer',
+    timeout: 5000,
+    url: 'http://example.com/20210319/1tv.json',
+    withCredentials: true
+  }
+  utils.fetchData(request).then(jest.fn).catch(jest.fn)
   expect(mockAxios).toHaveBeenCalledWith(
     expect.objectContaining({
       data: { accountID: '123' },
@@ -101,3 +113,40 @@ it('can fetch data', () => {
     })
   )
 })
+
+it('can build request async', async () => {
+  const config = utils.loadConfig('./tests/input/async.config.js')
+  return utils.buildRequest({}, config).then(request => {
+    expect(request).toMatchObject({
+      data: { accountID: '123' },
+      headers: {
+        'Content-Type': 'application/json',
+        Cookie: 'abc=123',
+        'User-Agent':
+          'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36 Edg/79.0.309.71'
+      },
+      maxContentLength: 5242880,
+      method: 'POST',
+      responseType: 'arraybuffer',
+      timeout: 5000,
+      url: 'http://example.com/20210319/1tv.json',
+      withCredentials: true
+    })
+  })
+})
+
+it('can load logo async', async () => {
+  const config = utils.loadConfig('./tests/input/async.config.js')
+  return utils.loadLogo({}, config).then(logo => {
+    expect(logo).toBe('http://example.com/logos/1TV.png?x=шеллы&sid=777')
+  })
+})
+
+it('can parse programs async', async () => {
+  const config = utils.loadConfig('./tests/input/async.config.js')
+  return utils
+    .parsePrograms({ channel: { xmltv_id: '1tv', lang: 'en' } }, config)
+    .then(programs => {
+      expect(programs.length).toBe(0)
+    })
+})