From 572ca9953c2edc321ea0c8ddc9cb4e4432520ede Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Wed, 18 Aug 2021 16:37:47 +0300 Subject: [PATCH 1/5] Update log.js --- scripts/helpers/log.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/helpers/log.js b/scripts/helpers/log.js index a2e8fe0a2..3120110b8 100644 --- a/scripts/helpers/log.js +++ b/scripts/helpers/log.js @@ -1,7 +1,8 @@ const log = {} -log.print = function (string) { - process.stdout.write(string) +log.print = function (message) { + if (typeof message === 'object') message = JSON.stringify(message, null, 2) + process.stdout.write(message) } log.start = function () { From 9d74ff30954505269e04a98e246ac2ae5f5d5396 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Wed, 18 Aug 2021 16:39:26 +0300 Subject: [PATCH 2/5] Update format.js --- scripts/format.js | 216 +++++++++++++++++++++++++++++----------------- 1 file changed, 135 insertions(+), 81 deletions(-) diff --git a/scripts/format.js b/scripts/format.js index e5d5d040f..1418fa578 100644 --- a/scripts/format.js +++ b/scripts/format.js @@ -7,99 +7,151 @@ const utils = require('./helpers/utils') const file = require('./helpers/file') const log = require('./helpers/log') +const ignoreStatus = ['Geo-blocked', 'Not 24/7'] + program .usage('[OPTIONS]...') - .option('-d, --debug', 'Enable debug mode') - .option('-s, --status', 'Update stream status') - .option('-r, --resolution', 'Detect stream resolution') + .option('--debug', 'Enable debug mode') + .option('-d, --delay ', 'Set delay for each request', parseNumber, 0) + .option('-t, --timeout ', 'Set timeout for each request', parseNumber, 5000) .option('-c, --country ', 'Comma-separated list of country codes', '') .option('-e, --exclude ', 'Comma-separated list of country codes to be excluded', '') - .option('--delay ', 'Set delay for each request', 0) - .option('--timeout ', 'Set timeout for each request', 5000) .parse(process.argv) -let bar -const ignoreStatus = ['Geo-blocked', 'Not 24/7'] const config = program.opts() const checker = new IPTVChecker({ timeout: config.timeout }) +let buffer, origins async function main() { log.start() - if (config.debug) log.print(`Debug mode enabled\n`) - if (config.status) log.print(`Status check enabled\n`) - if (config.resolution) log.print(`Resolution detection enabled\n`) - let playlists = parser.parseIndex().filter(i => i.url !== 'channels/unsorted.m3u') playlists = utils.filterPlaylists(playlists, config.country, config.exclude) if (!playlists.length) log.print(`No playlist is selected\n`) for (const playlist of playlists) { - await parser - .parsePlaylist(playlist.url) - .then(updatePlaylist) - .then(playlist => { - if (file.read(playlist.url) !== playlist.toString()) { - log.print(`File '${playlist.url}' has been updated\n`) - playlist.updated = true - } - - playlist.save() - }) + await parser.parsePlaylist(playlist.url).then(updatePlaylist).then(savePlaylist) } log.finish() } -async function updatePlaylist(playlist) { - if (!config.debug) { - bar = new ProgressBar(`Processing '${playlist.url}': [:bar] :current/:total (:percent) `, { - total: playlist.channels.length - }) - } else { - log.print(`Processing '${playlist.url}'...\n`) +function savePlaylist(playlist) { + if (file.read(playlist.url) !== playlist.toString()) { + log.print(`File '${playlist.url}' has been updated\n`) + playlist.updated = true } - for (const channel of playlist.channels) { - addMissingData(channel, playlist) - updateGroupTitle(channel) + playlist.save() +} + +async function updatePlaylist(playlist) { + const bar = new ProgressBar(`Processing '${playlist.url}': [:bar] :current/:total (:percent) `, { + total: playlist.channels.length + }) + + buffer = {} + origins = {} + for (const [i, channel] of playlist.channels.entries()) { + bar.tick() + updateDescription(channel, playlist) normalizeUrl(channel) - const checkOnline = config.status || config.resolution - const skipChannel = - channel.status && - ignoreStatus.map(i => i.toLowerCase()).includes(channel.status.toLowerCase()) - if (checkOnline && !skipChannel) { - await checker - .checkStream(channel.data) - .then(result => { - const status = parseStatus(result.status) - - if (config.status) { - updateStatus(channel, status) - } - - if (config.resolution && status === 'online') { - updateResolution(channel, result.status.metadata) - } - - if (config.debug && status === 'offline') { - log.print(` ERR: ${channel.url} (${result.status.reason})\n`) - } - }) - .catch(err => { - if (config.debug) log.print(` ERR: ${channel.url} (${err.message})\n`) - }) - - await utils.sleep(config.delay) + if (ignoreStatus.includes(channel.status)) { + continue } - if (!config.debug) bar.tick() + + await checker + .checkStream(channel.data) + .then(parseResult) + .then(result => { + updateStatus(channel, result.status) + if (result.status === 'online') { + buffer[i] = result + updateOrigins(channel, result.requests) + updateResolution(channel, result.resolution) + } else { + buffer[i] = null + if (config.debug) { + if (!bar.complete) log.print(`\n`) + log.print(` INFO: ${channel.url} (${result.error})\n`) + } + } + }) + .catch(err => { + buffer[i] = null + if (config.debug) { + if (!bar.complete) log.print(`\n`) + log.print(` ERR: ${channel.data.url} (${err.message})\n`) + } + }) + } + + for (const [i, channel] of playlist.channels.entries()) { + if (!buffer[i]) continue + const { requests } = buffer[i] + updateUrl(channel, requests) } return playlist } +function updateOrigins(channel, requests) { + if (!requests) return + const origin = new URL(channel.url) + const target = new URL(requests[0]) + const type = origin.host === target.host ? 'origin' : 'redirect' + requests.forEach(url => { + const key = utils.removeProtocol(url) + if (!origins[key] && type === 'origin') { + origins[key] = channel.url + } + }) +} + +function updateStatus(channel, status) { + switch (status) { + case 'online': + channel.status = null + break + case 'offline': + channel.status = 'Offline' + break + } +} + +function updateResolution(channel, resolution) { + if (!channel.resolution.height && resolution) { + channel.resolution = resolution + } +} + +function updateUrl(channel, requests) { + for (const request of requests) { + let key = utils.removeProtocol(channel.url) + if (origins[key]) { + channel.updateUrl(origins[key]) + break + } + + key = utils.removeProtocol(request) + if (origins[key]) { + channel.updateUrl(origins[key]) + break + } + } +} + +function parseResult(result) { + return { + status: parseStatus(result.status), + resolution: result.status.ok ? parseResolution(result.status.metadata.streams) : null, + requests: result.status.ok ? parseRequests(result.status.metadata.requests) : [], + error: !result.status.ok ? result.status.reason : null + } +} + function parseStatus(status) { if (status.ok) { return 'online' @@ -114,18 +166,28 @@ function parseStatus(status) { } } -function updateStatus(channel, status) { - switch (status) { - case 'online': - channel.status = null - break - case 'offline': - channel.status = 'Offline' - break - } +function parseResolution(streams) { + const resolution = streams + .filter(stream => stream.codec_type === 'video') + .reduce( + (acc, curr) => { + if (curr.height > acc.height) return { width: curr.width, height: curr.height } + return acc + }, + { width: 0, height: 0 } + ) + + return resolution.width > 0 && resolution.height > 0 ? resolution : null } -function addMissingData(channel, playlist) { +function parseRequests(requests) { + requests = requests.map(r => r.url) + requests.shift() + + return requests +} + +function updateDescription(channel, playlist) { const code = playlist.country.code // tvg-name if (!channel.tvg.name && channel.name) { @@ -142,9 +204,7 @@ function addMissingData(channel, playlist) { channel.countries = name ? [{ code, name }] : [] channel.tvg.country = channel.countries.map(c => c.code.toUpperCase()).join(';') } -} - -function updateGroupTitle(channel) { + // group-title channel.group.title = channel.category } @@ -154,14 +214,8 @@ function normalizeUrl(channel) { channel.updateUrl(decoded) } -function updateResolution(channel, metadata) { - const streams = metadata ? metadata.streams.filter(stream => stream.codec_type === 'video') : [] - if (!channel.resolution.height && streams.length) { - channel.resolution = streams.reduce((acc, curr) => { - if (curr.height > acc.height) return { width: curr.width, height: curr.height } - return acc - }) - } +function parseNumber(str) { + return parseInt(str) } main() From f4b2b6cef7da28e570e4ea6ed5df0b8c27918e0e Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Wed, 18 Aug 2021 16:40:27 +0300 Subject: [PATCH 3/5] Upgrade iptv-checker package --- package-lock.json | 14 +++++++------- package.json | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/package-lock.json b/package-lock.json index 99f5912b4..d377bf1b2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,7 +8,7 @@ "dependencies": { "@freearhey/iso-639-3": "^1.0.0", "commander": "^7.0.0", - "iptv-checker": "^0.20.2", + "iptv-checker": "^0.21.0", "iptv-playlist-parser": "^0.5.4", "m3u-linter": "^0.2.1", "markdown-include": "^0.4.3", @@ -1904,9 +1904,9 @@ "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" }, "node_modules/iptv-checker": { - "version": "0.20.2", - "resolved": "https://registry.npmjs.org/iptv-checker/-/iptv-checker-0.20.2.tgz", - "integrity": "sha512-3lzN3q1l3TApHvsP6Rt4ALrs8PE+Zb2gmuykwPzQNtlfazIUR73Yeq/ypwpqRwYJxSq5WMoEdQHwV4KcR7y5Ig==", + "version": "0.21.0", + "resolved": "https://registry.npmjs.org/iptv-checker/-/iptv-checker-0.21.0.tgz", + "integrity": "sha512-qWqq8mLl+qRgV0/fmBpZm8/Cf7FwSnIA1gt9PFlYbzilRDFtPoTRDm0eP9fCWHVwwZZsImE1aMLx8qkikL5Aog==", "dependencies": { "axios": "^0.21.1", "colors": "^1.4.0", @@ -5252,9 +5252,9 @@ "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" }, "iptv-checker": { - "version": "0.20.2", - "resolved": "https://registry.npmjs.org/iptv-checker/-/iptv-checker-0.20.2.tgz", - "integrity": "sha512-3lzN3q1l3TApHvsP6Rt4ALrs8PE+Zb2gmuykwPzQNtlfazIUR73Yeq/ypwpqRwYJxSq5WMoEdQHwV4KcR7y5Ig==", + "version": "0.21.0", + "resolved": "https://registry.npmjs.org/iptv-checker/-/iptv-checker-0.21.0.tgz", + "integrity": "sha512-qWqq8mLl+qRgV0/fmBpZm8/Cf7FwSnIA1gt9PFlYbzilRDFtPoTRDm0eP9fCWHVwwZZsImE1aMLx8qkikL5Aog==", "requires": { "axios": "^0.21.1", "colors": "^1.4.0", diff --git a/package.json b/package.json index 7688a6022..dff30960a 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,7 @@ "dependencies": { "@freearhey/iso-639-3": "^1.0.0", "commander": "^7.0.0", - "iptv-checker": "^0.20.2", + "iptv-checker": "^0.21.0", "iptv-playlist-parser": "^0.5.4", "m3u-linter": "^0.2.1", "markdown-include": "^0.4.3", From 044a7cc01c8b2a8153ebe360f6a7f0da1184702d Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Wed, 18 Aug 2021 16:46:06 +0300 Subject: [PATCH 4/5] Update auto-update.yml --- .github/workflows/auto-update.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/auto-update.yml b/.github/workflows/auto-update.yml index 9757021ef..02c34fa30 100644 --- a/.github/workflows/auto-update.yml +++ b/.github/workflows/auto-update.yml @@ -197,7 +197,7 @@ jobs: - name: Install Dependencies run: npm install - name: Format Playlists - run: node scripts/format.js --country=${{ matrix.country }} --status --resolution --debug + run: node scripts/format.js --country=${{ matrix.country }} --debug - name: Upload Artifact uses: actions/upload-artifact@v2 with: From c7af11ad8a316fc4610da23867a0d8e3dfa69d02 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Wed, 18 Aug 2021 17:21:05 +0300 Subject: [PATCH 5/5] Remove progress bar Not working inside GitHub workflow --- scripts/format.js | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/scripts/format.js b/scripts/format.js index 1418fa578..f82a0f754 100644 --- a/scripts/format.js +++ b/scripts/format.js @@ -1,7 +1,6 @@ const IPTVChecker = require('iptv-checker') const normalize = require('normalize-url') const { program } = require('commander') -const ProgressBar = require('progress') const parser = require('./helpers/parser') const utils = require('./helpers/utils') const file = require('./helpers/file') @@ -47,14 +46,13 @@ function savePlaylist(playlist) { } async function updatePlaylist(playlist) { - const bar = new ProgressBar(`Processing '${playlist.url}': [:bar] :current/:total (:percent) `, { - total: playlist.channels.length - }) + const total = playlist.channels.length + log.print(`Processing '${playlist.url}'...\n`) buffer = {} origins = {} for (const [i, channel] of playlist.channels.entries()) { - bar.tick() + const curr = i + 1 updateDescription(channel, playlist) normalizeUrl(channel) @@ -74,7 +72,6 @@ async function updatePlaylist(playlist) { } else { buffer[i] = null if (config.debug) { - if (!bar.complete) log.print(`\n`) log.print(` INFO: ${channel.url} (${result.error})\n`) } } @@ -82,7 +79,6 @@ async function updatePlaylist(playlist) { .catch(err => { buffer[i] = null if (config.debug) { - if (!bar.complete) log.print(`\n`) log.print(` ERR: ${channel.data.url} (${err.message})\n`) } })