From 92ae51171def09e54a7ba1252c0eda7411f64b5a Mon Sep 17 00:00:00 2001 From: freearhey <7253922+freearhey@users.noreply.github.com> Date: Sat, 7 Oct 2023 07:19:39 +0300 Subject: [PATCH] Rewrite api:load script --- .github/workflows/check.yml | 8 +- .github/workflows/format.yml | 5 - .github/workflows/update.yml | 2 - package-lock.json | 233 ++++++++++++++++++++++++++++++++--- scripts/commands/api/load.sh | 10 -- scripts/commands/api/load.ts | 21 ++++ scripts/core/apiClient.ts | 59 +++++++++ scripts/core/index.ts | 1 + yarn.lock | 90 ++++++++++++-- 9 files changed, 379 insertions(+), 50 deletions(-) delete mode 100755 scripts/commands/api/load.sh create mode 100644 scripts/commands/api/load.ts create mode 100644 scripts/core/apiClient.ts diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 19695343d..de727b673 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -20,14 +20,8 @@ jobs: - uses: actions/setup-node@v3 if: ${{ !env.ACT && steps.files.outputs.any_changed == 'true' }} with: - node-version: 16 + node-version: 18 cache: 'npm' - - name: download data from api - if: steps.files.outputs.any_changed == 'true' - run: | - mkdir -p temp/data - curl -L -o temp/data/blocklist.json https://iptv-org.github.io/api/blocklist.json - curl -L -o temp/data/channels.json https://iptv-org.github.io/api/channels.json - name: install dependencies if: steps.files.outputs.any_changed == 'true' run: npm install diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml index 444b524f1..d828c3ff9 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format.yml @@ -31,11 +31,6 @@ jobs: id: files with: files: streams/*.m3u - - name: download data from api - run: | - mkdir -p temp/data - curl -L -o temp/data/blocklist.json https://iptv-org.github.io/api/blocklist.json - curl -L -o temp/data/channels.json https://iptv-org.github.io/api/channels.json - uses: actions/setup-node@v3 if: ${{ !env.ACT }} with: diff --git a/.github/workflows/update.yml b/.github/workflows/update.yml index 781bcdaa6..a69ec8ac1 100644 --- a/.github/workflows/update.yml +++ b/.github/workflows/update.yml @@ -29,8 +29,6 @@ jobs: cache: 'npm' - name: install dependencies run: npm install - - name: load api data - run: npm run api:load - name: update internal playlists run: npm run playlist:update --silent >> $GITHUB_OUTPUT id: playlist-update diff --git a/package-lock.json b/package-lock.json index 70ecbb582..7051cdd20 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5,6 +5,7 @@ "packages": { "": { "name": "iptv", + "hasInstallScript": true, "license": "MIT", "dependencies": { "@freearhey/core": "^0.2.1", @@ -12,16 +13,20 @@ "@octokit/plugin-paginate-rest": "^7.1.2", "@octokit/plugin-rest-endpoint-methods": "^7.1.3", "@octokit/types": "^11.1.0", + "@types/cli-progress": "^3.11.3", "@types/jest": "^29.5.4", - "@types/jest-expect-message": "^1.1.0", "@types/lodash": "^4.14.198", + "@types/numeral": "^2.0.3", + "axios": "^1.5.1", "chalk": "^4.1.2", + "cli-progress": "^3.12.0", "commander": "^8.3.0", "iptv-playlist-parser": "^0.12.3", "jest-expect-message": "^1.1.3", "lodash": "^4.17.21", "m3u-linter": "^0.3.0", "markdown-include": "^0.4.3", + "numeral": "^2.0.6", "transliteration": "^2.3.5", "ts-jest": "^29.1.1" }, @@ -1633,6 +1638,14 @@ "@babel/types": "^7.20.7" } }, + "node_modules/@types/cli-progress": { + "version": "3.11.3", + "resolved": "https://registry.npmjs.org/@types/cli-progress/-/cli-progress-3.11.3.tgz", + "integrity": "sha512-/+C9xAdVtc+g5yHHkGBThgAA8rYpi5B+2ve3wLtybYj0JHEBs57ivR4x/zGfSsplRnV+psE91Nfin1soNKqz5Q==", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/fs-extra": { "version": "11.0.2", "resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-11.0.2.tgz", @@ -1681,15 +1694,6 @@ "pretty-format": "^29.0.0" } }, - "node_modules/@types/jest-expect-message": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@types/jest-expect-message/-/jest-expect-message-1.1.0.tgz", - "integrity": "sha512-PbmZ6pTBpJTzj7KdGP8l8qTyf6RA5BVWg0C51XjCb0GuifXVcu3jAR6+U/W47nmE5S3tLPPEo70UcuhSlSS1hg==", - "deprecated": "This is a stub types definition. jest-expect-message provides its own type definitions, so you do not need this installed.", - "dependencies": { - "jest-expect-message": "*" - } - }, "node_modules/@types/json-schema": { "version": "7.0.13", "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.13.tgz", @@ -1719,6 +1723,11 @@ "resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.18.tgz", "integrity": "sha512-eKj4f/BsN/qcculZiRSujogjvp5O/k4lOW5m35NopjZM/QwLOR075a8pJW5hD+Rtdm2DaCVPENS6KtSQnUD6BA==" }, + "node_modules/@types/numeral": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@types/numeral/-/numeral-2.0.3.tgz", + "integrity": "sha512-37VDYvFbx/I+Ve1kwcDUq5+qvgNl+uy9YUZhBMJ6YWHBtr2+d6CAgdn4R81NqBlZbTpvLxJOTUhwbki7+90+qQ==" + }, "node_modules/@types/semver": { "version": "7.5.2", "resolved": "https://registry.npmjs.org/@types/semver/-/semver-7.5.2.tgz", @@ -2130,6 +2139,21 @@ "sprintf-js": "~1.0.2" } }, + "node_modules/asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" + }, + "node_modules/axios": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.5.1.tgz", + "integrity": "sha512-Q28iYCWzNHjAm+yEAot5QaAMxhMghWLFVf7rRdwhUI+c2jix2DUXjAHXVi+s1ibs3mjPO/cCgbA++3BjD0vP/A==", + "dependencies": { + "follow-redirects": "^1.15.0", + "form-data": "^4.0.0", + "proxy-from-env": "^1.1.0" + } + }, "node_modules/babel-jest": { "version": "29.6.4", "resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-29.6.4.tgz", @@ -2397,6 +2421,17 @@ "integrity": "sha512-0TNiGstbQmCFwt4akjjBg5pLRTSyj/PkWQ1ZoO2zntmg9yLqSRxwEa4iCfQLGjqhiqBfOJa7W/E8wfGrTDmlZQ==", "peer": true }, + "node_modules/cli-progress": { + "version": "3.12.0", + "resolved": "https://registry.npmjs.org/cli-progress/-/cli-progress-3.12.0.tgz", + "integrity": "sha512-tRkV3HJ1ASwm19THiiLIXLO7Im7wlTuKnvkYaTkyoAPefqjNg7W7DHKUlGRxy9vxDvbyCYQkQozvptuMkGCg8A==", + "dependencies": { + "string-width": "^4.2.3" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/co": { "version": "4.6.0", "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz", @@ -2429,6 +2464,17 @@ "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" }, + "node_modules/combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "dependencies": { + "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, "node_modules/commander": { "version": "8.3.0", "resolved": "https://registry.npmjs.org/commander/-/commander-8.3.0.tgz", @@ -2516,6 +2562,14 @@ "node": ">=0.10.0" } }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "engines": { + "node": ">=0.4.0" + } + }, "node_modules/deprecation": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/deprecation/-/deprecation-2.3.1.tgz", @@ -3128,6 +3182,25 @@ "integrity": "sha512-36yxDn5H7OFZQla0/jFJmbIKTdZAQHngCedGxiMmpNfEZM0sdEeT+WczLQrjK6D7o2aiyLYDnkw0R3JK0Qv1RQ==", "dev": true }, + "node_modules/follow-redirects": { + "version": "1.15.3", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.3.tgz", + "integrity": "sha512-1VzOtuEM8pC9SFU1E+8KfTjZyMztRsgEfwQl44z8A25uy13jSzTj6dyK2Df52iV0vgHCfBwLhDWevLn95w5v6Q==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, "node_modules/foreground-child": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.1.1.tgz", @@ -3154,6 +3227,19 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/form-data": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", + "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/fs-extra": { "version": "11.1.1", "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.1.1.tgz", @@ -4701,6 +4787,25 @@ "node": ">=8.6" } }, + "node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, "node_modules/mimic-fn": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", @@ -4823,6 +4928,14 @@ "node": ">=8" } }, + "node_modules/numeral": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/numeral/-/numeral-2.0.6.tgz", + "integrity": "sha512-qaKRmtYPZ5qdw4jWJD6bxEf1FJEqllJrwxCLIm0sQU/A7v2/czigzOb+C2uSiFsa9lBUzeH7M1oK+Q+OLxL3kA==", + "engines": { + "node": "*" + } + }, "node_modules/object-treeify": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/object-treeify/-/object-treeify-2.1.1.tgz", @@ -5164,6 +5277,11 @@ "node": ">= 6" } }, + "node_modules/proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" + }, "node_modules/punycode": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.0.tgz", @@ -7351,6 +7469,14 @@ "@babel/types": "^7.20.7" } }, + "@types/cli-progress": { + "version": "3.11.3", + "resolved": "https://registry.npmjs.org/@types/cli-progress/-/cli-progress-3.11.3.tgz", + "integrity": "sha512-/+C9xAdVtc+g5yHHkGBThgAA8rYpi5B+2ve3wLtybYj0JHEBs57ivR4x/zGfSsplRnV+psE91Nfin1soNKqz5Q==", + "requires": { + "@types/node": "*" + } + }, "@types/fs-extra": { "version": "11.0.2", "resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-11.0.2.tgz", @@ -7399,14 +7525,6 @@ "pretty-format": "^29.0.0" } }, - "@types/jest-expect-message": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@types/jest-expect-message/-/jest-expect-message-1.1.0.tgz", - "integrity": "sha512-PbmZ6pTBpJTzj7KdGP8l8qTyf6RA5BVWg0C51XjCb0GuifXVcu3jAR6+U/W47nmE5S3tLPPEo70UcuhSlSS1hg==", - "requires": { - "jest-expect-message": "*" - } - }, "@types/json-schema": { "version": "7.0.13", "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.13.tgz", @@ -7436,6 +7554,11 @@ "resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.18.tgz", "integrity": "sha512-eKj4f/BsN/qcculZiRSujogjvp5O/k4lOW5m35NopjZM/QwLOR075a8pJW5hD+Rtdm2DaCVPENS6KtSQnUD6BA==" }, + "@types/numeral": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@types/numeral/-/numeral-2.0.3.tgz", + "integrity": "sha512-37VDYvFbx/I+Ve1kwcDUq5+qvgNl+uy9YUZhBMJ6YWHBtr2+d6CAgdn4R81NqBlZbTpvLxJOTUhwbki7+90+qQ==" + }, "@types/semver": { "version": "7.5.2", "resolved": "https://registry.npmjs.org/@types/semver/-/semver-7.5.2.tgz", @@ -7701,6 +7824,21 @@ "sprintf-js": "~1.0.2" } }, + "asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" + }, + "axios": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.5.1.tgz", + "integrity": "sha512-Q28iYCWzNHjAm+yEAot5QaAMxhMghWLFVf7rRdwhUI+c2jix2DUXjAHXVi+s1ibs3mjPO/cCgbA++3BjD0vP/A==", + "requires": { + "follow-redirects": "^1.15.0", + "form-data": "^4.0.0", + "proxy-from-env": "^1.1.0" + } + }, "babel-jest": { "version": "29.6.4", "resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-29.6.4.tgz", @@ -7892,6 +8030,14 @@ "integrity": "sha512-0TNiGstbQmCFwt4akjjBg5pLRTSyj/PkWQ1ZoO2zntmg9yLqSRxwEa4iCfQLGjqhiqBfOJa7W/E8wfGrTDmlZQ==", "peer": true }, + "cli-progress": { + "version": "3.12.0", + "resolved": "https://registry.npmjs.org/cli-progress/-/cli-progress-3.12.0.tgz", + "integrity": "sha512-tRkV3HJ1ASwm19THiiLIXLO7Im7wlTuKnvkYaTkyoAPefqjNg7W7DHKUlGRxy9vxDvbyCYQkQozvptuMkGCg8A==", + "requires": { + "string-width": "^4.2.3" + } + }, "co": { "version": "4.6.0", "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz", @@ -7917,6 +8063,14 @@ "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" }, + "combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "requires": { + "delayed-stream": "~1.0.0" + } + }, "commander": { "version": "8.3.0", "resolved": "https://registry.npmjs.org/commander/-/commander-8.3.0.tgz", @@ -7980,6 +8134,11 @@ "integrity": "sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==", "peer": true }, + "delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==" + }, "deprecation": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/deprecation/-/deprecation-2.3.1.tgz", @@ -8434,6 +8593,11 @@ "integrity": "sha512-36yxDn5H7OFZQla0/jFJmbIKTdZAQHngCedGxiMmpNfEZM0sdEeT+WczLQrjK6D7o2aiyLYDnkw0R3JK0Qv1RQ==", "dev": true }, + "follow-redirects": { + "version": "1.15.3", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.3.tgz", + "integrity": "sha512-1VzOtuEM8pC9SFU1E+8KfTjZyMztRsgEfwQl44z8A25uy13jSzTj6dyK2Df52iV0vgHCfBwLhDWevLn95w5v6Q==" + }, "foreground-child": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.1.1.tgz", @@ -8450,6 +8614,16 @@ } } }, + "form-data": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", + "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", + "requires": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + } + }, "fs-extra": { "version": "11.1.1", "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.1.1.tgz", @@ -9619,6 +9793,19 @@ "picomatch": "^2.3.1" } }, + "mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==" + }, + "mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "requires": { + "mime-db": "1.52.0" + } + }, "mimic-fn": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", @@ -9714,6 +9901,11 @@ "path-key": "^3.0.0" } }, + "numeral": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/numeral/-/numeral-2.0.6.tgz", + "integrity": "sha512-qaKRmtYPZ5qdw4jWJD6bxEf1FJEqllJrwxCLIm0sQU/A7v2/czigzOb+C2uSiFsa9lBUzeH7M1oK+Q+OLxL3kA==" + }, "object-treeify": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/object-treeify/-/object-treeify-2.1.1.tgz", @@ -9956,6 +10148,11 @@ "sisteransi": "^1.0.5" } }, + "proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" + }, "punycode": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.0.tgz", diff --git a/scripts/commands/api/load.sh b/scripts/commands/api/load.sh deleted file mode 100755 index 7dea082ea..000000000 --- a/scripts/commands/api/load.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -mkdir -p temp/data -curl -L -o temp/data/blocklist.json https://iptv-org.github.io/api/blocklist.json -curl -L -o temp/data/categories.json https://iptv-org.github.io/api/categories.json -curl -L -o temp/data/channels.json https://iptv-org.github.io/api/channels.json -curl -L -o temp/data/countries.json https://iptv-org.github.io/api/countries.json -curl -L -o temp/data/languages.json https://iptv-org.github.io/api/languages.json -curl -L -o temp/data/regions.json https://iptv-org.github.io/api/regions.json -curl -L -o temp/data/subdivisions.json https://iptv-org.github.io/api/subdivisions.json \ No newline at end of file diff --git a/scripts/commands/api/load.ts b/scripts/commands/api/load.ts new file mode 100644 index 000000000..68e6e18a4 --- /dev/null +++ b/scripts/commands/api/load.ts @@ -0,0 +1,21 @@ +import { Logger } from '@freearhey/core' +import { ApiClient } from '../../core' + +async function main() { + const logger = new Logger() + const client = new ApiClient({ logger }) + + const requests = [ + client.download('blocklist.json'), + client.download('categories.json'), + client.download('channels.json'), + client.download('countries.json'), + client.download('languages.json'), + client.download('regions.json'), + client.download('subdivisions.json') + ] + + await Promise.all(requests) +} + +main() diff --git a/scripts/core/apiClient.ts b/scripts/core/apiClient.ts new file mode 100644 index 000000000..66fa28a87 --- /dev/null +++ b/scripts/core/apiClient.ts @@ -0,0 +1,59 @@ +import { Logger, Storage } from '@freearhey/core' +import axios, { AxiosInstance, AxiosResponse, AxiosProgressEvent } from 'axios' +import cliProgress, { MultiBar } from 'cli-progress' +import numeral from 'numeral' + +export class ApiClient { + progressBar: MultiBar + client: AxiosInstance + storage: Storage + logger: Logger + + constructor({ logger }: { logger: Logger }) { + this.logger = logger + this.client = axios.create({ + responseType: 'stream' + }) + this.storage = new Storage() + this.progressBar = new cliProgress.MultiBar({ + stopOnComplete: true, + hideCursor: true, + forceRedraw: true, + barsize: 36, + format(options, params, payload) { + const filename = payload.filename.padEnd(18, ' ') + const barsize = options.barsize || 40 + const percent = (params.progress * 100).toFixed(2) + const speed = payload.speed ? numeral(payload.speed).format('0.0 b') + '/s' : 'N/A' + const total = numeral(params.total).format('0.0 b') + const completeSize = Math.round(params.progress * barsize) + const incompleteSize = barsize - completeSize + const bar = + options.barCompleteString && options.barIncompleteString + ? options.barCompleteString.substr(0, completeSize) + + options.barGlue + + options.barIncompleteString.substr(0, incompleteSize) + : '-'.repeat(barsize) + + return `${filename} [${bar}] ${percent}% | ETA: ${params.eta}s | ${total} | ${speed}` + } + }) + } + + async download(filename: string) { + const stream = await this.storage.createStream(`/temp/data/${filename}`) + + const bar = this.progressBar.create(0, 0, { filename }) + + this.client + .get(`https://iptv-org.github.io/api/${filename}`, { + onDownloadProgress({ total, loaded, rate }: AxiosProgressEvent) { + if (total) bar.setTotal(total) + bar.update(loaded, { speed: rate }) + } + }) + .then((response: AxiosResponse) => { + response.data.pipe(stream) + }) + } +} diff --git a/scripts/core/index.ts b/scripts/core/index.ts index a1419eb00..7fd792daf 100644 --- a/scripts/core/index.ts +++ b/scripts/core/index.ts @@ -5,3 +5,4 @@ export * from './markdown' export * from './issueLoader' export * from './issueParser' export * from './htmlTable' +export * from './apiClient' diff --git a/yarn.lock b/yarn.lock index dd2ec4eec..922ef98bb 100644 --- a/yarn.lock +++ b/yarn.lock @@ -837,6 +837,13 @@ dependencies: "@babel/types" "^7.20.7" +"@types/cli-progress@^3.11.3": + version "3.11.3" + resolved "https://registry.npmjs.org/@types/cli-progress/-/cli-progress-3.11.3.tgz" + integrity sha512-/+C9xAdVtc+g5yHHkGBThgAA8rYpi5B+2ve3wLtybYj0JHEBs57ivR4x/zGfSsplRnV+psE91Nfin1soNKqz5Q== + dependencies: + "@types/node" "*" + "@types/fs-extra@^11.0.2": version "11.0.2" resolved "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-11.0.2.tgz" @@ -871,13 +878,6 @@ dependencies: "@types/istanbul-lib-report" "*" -"@types/jest-expect-message@^1.1.0": - version "1.1.0" - resolved "https://registry.npmjs.org/@types/jest-expect-message/-/jest-expect-message-1.1.0.tgz" - integrity sha512-PbmZ6pTBpJTzj7KdGP8l8qTyf6RA5BVWg0C51XjCb0GuifXVcu3jAR6+U/W47nmE5S3tLPPEo70UcuhSlSS1hg== - dependencies: - jest-expect-message "*" - "@types/jest@^29.5.4": version "29.5.4" resolved "https://registry.npmjs.org/@types/jest/-/jest-29.5.4.tgz" @@ -913,6 +913,11 @@ resolved "https://registry.npmjs.org/@types/node/-/node-17.0.18.tgz" integrity sha512-eKj4f/BsN/qcculZiRSujogjvp5O/k4lOW5m35NopjZM/QwLOR075a8pJW5hD+Rtdm2DaCVPENS6KtSQnUD6BA== +"@types/numeral@^2.0.3": + version "2.0.3" + resolved "https://registry.npmjs.org/@types/numeral/-/numeral-2.0.3.tgz" + integrity sha512-37VDYvFbx/I+Ve1kwcDUq5+qvgNl+uy9YUZhBMJ6YWHBtr2+d6CAgdn4R81NqBlZbTpvLxJOTUhwbki7+90+qQ== + "@types/semver@^7.5.0": version "7.5.2" resolved "https://registry.npmjs.org/@types/semver/-/semver-7.5.2.tgz" @@ -1116,6 +1121,20 @@ array-union@^2.1.0: resolved "https://registry.npmjs.org/array-union/-/array-union-2.1.0.tgz" integrity sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw== +asynckit@^0.4.0: + version "0.4.0" + resolved "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz" + integrity sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q== + +axios@^1.5.1: + version "1.5.1" + resolved "https://registry.npmjs.org/axios/-/axios-1.5.1.tgz" + integrity sha512-Q28iYCWzNHjAm+yEAot5QaAMxhMghWLFVf7rRdwhUI+c2jix2DUXjAHXVi+s1ibs3mjPO/cCgbA++3BjD0vP/A== + dependencies: + follow-redirects "^1.15.0" + form-data "^4.0.0" + proxy-from-env "^1.1.0" + babel-jest@^29.0.0, babel-jest@^29.6.4: version "29.6.4" resolved "https://registry.npmjs.org/babel-jest/-/babel-jest-29.6.4.tgz" @@ -1299,6 +1318,13 @@ cjs-module-lexer@^1.0.0: resolved "https://registry.npmjs.org/cjs-module-lexer/-/cjs-module-lexer-1.2.3.tgz" integrity sha512-0TNiGstbQmCFwt4akjjBg5pLRTSyj/PkWQ1ZoO2zntmg9yLqSRxwEa4iCfQLGjqhiqBfOJa7W/E8wfGrTDmlZQ== +cli-progress@^3.12.0: + version "3.12.0" + resolved "https://registry.npmjs.org/cli-progress/-/cli-progress-3.12.0.tgz" + integrity sha512-tRkV3HJ1ASwm19THiiLIXLO7Im7wlTuKnvkYaTkyoAPefqjNg7W7DHKUlGRxy9vxDvbyCYQkQozvptuMkGCg8A== + dependencies: + string-width "^4.2.3" + cliui@^8.0.1: version "8.0.1" resolved "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz" @@ -1342,6 +1368,13 @@ color-name@1.1.3: resolved "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz" integrity sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw== +combined-stream@^1.0.8: + version "1.0.8" + resolved "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz" + integrity sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg== + dependencies: + delayed-stream "~1.0.0" + commander@^7.2.0: version "7.2.0" resolved "https://registry.npmjs.org/commander/-/commander-7.2.0.tgz" @@ -1405,6 +1438,11 @@ deepmerge@^4.2.2: resolved "https://registry.npmjs.org/deepmerge/-/deepmerge-4.3.1.tgz" integrity sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A== +delayed-stream@~1.0.0: + version "1.0.0" + resolved "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz" + integrity sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ== + deprecation@^2.0.0: version "2.3.1" resolved "https://registry.npmjs.org/deprecation/-/deprecation-2.3.1.tgz" @@ -1714,6 +1752,11 @@ flatted@^3.2.7: resolved "https://registry.npmjs.org/flatted/-/flatted-3.2.9.tgz" integrity sha512-36yxDn5H7OFZQla0/jFJmbIKTdZAQHngCedGxiMmpNfEZM0sdEeT+WczLQrjK6D7o2aiyLYDnkw0R3JK0Qv1RQ== +follow-redirects@^1.15.0: + version "1.15.3" + resolved "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.3.tgz" + integrity sha512-1VzOtuEM8pC9SFU1E+8KfTjZyMztRsgEfwQl44z8A25uy13jSzTj6dyK2Df52iV0vgHCfBwLhDWevLn95w5v6Q== + foreground-child@^3.1.0: version "3.1.1" resolved "https://registry.npmjs.org/foreground-child/-/foreground-child-3.1.1.tgz" @@ -1722,6 +1765,15 @@ foreground-child@^3.1.0: cross-spawn "^7.0.0" signal-exit "^4.0.1" +form-data@^4.0.0: + version "4.0.0" + resolved "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz" + integrity sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww== + dependencies: + asynckit "^0.4.0" + combined-stream "^1.0.8" + mime-types "^2.1.12" + fs-extra@^11.1.1: version "11.1.1" resolved "https://registry.npmjs.org/fs-extra/-/fs-extra-11.1.1.tgz" @@ -2217,7 +2269,7 @@ jest-environment-node@^29.6.4: jest-mock "^29.6.3" jest-util "^29.6.3" -jest-expect-message@*, jest-expect-message@^1.1.3: +jest-expect-message@^1.1.3: version "1.1.3" resolved "https://registry.npmjs.org/jest-expect-message/-/jest-expect-message-1.1.3.tgz" integrity sha512-bTK77T4P+zto+XepAX3low8XVQxDgaEqh3jSTQOG8qvPpD69LsIdyJTa+RmnJh3HNSzJng62/44RPPc7OIlFxg== @@ -2671,6 +2723,18 @@ micromatch@^4.0.4: braces "^3.0.2" picomatch "^2.3.1" +mime-db@1.52.0: + version "1.52.0" + resolved "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz" + integrity sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg== + +mime-types@^2.1.12: + version "2.1.35" + resolved "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz" + integrity sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw== + dependencies: + mime-db "1.52.0" + mimic-fn@^2.1.0: version "2.1.0" resolved "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz" @@ -2749,6 +2813,11 @@ npm-run-path@^4.0.1: dependencies: path-key "^3.0.0" +numeral@^2.0.6: + version "2.0.6" + resolved "https://registry.npmjs.org/numeral/-/numeral-2.0.6.tgz" + integrity sha512-qaKRmtYPZ5qdw4jWJD6bxEf1FJEqllJrwxCLIm0sQU/A7v2/czigzOb+C2uSiFsa9lBUzeH7M1oK+Q+OLxL3kA== + object-treeify@^2.1.1: version "2.1.1" resolved "https://registry.npmjs.org/object-treeify/-/object-treeify-2.1.1.tgz" @@ -2959,6 +3028,11 @@ prompts@^2.0.1: kleur "^3.0.3" sisteransi "^1.0.5" +proxy-from-env@^1.1.0: + version "1.1.0" + resolved "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz" + integrity sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg== + punycode@^2.1.0: version "2.3.0" resolved "https://registry.npmjs.org/punycode/-/punycode-2.3.0.tgz"