Update scripts

scripts/commands/api/generate.ts

@@ -1,16 +1,16 @@
-import { API_DIR, DB_DIR } from '../../constants'
+import { API_DIR, STREAMS_DIR } from '../../constants'
-import { Logger, Database, Collection, Storage } from '../../core'
+import { Logger, PlaylistParser, Storage } from '../../core'
 import { Stream } from '../../models'
 
 async function main() {
   const logger = new Logger()
 
-  logger.info(`loading streams...`)
+  logger.info('loading streams...')
-  const db = new Database(DB_DIR)
+  const streamsStorage = new Storage(STREAMS_DIR)
-  const dbStreams = await db.load('streams.db')
+  const parser = new PlaylistParser({ storage: streamsStorage })
-  const docs = await dbStreams.find({})
+  const files = await streamsStorage.list('**/*.m3u')
-
+  let streams = await parser.parse(files)
-  const streams = new Collection(docs as any[])
+  streams = streams
     .map(data => new Stream(data))
     .orderBy((stream: Stream) => stream.channel)
     .map((stream: Stream) => stream.toJSON())
@@ -18,8 +18,8 @@ async function main() {
   logger.info(`found ${streams.count()} streams`)
 
   logger.info('saving to .api/streams.json...')
-  const storage = new Storage(API_DIR)
+  const apiStorage = new Storage(API_DIR)
-  await storage.save('streams.json', streams.toJSON())
+  await apiStorage.save('streams.json', streams.toJSON())
 }
 
 main()

scripts/commands/api/load.sh

@@ -4,7 +4,6 @@ mkdir -p temp/data
 curl -L -o temp/data/blocklist.json https://iptv-org.github.io/api/blocklist.json 
 curl -L -o temp/data/categories.json https://iptv-org.github.io/api/categories.json 
 curl -L -o temp/data/channels.json https://iptv-org.github.io/api/channels.json
-curl -L -o temp/data/streams.json https://iptv-org.github.io/api/streams.json
 curl -L -o temp/data/countries.json https://iptv-org.github.io/api/countries.json 
 curl -L -o temp/data/languages.json https://iptv-org.github.io/api/languages.json 
 curl -L -o temp/data/regions.json https://iptv-org.github.io/api/regions.json 

scripts/commands/database/create.ts

@@ -1,33 +0,0 @@
-import { Storage, Logger, PlaylistParser, Collection, Database } from '../../core'
-import { Stream, Playlist } from '../../models'
-import { STREAMS_DIR, DB_DIR } from '../../constants'
-
-async function main() {
-  const logger = new Logger()
-
-  logger.info(`looking for streams...`)
-  const storage = new Storage(STREAMS_DIR)
-  const parser = new PlaylistParser({
-    storage
-  })
-  const files = await storage.list(`**/*.m3u`)
-  let streams = new Collection()
-  for (let filepath of files) {
-    const playlist: Playlist = await parser.parse(filepath)
-    streams = streams.concat(playlist.streams)
-  }
-
-  logger.info(`found ${streams.count()} streams`)
-
-  logger.info('clean up the storage...')
-  const dbStorage = new Storage(DB_DIR)
-  await dbStorage.clear('streams.db')
-
-  logger.info('saving streams to the database...')
-  const db = new Database(DB_DIR)
-  const dbStreams = await db.load('streams.db')
-  const data = streams.map((stream: Stream) => stream.data()).all()
-  await dbStreams.insert(data)
-}
-
-main()

scripts/commands/playlist/format.ts

@@ -0,0 +1,67 @@
+import { STREAMS_DIR, DATA_DIR } from '../../constants'
+import { Storage, Logger, PlaylistParser, Collection } from '../../core'
+import { Stream, Playlist, Channel } from '../../models'
+import { program } from 'commander'
+
+program.argument('[filepath]', 'Path to file to validate').parse(process.argv)
+
+async function main() {
+  const storage = new Storage(STREAMS_DIR)
+  const logger = new Logger()
+
+  logger.info('loading channels from api...')
+  const dataStorage = new Storage(DATA_DIR)
+  const channelsContent = await dataStorage.json('channels.json')
+  const groupedChannels = new Collection(channelsContent)
+    .map(data => new Channel(data))
+    .keyBy((channel: Channel) => channel.id)
+
+  logger.info('loading streams...')
+  const parser = new PlaylistParser({ storage })
+  const files = program.args.length ? program.args : await storage.list('**/*.m3u')
+  let streams = await parser.parse(files)
+
+  logger.info(`found ${streams.count()} streams`)
+
+  logger.info('normalizing links...')
+  streams = streams.map(stream => {
+    stream.normalizeURL()
+    return stream
+  })
+
+  logger.info('removing duplicates...')
+  streams = streams.uniqBy(stream => stream.url)
+
+  logger.info('removing wrong id...')
+  streams = streams.map((stream: Stream) => {
+    if (groupedChannels.missing(stream.channel)) {
+      stream.channel = ''
+    }
+
+    return stream
+  })
+
+  logger.info('sorting links...')
+  streams = streams.orderBy(
+    [
+      (stream: Stream) => stream.name,
+      (stream: Stream) => parseInt(stream.quality.replace('p', '')),
+      (stream: Stream) => stream.label,
+      (stream: Stream) => stream.url
+    ],
+    ['asc', 'desc', 'asc', 'asc']
+  )
+
+  logger.info('saving...')
+  const groupedStreams = streams.groupBy((stream: Stream) => stream.filepath)
+  for (let filepath of groupedStreams.keys()) {
+    const streams = groupedStreams.get(filepath) || []
+
+    if (!streams.length) return
+
+    const playlist = new Playlist(streams, { public: false })
+    await storage.save(filepath, playlist.toString())
+  }
+}
+
+main()

scripts/commands/playlist/generate.ts

@@ -1,6 +1,5 @@
-import { File, Storage } from '../../core'
+import { File, PlaylistParser, Storage } from '../../core'
 import { Stream, Category, Channel, Language, Country, Region, Subdivision } from '../../models'
-import { Database } from '../../core/database'
 import { Collection } from '../../core/collection'
 import { Logger } from '../../core/logger'
 import _ from 'lodash'
@@ -16,32 +15,29 @@ import {
   IndexLanguageGenerator,
   IndexRegionGenerator
 } from '../../generators'
-import { DATA_DIR, DB_DIR, LOGS_DIR } from '../../constants'
+import { DATA_DIR, LOGS_DIR, STREAMS_DIR } from '../../constants'
 
 async function main() {
   const logger = new Logger()
+  const dataStorage = new Storage(DATA_DIR)
 
-  const storage = new Storage(DATA_DIR)
+  logger.info('loading data from api...')
-
+  const channelsContent = await dataStorage.json('channels.json')
-  const channelsContent = await storage.json('channels.json')
   const channels = new Collection(channelsContent).map(data => new Channel(data))
-
+  const categoriesContent = await dataStorage.json('categories.json')
-  const categoriesContent = await storage.json('categories.json')
   const categories = new Collection(categoriesContent).map(data => new Category(data))
-
+  const countriesContent = await dataStorage.json('countries.json')
-  const countriesContent = await storage.json('countries.json')
   const countries = new Collection(countriesContent).map(data => new Country(data))
-
+  const languagesContent = await dataStorage.json('languages.json')
-  const languagesContent = await storage.json('languages.json')
   const languages = new Collection(languagesContent).map(data => new Language(data))
-
+  const regionsContent = await dataStorage.json('regions.json')
-  const regionsContent = await storage.json('regions.json')
   const regions = new Collection(regionsContent).map(data => new Region(data))
-
+  const subdivisionsContent = await dataStorage.json('subdivisions.json')
-  const subdivisionsContent = await storage.json('subdivisions.json')
   const subdivisions = new Collection(subdivisionsContent).map(data => new Subdivision(data))
 
+  logger.info('loading streams...')
   const streams = await loadStreams({ channels, categories, languages })
+  logger.info(`found ${streams.count()} streams`)
 
   const generatorsLogger = new Logger({
     stream: await new Storage(LOGS_DIR).createStream(`generators.log`)
@@ -49,7 +45,6 @@ async function main() {
 
   logger.info('generating categories/...')
   await new CategoriesGenerator({ categories, streams, logger: generatorsLogger }).generate()
-
   logger.info('generating countries/...')
   await new CountriesGenerator({
     countries,
@@ -58,10 +53,8 @@ async function main() {
     subdivisions,
     logger: generatorsLogger
   }).generate()
-
   logger.info('generating languages/...')
   await new LanguagesGenerator({ streams, logger: generatorsLogger }).generate()
-
   logger.info('generating regions/...')
   await new RegionsGenerator({
     streams,
@@ -69,16 +62,12 @@ async function main() {
     subdivisions,
     logger: generatorsLogger
   }).generate()
-
   logger.info('generating index.m3u...')
   await new IndexGenerator({ streams, logger: generatorsLogger }).generate()
-
   logger.info('generating index.nsfw.m3u...')
   await new IndexNsfwGenerator({ streams, logger: generatorsLogger }).generate()
-
   logger.info('generating index.category.m3u...')
   await new IndexCategoryGenerator({ streams, logger: generatorsLogger }).generate()
-
   logger.info('generating index.country.m3u...')
   await new IndexCountryGenerator({
     streams,
@@ -87,10 +76,8 @@ async function main() {
     subdivisions,
     logger: generatorsLogger
   }).generate()
-
   logger.info('generating index.language.m3u...')
   await new IndexLanguageGenerator({ streams, logger: generatorsLogger }).generate()
-
   logger.info('generating index.region.m3u...')
   await new IndexRegionGenerator({ streams, regions, logger: generatorsLogger }).generate()
 }
@@ -110,11 +97,12 @@ async function loadStreams({
   const groupedCategories = categories.keyBy(category => category.id)
   const groupedLanguages = languages.keyBy(language => language.code)
 
-  const db = new Database(DB_DIR)
+  const storage = new Storage(STREAMS_DIR)
-  const dbStreams = await db.load('streams.db')
+  const parser = new PlaylistParser({ storage })
-  const docs = await dbStreams.find({})
+  const files = await storage.list('**/*.m3u')
-  const streams = new Collection(docs as any[])
+  let streams = await parser.parse(files)
-    .map((data: any) => new Stream(data))
+
+  streams = streams
     .orderBy([(stream: Stream) => stream.channel, (stream: Stream) => stream.url], ['asc', 'asc'])
     .uniqBy((stream: Stream) => stream.channel || _.uniqueId())
     .map((stream: Stream) => {

scripts/commands/playlist/update.ts

@@ -1,6 +1,6 @@
-import { DB_DIR, DATA_DIR, STREAMS_DIR } from '../../constants'
+import { DATA_DIR, STREAMS_DIR } from '../../constants'
-import { Database, Storage, Logger, Collection, Dictionary, IssueLoader } from '../../core'
+import { Storage, Logger, Collection, Dictionary, IssueLoader, PlaylistParser } from '../../core'
-import { Stream, Playlist, Channel } from '../../models'
+import { Stream, Playlist, Channel, Issue } from '../../models'
 
 let processedIssues = new Collection()
 let streams: Collection
@@ -10,19 +10,19 @@ async function main() {
   const logger = new Logger({ disabled: true })
   const loader = new IssueLoader()
 
-  logger.info('loading streams...')
+  logger.info('loading channels from api...')
-  const db = new Database(DB_DIR)
+  const dataStorage = new Storage(DATA_DIR)
-  const docs = await db.load('streams.db')
+  const channelsContent = await dataStorage.json('channels.json')
-  const dbStreams = await docs.find({})
-
-  streams = new Collection(dbStreams as any[]).map(data => new Stream(data))
-
-  const storage = new Storage(DATA_DIR)
-  const channelsContent = await storage.json('channels.json')
   groupedChannels = new Collection(channelsContent)
     .map(data => new Channel(data))
     .keyBy((channel: Channel) => channel.id)
 
+  logger.info('loading streams...')
+  const streamsStorage = new Storage(STREAMS_DIR)
+  const parser = new PlaylistParser({ storage: streamsStorage })
+  const files = await streamsStorage.list('**/*.m3u')
+  streams = await parser.parse(files)
+
   logger.info('removing broken streams...')
   await removeStreams(loader)
 
@@ -32,25 +32,7 @@ async function main() {
   logger.info('add new streams...')
   await addStreams(loader)
 
-  logger.info('normalizing links...')
-  streams = streams.map(stream => {
-    stream.normalizeURL()
-    return stream
-  })
-
-  logger.info('sorting links...')
-  streams = streams.orderBy(
-    [
-      (stream: Stream) => stream.name,
-      (stream: Stream) => parseInt(stream.quality.replace('p', '')),
-      (stream: Stream) => stream.label,
-      (stream: Stream) => stream.url
-    ],
-    ['asc', 'desc', 'asc', 'asc']
-  )
-
   logger.info('saving...')
-  const streamsStorage = new Storage(STREAMS_DIR)
   const groupedStreams = streams.groupBy((stream: Stream) => stream.filepath)
   for (let filepath of groupedStreams.keys()) {
     const streams = groupedStreams.get(filepath) || []
@@ -69,19 +51,22 @@ main()
 
 async function removeStreams(loader: IssueLoader) {
   const issues = await loader.load({ labels: ['streams:remove', 'approved'] })
-  issues.forEach((data: Dictionary) => {
+  issues.forEach((issue: Issue) => {
+    const data = issue.data
     if (data.missing('stream_url')) return
 
     const removed = streams.remove((_stream: Stream) => _stream.url === data.get('stream_url'))
     if (removed.notEmpty()) {
-      processedIssues.add(data.get('issue_number'))
+      processedIssues.add(issue.number)
     }
   })
 }
 
 async function editStreams(loader: IssueLoader) {
   const issues = await loader.load({ labels: ['streams:edit', 'approved'] })
-  issues.forEach((data: Dictionary) => {
+  issues.forEach((issue: Issue) => {
+    const data = issue.data
+
     if (data.missing('stream_url')) return
 
     let stream = streams.first(
@@ -111,13 +96,14 @@ async function editStreams(loader: IssueLoader) {
     streams.remove((_stream: Stream) => _stream.channel === stream.channel)
     streams.add(stream)
 
-    processedIssues.add(data.get('issue_number'))
+    processedIssues.add(issue.number)
   })
 }
 
 async function addStreams(loader: IssueLoader) {
   const issues = await loader.load({ labels: ['streams:add', 'approved'] })
-  issues.forEach((data: Dictionary) => {
+  issues.forEach((issue: Issue) => {
+    const data = issue.data
     if (data.missing('channel_id') || data.missing('stream_url')) return
     if (streams.includes((_stream: Stream) => _stream.url === data.get('stream_url'))) return
 
@@ -138,6 +124,6 @@ async function addStreams(loader: IssueLoader) {
     })
 
     streams.add(stream)
-    processedIssues.add(data.get('issue_number'))
+    processedIssues.add(issue.number)
   })
 }

scripts/commands/playlist/validate.ts

@@ -5,7 +5,6 @@ import chalk from 'chalk'
 import { transliterate } from 'transliteration'
 import _ from 'lodash'
 import { DATA_DIR, STREAMS_DIR } from '../../constants'
-import path from 'path'
 
 program.argument('[filepath]', 'Path to file to validate').parse(process.argv)
 
@@ -19,31 +18,35 @@ async function main() {
   const logger = new Logger()
 
   logger.info(`loading blocklist...`)
-  const storage = new Storage(DATA_DIR)
+  const dataStorage = new Storage(DATA_DIR)
-  const channelsContent = await storage.json('channels.json')
+  const channelsContent = await dataStorage.json('channels.json')
   const channels = new Collection(channelsContent).map(data => new Channel(data))
-  const blocklistContent = await storage.json('blocklist.json')
+  const blocklistContent = await dataStorage.json('blocklist.json')
   const blocklist = new Collection(blocklistContent).map(data => new Blocked(data))
 
   logger.info(`found ${blocklist.count()} records`)
 
-  let errors = new Collection()
+  logger.info('loading streams...')
-  let warnings = new Collection()
   const streamsStorage = new Storage(STREAMS_DIR)
   const parser = new PlaylistParser({ storage: streamsStorage })
   const files = program.args.length ? program.args : await streamsStorage.list('**/*.m3u')
-  for (const filepath of files) {
+  const streams = await parser.parse(files)
-    const file = new File(filepath)
-    if (file.extension() !== 'm3u') continue
 
+  logger.info(`found ${streams.count()} streams`)
+
+  let errors = new Collection()
+  let warnings = new Collection()
+  let groupedStreams = streams.groupBy((stream: Stream) => stream.filepath)
+  for (const filepath of groupedStreams.keys()) {
+    const streams = groupedStreams.get(filepath)
+    if (!streams) continue
+
+    const file = new File(filepath)
     const [, countryCode] = file.basename().match(/([a-z]{2})(|_.*)\.m3u/i) || [null, '']
 
     const log = new Collection()
     const buffer = new Dictionary()
-    try {
+    streams.forEach((stream: Stream) => {
-      const relativeFilepath = filepath.replace(path.normalize(STREAMS_DIR), '')
-      const playlist = await parser.parse(relativeFilepath)
-      playlist.streams.forEach((stream: Stream) => {
       const channelNotInDatabase =
         stream.channel && !channels.first((channel: Channel) => channel.id === stream.channel)
       if (channelNotInDatabase) {
@@ -79,13 +82,6 @@ async function main() {
         })
       }
     })
-    } catch (error) {
-      log.add({
-        type: 'error',
-        line: 0,
-        message: error.message.toLowerCase()
-      })
-    }
 
     if (log.notEmpty()) {
       logger.info(`\n${chalk.underline(filepath)}`)

scripts/commands/report/create.ts

@@ -1,36 +1,43 @@
-import { DATA_DIR } from '../../constants'
+import { DATA_DIR, STREAMS_DIR } from '../../constants'
-import { Collection, Dictionary, IssueLoader, Storage } from '../../core'
+import { Collection, Dictionary, IssueLoader, Storage, Logger, PlaylistParser } from '../../core'
-import { Blocked, Channel, Stream } from '../../models'
+import { Blocked, Channel, Issue, Stream } from '../../models'
 
 async function main() {
+  const logger = new Logger()
   const loader = new IssueLoader()
 
   const storage = new Storage(DATA_DIR)
 
+  logger.info('loading channels from api...')
   const channelsContent = await storage.json('channels.json')
   const groupedChannels = new Collection(channelsContent)
     .map(data => new Channel(data))
     .groupBy((channel: Channel) => channel.id)
 
-  const streamsContent = await storage.json('streams.json')
+  logger.info('loading blocklist from api...')
-  const groupedStreams = new Collection(streamsContent)
-    .map(data => new Stream(data))
-    .groupBy((stream: Stream) => stream.url)
-
   const blocklistContent = await storage.json('blocklist.json')
   const groupedBlocklist = new Collection(blocklistContent)
     .map(data => new Blocked(data))
     .groupBy((blocked: Blocked) => blocked.channel)
 
+  logger.info('loading streams...')
+  const streamsStorage = new Storage(STREAMS_DIR)
+  const parser = new PlaylistParser({ storage: streamsStorage })
+  const files = await streamsStorage.list('**/*.m3u')
+  const streams = await parser.parse(files)
+  const groupedStreams = streams.groupBy((stream: Stream) => stream.url)
+
+  logger.info('loading issue from github...')
   const issues = await loader.load({ labels: ['streams:add'] })
 
+  logger.info('creating report...')
   const buffer = new Dictionary()
-  const report = issues.map(data => {
+  const report = issues.map((issue: Issue) => {
-    const channelId = data.get('channel_id') || undefined
+    const channelId = issue.data.get('channel_id') || undefined
-    const streamUrl = data.get('stream_url') || undefined
+    const streamUrl = issue.data.get('stream_url') || undefined
 
     const result = new Dictionary({
-      issueNumber: data.get('issue_number'),
+      issueNumber: issue.number,
       channelId,
       status: undefined
     })

scripts/constants.ts

@@ -5,7 +5,6 @@ export const README_DIR = process.env.README_DIR || './.readme'
 export const API_DIR = process.env.API_DIR || './.api'
 export const DATA_DIR = process.env.DATA_DIR || './temp/data'
 export const LOGS_DIR = process.env.LOGS_DIR || './temp/logs'
-export const DB_DIR = process.env.DB_DIR || './temp/database'
 export const TESTING = process.env.NODE_ENV === 'test' ? true : false
 export const OWNER = 'iptv-org'
 export const REPO = 'iptv'

scripts/core/database.ts

@@ -1,22 +0,0 @@
-import Datastore from '@seald-io/nedb'
-import * as path from 'path'
-
-export class Database {
-  rootDir: string
-
-  constructor(rootDir: string) {
-    this.rootDir = rootDir
-  }
-
-  async load(filepath: string) {
-    const absFilepath = path.join(this.rootDir, filepath)
-
-    return new Datastore({
-      filename: path.resolve(absFilepath),
-      autoload: true,
-      onload: (error: Error): any => {
-        if (error) console.error(error.message)
-      }
-    })
-  }
-}

scripts/core/index.ts

@@ -1,4 +1,3 @@
-export * from './database'
 export * from './logger'
 export * from './playlistParser'
 export * from './numberParser'

scripts/core/issueParser.ts

@@ -1,11 +1,8 @@
 import { Dictionary } from './'
+import { Issue } from '../models'
+import _ from 'lodash'
 
-export class IssueParser {
+const FIELDS = new Dictionary({
-  parse(issue: any): Dictionary {
-    const data = new Dictionary()
-    data.set('issue_number', issue.number)
-
-    const idDict = new Dictionary({
   'Channel ID': 'channel_id',
   'Channel ID (required)': 'channel_id',
   'Broken Link': 'stream_url',
@@ -22,12 +19,13 @@ export class IssueParser {
   'Possible Replacement (optional)': 'possible_replacement',
   Notes: 'notes',
   'Notes (optional)': 'notes'
-    })
+})
 
+export class IssueParser {
+  parse(issue: any): Issue {
     const fields = issue.body.split('###')
 
-    if (!fields.length) return data
+    const data = new Dictionary()
-
     fields.forEach((field: string) => {
       let [_label, , _value] = field.split(/\r?\n/)
       _label = _label ? _label.trim() : ''
@@ -35,7 +33,7 @@ export class IssueParser {
 
       if (!_label || !_value) return data
 
-      const id: string = idDict.get(_label)
+      const id: string = FIELDS.get(_label)
       const value: string = _value === '_No response_' || _value === 'None' ? '' : _value
 
       if (!id) return
@@ -43,6 +41,6 @@ export class IssueParser {
       data.set(id, value)
     })
 
-    return data
+    return new Issue({ number: issue.number, data })
   }
 }

scripts/core/playlistParser.ts

@@ -1,6 +1,8 @@
 import parser from 'iptv-playlist-parser'
-import { Playlist, Stream } from '../models'
+import { Stream } from '../models'
 import { Collection, Storage } from './'
+import path from 'path'
+import { STREAMS_DIR } from '../constants'
 
 export class PlaylistParser {
   storage: Storage
@@ -9,7 +11,19 @@ export class PlaylistParser {
     this.storage = storage
   }
 
-  async parse(filepath: string): Promise<Playlist> {
+  async parse(files: string[]): Promise<Collection> {
+    let streams = new Collection()
+
+    for (let filepath of files) {
+      const relativeFilepath = filepath.replace(path.normalize(STREAMS_DIR), '')
+      const _streams: Collection = await this.parseFile(relativeFilepath)
+      streams = streams.concat(_streams)
+    }
+
+    return streams
+  }
+
+  async parseFile(filepath: string): Promise<Collection> {
     const streams = new Collection()
 
     const content = await this.storage.read(filepath)
@@ -32,7 +46,7 @@ export class PlaylistParser {
       streams.add(stream)
     })
 
-    return new Playlist(streams)
+    return streams
   }
 }
 

scripts/core/storage.ts

@@ -10,10 +10,12 @@ export class Storage {
     this.rootDir = path.normalize(rootDir || './')
   }
 
-  list(pattern: string): Promise<string[]> {
+  async list(pattern: string): Promise<string[]> {
-    return glob(pattern, {
+    const files = await glob(pattern, {
       cwd: this.rootDir
     })
+
+    return files.sort()
   }
 
   async createDir(dir: string): Promise<void> {

scripts/models/index.ts

@@ -1,3 +1,4 @@
+export * from './issue'
 export * from './playlist'
 export * from './blocked'
 export * from './stream'

scripts/models/issue.ts

@@ -0,0 +1,16 @@
+import { Dictionary } from '../core'
+
+type IssueProps = {
+  number: number
+  data: Dictionary
+}
+
+export class Issue {
+  number: number
+  data: Dictionary
+
+  constructor({ number, data }: IssueProps) {
+    this.number = number
+    this.data = data
+  }
+}