From 958703d6d60d456b59251a783a72cb97dc0820de Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Sun, 12 Dec 2021 07:10:52 +0300 Subject: [PATCH] Remove old scripts --- scripts/create-matrix.js | 9 - scripts/filter.js | 43 ----- scripts/format.js | 307 --------------------------------- scripts/generate.js | 232 ------------------------- scripts/helpers/Channel.js | 162 ----------------- scripts/helpers/Playlist.js | 41 ----- scripts/helpers/db.js | 237 ------------------------- scripts/helpers/epg.js | 12 -- scripts/helpers/file.js | 53 ------ scripts/helpers/log.js | 17 -- scripts/helpers/parser.js | 20 --- scripts/helpers/utils.js | 86 --------- scripts/remove-broken-links.js | 42 ----- scripts/remove-duplicates.js | 81 --------- scripts/sort.js | 41 ----- scripts/update-readme.js | 142 --------------- 16 files changed, 1525 deletions(-) delete mode 100644 scripts/create-matrix.js delete mode 100644 scripts/filter.js delete mode 100644 scripts/format.js delete mode 100644 scripts/generate.js delete mode 100644 scripts/helpers/Channel.js delete mode 100644 scripts/helpers/Playlist.js delete mode 100644 scripts/helpers/db.js delete mode 100644 scripts/helpers/epg.js delete mode 100644 scripts/helpers/file.js delete mode 100644 scripts/helpers/log.js delete mode 100644 scripts/helpers/parser.js delete mode 100644 scripts/helpers/utils.js delete mode 100644 scripts/remove-broken-links.js delete mode 100644 scripts/remove-duplicates.js delete mode 100644 scripts/sort.js delete mode 100644 scripts/update-readme.js diff --git a/scripts/create-matrix.js b/scripts/create-matrix.js deleted file mode 100644 index 6e306d42b..000000000 --- a/scripts/create-matrix.js +++ /dev/null @@ -1,9 +0,0 @@ -const file = require('./helpers/file') - -file.list().then(files => { - files = files.filter(file => file !== 'channels/unsorted.m3u') - const country = files.map(file => file.replace(/channels\/|\.m3u/gi, '')) - const matrix = { country } - const output = `::set-output name=matrix::${JSON.stringify(matrix)}` - console.log(output) -}) diff --git a/scripts/filter.js b/scripts/filter.js deleted file mode 100644 index f6676c937..000000000 --- a/scripts/filter.js +++ /dev/null @@ -1,43 +0,0 @@ -const blacklist = require('./data/blacklist.json') -const parser = require('./helpers/parser') -const file = require('./helpers/file') -const log = require('./helpers/log') - -async function main() { - log.start() - - const files = await file.list() - if (!files.length) log.print(`No files is selected\n`) - for (const file of files) { - log.print(`\nProcessing '${file}'...`) - await parser - .parsePlaylist(file) - .then(removeBlacklisted) - .then(p => p.save()) - } - - log.print('\n') - log.finish() -} - -function removeBlacklisted(playlist) { - const channels = playlist.channels.filter(channel => { - return !blacklist.find(item => { - const regexp = new RegExp(item.regex, 'i') - const hasSameName = regexp.test(channel.name) - const fromSameCountry = playlist.country.code === item.country - - return hasSameName && fromSameCountry - }) - }) - - if (playlist.channels.length !== channels.length) { - log.print(`updated`) - playlist.channels = channels - playlist.updated = true - } - - return playlist -} - -main() diff --git a/scripts/format.js b/scripts/format.js deleted file mode 100644 index 74bf3e276..000000000 --- a/scripts/format.js +++ /dev/null @@ -1,307 +0,0 @@ -const axios = require('axios') -const { program } = require('commander') -const normalize = require('normalize-url') -const IPTVChecker = require('iptv-checker') -const parser = require('./helpers/parser') -const utils = require('./helpers/utils') -const file = require('./helpers/file') -const log = require('./helpers/log') -const epg = require('./helpers/epg') - -const ignoreStatus = ['Geo-blocked'] - -program - .usage('[OPTIONS]...') - .option('--debug', 'Enable debug mode') - .option('--offline', 'Enable offline mode') - .option('-d, --delay ', 'Set delay for each request', parseNumber, 0) - .option('-t, --timeout ', 'Set timeout for each request', parseNumber, 5000) - .option('-c, --country ', 'Comma-separated list of country codes', '') - .option('-e, --exclude ', 'Comma-separated list of country codes to be excluded', '') - .parse(process.argv) - -const config = program.opts() -const checker = new IPTVChecker({ - timeout: config.timeout -}) - -let buffer, origins -async function main() { - log.start() - - const include = config.country.split(',').filter(i => i) - const exclude = config.exclude.split(',').filter(i => i) - let files = await file.list(include, exclude) - if (!files.length) log.print(`No files is selected\n`) - for (const file of files) { - await parser.parsePlaylist(file).then(updatePlaylist).then(savePlaylist) - } - - log.finish() -} - -function savePlaylist(playlist) { - if (file.read(playlist.url) !== playlist.toString()) { - log.print(`File '${playlist.url}' has been updated\n`) - playlist.updated = true - } - - playlist.save() -} - -async function updatePlaylist(playlist) { - const total = playlist.channels.length - log.print(`Processing '${playlist.url}'...\n`) - - let channels = {} - let codes = {} - if (!config.offline) { - channels = await loadChannelsJson() - codes = await loadCodes() - } - - buffer = {} - origins = {} - for (const [i, channel] of playlist.channels.entries()) { - const curr = i + 1 - updateTvgName(channel) - updateTvgId(channel, playlist) - updateTvgCountry(channel) - normalizeUrl(channel) - - const data = channels[channel.tvg.id] - const epgData = codes[channel.tvg.id] - updateLogo(channel, data, epgData) - updateGroupTitle(channel, data) - updateTvgLanguage(channel, data) - - if (config.offline || ignoreStatus.includes(channel.status)) { - continue - } - - await checker - .checkStream(channel.data) - .then(parseResult) - .then(result => { - updateStatus(channel, result.status) - if (result.status === 'online') { - buffer[i] = result - updateOrigins(channel, result.requests) - updateResolution(channel, result.resolution) - } else { - buffer[i] = null - if (config.debug) { - log.print(` INFO: ${channel.url} (${result.error})\n`) - } - } - }) - .catch(err => { - buffer[i] = null - if (config.debug) { - log.print(` ERR: ${channel.data.url} (${err.message})\n`) - } - }) - } - - for (const [i, channel] of playlist.channels.entries()) { - if (!buffer[i]) continue - const { requests } = buffer[i] - updateUrl(channel, requests) - } - - return playlist -} - -function updateOrigins(channel, requests) { - if (!requests) return - const origin = new URL(channel.url) - const target = new URL(requests[0]) - const type = origin.host === target.host ? 'origin' : 'redirect' - requests.forEach(url => { - const key = utils.removeProtocol(url) - if (!origins[key] && type === 'origin') { - origins[key] = channel.url - } - }) -} - -function updateStatus(channel, status) { - switch (status) { - case 'online': - if (channel.status !== 'Not 24/7') - channel.status = channel.status === 'Offline' ? 'Not 24/7' : null - break - case 'error_403': - if (!channel.status) channel.status = 'Geo-blocked' - break - case 'offline': - if (channel.status !== 'Not 24/7') channel.status = 'Offline' - break - } -} - -function updateResolution(channel, resolution) { - if (!channel.resolution.height && resolution) { - channel.resolution = resolution - } -} - -function updateUrl(channel, requests) { - for (const request of requests) { - let key = utils.removeProtocol(channel.url) - if (origins[key]) { - channel.updateUrl(origins[key]) - break - } - - key = utils.removeProtocol(request) - if (origins[key]) { - channel.updateUrl(origins[key]) - break - } - } -} - -function parseResult(result) { - return { - status: parseStatus(result.status), - resolution: result.status.ok ? parseResolution(result.status.metadata.streams) : null, - requests: result.status.ok ? parseRequests(result.status.metadata.requests) : [], - error: !result.status.ok ? result.status.reason : null - } -} - -function parseStatus(status) { - if (status.ok) { - return 'online' - } else if (status.reason.includes('timed out')) { - return 'timeout' - } else if (status.reason.includes('403')) { - return 'error_403' - } else if (status.reason.includes('not one of 40{0,1,3,4}')) { - return 'error_40x' // 402, 451 - } else { - return 'offline' - } -} - -function parseResolution(streams) { - const resolution = streams - .filter(stream => stream.codec_type === 'video') - .reduce( - (acc, curr) => { - if (curr.height > acc.height) return { width: curr.width, height: curr.height } - return acc - }, - { width: 0, height: 0 } - ) - - return resolution.width > 0 && resolution.height > 0 ? resolution : null -} - -function parseRequests(requests) { - requests = requests.map(r => r.url) - requests.shift() - - return requests -} - -function updateTvgName(channel) { - if (!channel.tvg.name) { - channel.tvg.name = channel.name.replace(/\"/gi, '') - } -} - -function updateTvgId(channel, playlist) { - const code = playlist.country.code - if (!channel.tvg.id && channel.tvg.name) { - const id = utils.name2id(channel.tvg.name) - channel.tvg.id = id ? `${id}.${code}` : '' - } -} - -function updateTvgCountry(channel) { - if (!channel.countries.length && channel.tvg.id) { - const code = channel.tvg.id.split('.')[1] || null - const name = utils.code2name(code) - channel.countries = name ? [{ code, name }] : [] - channel.tvg.country = channel.countries.map(c => c.code.toUpperCase()).join(';') - } -} - -function updateLogo(channel, data, epgData) { - if (!channel.logo) { - if (data && data.logo) { - channel.logo = data.logo - } else if (epgData && epgData.logo) { - channel.logo = epgData.logo - } - } -} - -function updateTvgLanguage(channel, data) { - if (!channel.tvg.language) { - if (data && data.languages.length) { - channel.tvg.language = data.languages.map(l => l.name).join(';') - } else if (channel.countries.length) { - const countryCode = channel.countries[0].code - channel.tvg.language = utils.country2language(countryCode) - } - } -} - -function updateGroupTitle(channel, data) { - if (!channel.group.title) { - if (channel.category) { - channel.group.title = channel.category - } else if (data && data.category) { - channel.group.title = data.category - } - } -} - -function normalizeUrl(channel) { - const normalized = normalize(channel.url, { stripWWW: false }) - const decoded = decodeURIComponent(normalized).replace(/\s/g, '+') - channel.updateUrl(decoded) -} - -function parseNumber(str) { - return parseInt(str) -} - -function loadCodes() { - return epg.codes - .load() - .then(codes => { - let output = {} - codes.forEach(item => { - output[item['tvg_id']] = item - }) - return output - }) - .catch(console.log) -} - -function loadChannelsJson() { - return axios - .get('https://iptv-org.github.io/iptv/channels.json') - .then(r => r.data) - .then(channels => { - let output = {} - channels.forEach(channel => { - const item = output[channel.tvg.id] - if (!item) { - output[channel.tvg.id] = channel - } else { - item.logo = item.logo || channel.logo - item.languages = item.languages.length ? item.languages : channel.languages - item.category = item.category || channel.category - } - }) - return output - }) - .catch(console.log) -} - -main() diff --git a/scripts/generate.js b/scripts/generate.js deleted file mode 100644 index 07aaf7e1a..000000000 --- a/scripts/generate.js +++ /dev/null @@ -1,232 +0,0 @@ -const file = require('./helpers/file') -const log = require('./helpers/log') -const db = require('./helpers/db') - -const ROOT_DIR = './.gh-pages' - -async function main() { - await loadDatabase() - createRootDirectory() - createNoJekyllFile() - generateIndex() - generateCategoryIndex() - generateCountryIndex() - generateLanguageIndex() - generateCategories() - generateCountries() - generateLanguages() - generateChannelsJson() - showResults() -} - -async function loadDatabase() { - log.print('Loading database...\n') - await db.load() -} - -function createRootDirectory() { - log.print('Creating .gh-pages folder...\n') - file.createDir(ROOT_DIR) -} - -function createNoJekyllFile() { - log.print('Creating .nojekyll...\n') - file.create(`${ROOT_DIR}/.nojekyll`) -} - -function generateIndex() { - log.print('Generating index.m3u...\n') - const channels = db.channels - .sortBy(['name', 'status', 'resolution.height', 'url'], ['asc', 'asc', 'desc', 'asc']) - .removeDuplicates() - .removeOffline() - .get() - const guides = channels.map(channel => channel.tvg.url) - - const filename = `${ROOT_DIR}/index.m3u` - const urlTvg = generateUrlTvg(guides) - file.create(filename, `#EXTM3U url-tvg="${urlTvg}"\n`) - - const nsfwFilename = `${ROOT_DIR}/index.nsfw.m3u` - file.create(nsfwFilename, `#EXTM3U url-tvg="${urlTvg}"\n`) - - for (const channel of channels) { - if (!channel.isNSFW()) { - file.append(filename, channel.toString()) - } - file.append(nsfwFilename, channel.toString()) - } -} - -function generateCategoryIndex() { - log.print('Generating index.category.m3u...\n') - const channels = db.channels - .sortBy( - ['category', 'name', 'status', 'resolution.height', 'url'], - ['asc', 'asc', 'asc', 'desc', 'asc'] - ) - .removeDuplicates() - .removeOffline() - .get() - const guides = channels.map(channel => channel.tvg.url) - - const filename = `${ROOT_DIR}/index.category.m3u` - const urlTvg = generateUrlTvg(guides) - file.create(filename, `#EXTM3U url-tvg="${urlTvg}"\n`) - - for (const channel of channels) { - file.append(filename, channel.toString()) - } -} - -function generateCountryIndex() { - log.print('Generating index.country.m3u...\n') - - const guides = [] - const lines = [] - for (const country of [{ code: 'undefined' }, ...db.countries.sortBy(['name']).all()]) { - const channels = db.channels - .sortBy(['name', 'status', 'resolution.height', 'url'], ['asc', 'asc', 'desc', 'asc']) - .forCountry(country) - .removeDuplicates() - .removeNSFW() - .removeOffline() - .get() - for (const channel of channels) { - const groupTitle = channel.group.title - channel.group.title = country.name || '' - lines.push(channel.toString()) - channel.group.title = groupTitle - guides.push(channel.tvg.url) - } - } - - const filename = `${ROOT_DIR}/index.country.m3u` - const urlTvg = generateUrlTvg(guides) - file.create(filename, `#EXTM3U url-tvg="${urlTvg}"\n${lines.join('')}`) -} - -function generateLanguageIndex() { - log.print('Generating index.language.m3u...\n') - - const guides = [] - const lines = [] - for (const language of [{ code: 'undefined' }, ...db.languages.sortBy(['name']).all()]) { - const channels = db.channels - .sortBy(['name', 'status', 'resolution.height', 'url'], ['asc', 'asc', 'desc', 'asc']) - .forLanguage(language) - .removeDuplicates() - .removeNSFW() - .removeOffline() - .get() - for (const channel of channels) { - const groupTitle = channel.group.title - channel.group.title = language.name || '' - lines.push(channel.toString()) - channel.group.title = groupTitle - guides.push(channel.tvg.url) - } - } - - const filename = `${ROOT_DIR}/index.language.m3u` - const urlTvg = generateUrlTvg(guides) - file.create(filename, `#EXTM3U url-tvg="${urlTvg}"\n${lines.join('')}`) -} - -function generateCategories() { - log.print(`Generating /categories...\n`) - const outputDir = `${ROOT_DIR}/categories` - file.createDir(outputDir) - - for (const category of [...db.categories.all(), { id: 'other' }]) { - const channels = db.channels - .sortBy(['name', 'status', 'resolution.height', 'url'], ['asc', 'asc', 'desc', 'asc']) - .forCategory(category) - .removeDuplicates() - .removeOffline() - .get() - const guides = channels.map(channel => channel.tvg.url) - - const filename = `${outputDir}/${category.id}.m3u` - const urlTvg = generateUrlTvg(guides) - file.create(filename, `#EXTM3U url-tvg="${urlTvg}"\n`) - for (const channel of channels) { - file.append(filename, channel.toString()) - } - } -} - -function generateCountries() { - log.print(`Generating /countries...\n`) - const outputDir = `${ROOT_DIR}/countries` - file.createDir(outputDir) - - for (const country of [...db.countries.all(), { code: 'undefined' }]) { - const channels = db.channels - .sortBy(['name', 'status', 'resolution.height', 'url'], ['asc', 'asc', 'desc', 'asc']) - .forCountry(country) - .removeDuplicates() - .removeOffline() - .removeNSFW() - .get() - const guides = channels.map(channel => channel.tvg.url) - - const filename = `${outputDir}/${country.code}.m3u` - const urlTvg = generateUrlTvg(guides) - file.create(filename, `#EXTM3U url-tvg="${urlTvg}"\n`) - for (const channel of channels) { - file.append(filename, channel.toString()) - } - } -} - -function generateLanguages() { - log.print(`Generating /languages...\n`) - const outputDir = `${ROOT_DIR}/languages` - file.createDir(outputDir) - - for (const language of [...db.languages.all(), { code: 'undefined' }]) { - const channels = db.channels - .sortBy(['name', 'status', 'resolution.height', 'url'], ['asc', 'asc', 'desc', 'asc']) - .forLanguage(language) - .removeDuplicates() - .removeOffline() - .removeNSFW() - .get() - const guides = channels.map(channel => channel.tvg.url) - - const filename = `${outputDir}/${language.code}.m3u` - const urlTvg = generateUrlTvg(guides) - file.create(filename, `#EXTM3U url-tvg="${urlTvg}"\n`) - for (const channel of channels) { - file.append(filename, channel.toString()) - } - } -} - -function generateChannelsJson() { - log.print('Generating channels.json...\n') - const filename = `${ROOT_DIR}/channels.json` - const channels = db.channels - .sortBy(['name', 'status', 'resolution.height', 'url'], ['asc', 'asc', 'desc', 'asc']) - .get() - .map(c => c.toObject()) - file.create(filename, JSON.stringify(channels)) -} - -function showResults() { - log.print( - `Total: ${db.channels.count()} channels, ${db.countries.count()} countries, ${db.languages.count()} languages, ${db.categories.count()} categories.\n` - ) -} - -function generateUrlTvg(guides) { - const output = guides.reduce((acc, curr) => { - if (curr && !acc.includes(curr)) acc.push(curr) - return acc - }, []) - - return output.sort().join(',') -} - -main() diff --git a/scripts/helpers/Channel.js b/scripts/helpers/Channel.js deleted file mode 100644 index 3d177ad14..000000000 --- a/scripts/helpers/Channel.js +++ /dev/null @@ -1,162 +0,0 @@ -const categories = require('../data/categories') -const utils = require('./utils') -const file = require('./file') - -const sfwCategories = categories.filter(c => !c.nsfw).map(c => c.name) -const nsfwCategories = categories.filter(c => c.nsfw).map(c => c.name) - -module.exports = class Channel { - constructor(data) { - this.data = data - this.raw = data.raw - this.tvg = data.tvg - this.http = data.http - this.url = data.url - this.logo = data.tvg.logo - this.group = data.group - this.name = this.parseName(data.name) - this.status = this.parseStatus(data.name) - this.resolution = this.parseResolution(data.name) - this.category = this.parseCategory(data.group.title) - this.countries = this.parseCountries(data.tvg.country) - this.languages = this.parseLanguages(data.tvg.language) - this.hash = this.generateHash() - } - - generateHash() { - return `${this.tvg.id}:${this.tvg.country}:${this.tvg.language}:${this.logo}:${this.group.title}:${this.name}`.toLowerCase() - } - - updateUrl(url) { - this.url = url - this.data.url = url - } - - parseName(title) { - return title - .trim() - .split(' ') - .map(s => s.trim()) - .filter(s => { - return !/\[|\]/i.test(s) && !/\((\d+)P\)/i.test(s) - }) - .join(' ') - } - - parseStatus(title) { - const match = title.match(/\[(.*)\]/i) - return match ? match[1] : null - } - - parseResolution(title) { - const match = title.match(/\((\d+)P\)/i) - const height = match ? parseInt(match[1]) : null - - return { width: null, height } - } - - parseCategory(string) { - const category = categories.find(c => c.id === string.toLowerCase()) - if (!category) return '' - - return category.name - } - - parseCountries(string) { - const list = string.split(';') - return list - .reduce((acc, curr) => { - const codes = utils.region2codes(curr) - if (codes.length) { - for (let code of codes) { - if (!acc.includes(code)) { - acc.push(code) - } - } - } else { - acc.push(curr) - } - - return acc - }, []) - .map(code => { - const name = code ? utils.code2name(code) : null - if (!name) return null - - return { code: code.toLowerCase(), name } - }) - .filter(c => c) - } - - parseLanguages(string) { - const list = string.split(';') - return list - .map(name => { - const code = name ? utils.language2code(name) : null - if (!code) return null - - return { code, name } - }) - .filter(l => l) - } - - isSFW() { - return sfwCategories.includes(this.category) - } - - isNSFW() { - return nsfwCategories.includes(this.category) - } - - getInfo() { - let info = `-1 tvg-id="${this.tvg.id}" tvg-country="${this.tvg.country || ''}" tvg-language="${ - this.tvg.language || '' - }" tvg-logo="${this.logo || ''}"` - - if (this.http['user-agent']) { - info += ` user-agent="${this.http['user-agent']}"` - } - - info += ` group-title="${this.group.title || ''}",${this.name}` - - if (this.resolution.height) { - info += ` (${this.resolution.height}p)` - } - - if (this.status) { - info += ` [${this.status}]` - } - - if (this.http['referrer']) { - info += `\n#EXTVLCOPT:http-referrer=${this.http['referrer']}` - } - - if (this.http['user-agent']) { - info += `\n#EXTVLCOPT:http-user-agent=${this.http['user-agent']}` - } - - return info - } - - toString(raw = false) { - if (raw) return this.raw + '\n' - - return '#EXTINF:' + this.getInfo() + '\n' + this.url + '\n' - } - - toObject() { - return { - name: this.name, - logo: this.logo || null, - url: this.url, - category: this.category || null, - languages: this.languages, - countries: this.countries, - tvg: { - id: this.tvg.id || null, - name: this.tvg.name || this.name.replace(/\"/gi, ''), - url: this.tvg.url || null - } - } - } -} diff --git a/scripts/helpers/Playlist.js b/scripts/helpers/Playlist.js deleted file mode 100644 index 0df181a2d..000000000 --- a/scripts/helpers/Playlist.js +++ /dev/null @@ -1,41 +0,0 @@ -const Channel = require('./Channel') -const file = require('./file') - -module.exports = class Playlist { - constructor({ header, items, url, name, country }) { - this.url = url - this.name = name - this.country = country - this.header = header - this.channels = items.map(item => new Channel(item)).filter(channel => channel.url) - this.updated = false - } - - getHeader() { - let header = ['#EXTM3U'] - for (let key in this.header.attrs) { - let value = this.header.attrs[key] - if (value) { - header.push(`${key}="${value}"`) - } - } - - return header.join(' ') - } - - toString(options = {}) { - const config = { raw: false, ...options } - let output = `${this.getHeader()}\n` - for (let channel of this.channels) { - output += channel.toString(config.raw) - } - - return output - } - - save() { - if (this.updated) { - file.create(this.url, this.toString()) - } - } -} diff --git a/scripts/helpers/db.js b/scripts/helpers/db.js deleted file mode 100644 index 52587ce38..000000000 --- a/scripts/helpers/db.js +++ /dev/null @@ -1,237 +0,0 @@ -const categories = require('../data/categories') -const parser = require('./parser') -const utils = require('./utils') -const file = require('./file') -const epg = require('./epg') - -const db = {} - -db.load = async function () { - const files = await file.list() - const codes = await epg.codes.load() - for (const file of files) { - const playlist = await parser.parsePlaylist(file) - for (const channel of playlist.channels) { - const code = codes.find(ch => ch['tvg_id'] === channel.tvg.id) - if (code && Array.isArray(code.guides) && code.guides.length) { - channel.tvg.url = code.guides[0] - } - - db.channels.add(channel) - - for (const country of channel.countries) { - if (!db.countries.has(country)) { - db.countries.add(country) - } - } - - for (const language of channel.languages) { - if (!db.languages.has(language)) { - db.languages.add(language) - } - } - } - - db.playlists.add(playlist) - } -} - -db.channels = { - list: [], - filter: null, - duplicates: true, - offline: true, - nsfw: true, - add(channel) { - this.list.push(channel) - }, - get() { - let output - if (this.filter) { - switch (this.filter.field) { - case 'countries': - if (this.filter.value === 'undefined') { - output = this.list.filter(channel => !channel.countries.length) - } else { - output = this.list.filter(channel => - channel.countries.map(c => c.code).includes(this.filter.value) - ) - } - break - case 'languages': - if (this.filter.value === 'undefined') { - output = this.list.filter(channel => !channel.languages.length) - } else { - output = this.list.filter(channel => - channel.languages.map(c => c.code).includes(this.filter.value) - ) - } - break - case 'category': - if (this.filter.value === 'other') { - output = this.list.filter(channel => !channel.category) - } else { - output = this.list.filter( - channel => channel.category.toLowerCase() === this.filter.value - ) - } - break - } - } else { - output = this.list - } - - if (!this.duplicates) { - const buffer = [] - output = output.filter(channel => { - if (buffer.includes(channel.hash)) return false - buffer.push(channel.hash) - - return true - }) - } - - if (!this.nsfw) { - output = output.filter(channel => !channel.isNSFW()) - } - - if (!this.offline) { - output = output.filter(channel => channel.status !== 'Offline') - } - - this.nsfw = true - this.duplicates = true - this.offline = true - this.filter = null - - return output - }, - removeDuplicates() { - this.duplicates = false - - return this - }, - removeNSFW() { - this.nsfw = false - - return this - }, - removeOffline() { - this.offline = false - - return this - }, - all() { - return this.list - }, - forCountry(country) { - this.filter = { - field: 'countries', - value: country.code - } - - return this - }, - forLanguage(language) { - this.filter = { - field: 'languages', - value: language.code - } - - return this - }, - forCategory(category) { - this.filter = { - field: 'category', - value: category.id - } - - return this - }, - count() { - return this.get().length - }, - sortBy(fields, order) { - this.list = utils.sortBy(this.list, fields, order) - - return this - } -} - -db.countries = { - list: [], - has(country) { - return this.list.map(c => c.code).includes(country.code) - }, - add(country) { - this.list.push(country) - }, - all() { - return this.list - }, - count() { - return this.list.length - }, - sortBy(fields, order) { - this.list = utils.sortBy(this.list, fields, order) - - return this - } -} - -db.languages = { - list: [], - has(language) { - return this.list.map(c => c.code).includes(language.code) - }, - add(language) { - this.list.push(language) - }, - all() { - return this.list - }, - count() { - return this.list.length - }, - sortBy(fields, order) { - this.list = utils.sortBy(this.list, fields, order) - - return this - } -} - -db.categories = { - list: categories, - all() { - return this.list - }, - count() { - return this.list.length - } -} - -db.playlists = { - list: [], - add(playlist) { - this.list.push(playlist) - }, - all() { - return this.list - }, - only(list = []) { - return this.list.filter(playlist => list.includes(playlist.filename)) - }, - except(list = []) { - return this.list.filter(playlist => !list.includes(playlist.filename)) - }, - sortBy(fields, order) { - this.list = utils.sortBy(this.list, fields, order) - - return this - }, - count() { - return this.list.length - } -} - -module.exports = db diff --git a/scripts/helpers/epg.js b/scripts/helpers/epg.js deleted file mode 100644 index 202a89e89..000000000 --- a/scripts/helpers/epg.js +++ /dev/null @@ -1,12 +0,0 @@ -const axios = require('axios') - -module.exports = { - codes: { - async load() { - return await axios - .get('https://iptv-org.github.io/epg/codes.json') - .then(r => r.data) - .catch(console.log) - } - } -} diff --git a/scripts/helpers/file.js b/scripts/helpers/file.js deleted file mode 100644 index 44ff35135..000000000 --- a/scripts/helpers/file.js +++ /dev/null @@ -1,53 +0,0 @@ -const markdownInclude = require('markdown-include') -const path = require('path') -const glob = require('glob') -const fs = require('fs') - -const rootPath = path.resolve(__dirname) + '/../../' -const file = {} - -file.list = function (include = [], exclude = []) { - return new Promise(resolve => { - glob('channels/**/*.m3u', function (err, files) { - if (include.length) { - include = include.map(filename => `channels/${filename}.m3u`) - files = files.filter(filename => include.includes(filename)) - } - - if (exclude.length) { - exclude = exclude.map(filename => `channels/${filename}.m3u`) - files = files.filter(filename => !exclude.includes(filename)) - } - - resolve(files) - }) - }) -} - -file.getFilename = function (filename) { - return path.parse(filename).name -} - -file.createDir = function (dir) { - if (!fs.existsSync(dir)) { - fs.mkdirSync(dir) - } -} - -file.read = function (filename) { - return fs.readFileSync(rootPath + filename, { encoding: 'utf8' }) -} - -file.append = function (filename, data) { - fs.appendFileSync(rootPath + filename, data) -} - -file.create = function (filename, data = '') { - fs.writeFileSync(rootPath + filename, data) -} - -file.compileMarkdown = function (filename) { - markdownInclude.compileFiles(rootPath + filename) -} - -module.exports = file diff --git a/scripts/helpers/log.js b/scripts/helpers/log.js deleted file mode 100644 index 3120110b8..000000000 --- a/scripts/helpers/log.js +++ /dev/null @@ -1,17 +0,0 @@ -const log = {} - -log.print = function (message) { - if (typeof message === 'object') message = JSON.stringify(message, null, 2) - process.stdout.write(message) -} - -log.start = function () { - this.print('Starting...\n') - console.time('Done in') -} - -log.finish = function () { - console.timeEnd('Done in') -} - -module.exports = log diff --git a/scripts/helpers/parser.js b/scripts/helpers/parser.js deleted file mode 100644 index 138af5907..000000000 --- a/scripts/helpers/parser.js +++ /dev/null @@ -1,20 +0,0 @@ -const playlistParser = require('iptv-playlist-parser') -const Playlist = require('./Playlist') -const utils = require('./utils') -const file = require('./file') - -const parser = {} - -parser.parsePlaylist = async function (url) { - const content = file.read(url) - const result = playlistParser.parse(content) - const filename = file.getFilename(url) - const country = { - code: filename, - name: utils.code2name(filename) - } - - return new Playlist({ header: result.header, items: result.items, url, filename, country }) -} - -module.exports = parser diff --git a/scripts/helpers/utils.js b/scripts/helpers/utils.js deleted file mode 100644 index 0a140ff67..000000000 --- a/scripts/helpers/utils.js +++ /dev/null @@ -1,86 +0,0 @@ -const { orderBy } = require('natural-orderby') -const transliteration = require('transliteration') -const countries = require('../data/countries') -const categories = require('../data/categories') -const languages = require('../data/languages') -const regions = require('../data/regions') - -const utils = {} -const intlDisplayNames = new Intl.DisplayNames(['en'], { - style: 'narrow', - type: 'region' -}) - -utils.name2id = function (name) { - return transliteration - .transliterate(name) - .replace(/\+/gi, 'Plus') - .replace(/[^a-z\d]+/gi, '') -} - -utils.code2flag = function (code) { - code = code.toUpperCase() - switch (code) { - case 'UK': - return '🇬🇧' - case 'INT': - return '🌍' - case 'UNDEFINED': - return '' - default: - return code.replace(/./g, char => String.fromCodePoint(char.charCodeAt(0) + 127397)) - } -} - -utils.region2codes = function (region) { - region = region.toUpperCase() - - return regions[region] ? regions[region].codes : [] -} - -utils.code2name = function (code) { - try { - code = code.toUpperCase() - if (regions[code]) return regions[code].name - if (code === 'US') return 'United States' - if (code === 'INT') return 'International' - return intlDisplayNames.of(code) - } catch (e) { - return null - } -} - -utils.language2code = function (name) { - const lang = languages.find(l => l.name === name) - - return lang && lang.code ? lang.code : null -} - -utils.country2language = function (code) { - const country = countries[code.toUpperCase()] - if (!country.languages.length) return '' - const language = languages.find(l => l.code === country.languages[0]) - - return language ? language.name : '' -} - -utils.sortBy = function (arr, fields, order = null) { - fields = fields.map(field => { - if (field === 'resolution.height') return channel => channel.resolution.height || 0 - if (field === 'status') return channel => channel.status || '' - return channel => channel[field] - }) - return orderBy(arr, fields, order) -} - -utils.removeProtocol = function (string) { - return string.replace(/(^\w+:|^)\/\//, '') -} - -utils.sleep = function (ms) { - return function (x) { - return new Promise(resolve => setTimeout(() => resolve(x), ms)) - } -} - -module.exports = utils diff --git a/scripts/remove-broken-links.js b/scripts/remove-broken-links.js deleted file mode 100644 index a6fff7a85..000000000 --- a/scripts/remove-broken-links.js +++ /dev/null @@ -1,42 +0,0 @@ -const parser = require('./helpers/parser') -const file = require('./helpers/file') -const log = require('./helpers/log') - -async function main() { - log.start() - - let files = await file.list() - if (!files.length) log.print(`No files is selected\n`) - files = files.filter(file => file !== 'channels/unsorted.m3u') - for (const file of files) { - log.print(`\nProcessing '${file}'...`) - await parser - .parsePlaylist(file) - .then(removeBrokenLinks) - .then(p => p.save()) - } - - log.print('\n') - log.finish() -} - -async function removeBrokenLinks(playlist) { - const buffer = [] - const channels = playlist.channels.filter(channel => { - const sameHash = buffer.find(item => item.hash === channel.hash) - if (sameHash && channel.status === 'Offline') return false - - buffer.push(channel) - return true - }) - - if (playlist.channels.length !== channels.length) { - log.print('updated') - playlist.channels = channels - playlist.updated = true - } - - return playlist -} - -main() diff --git a/scripts/remove-duplicates.js b/scripts/remove-duplicates.js deleted file mode 100644 index c75882f70..000000000 --- a/scripts/remove-duplicates.js +++ /dev/null @@ -1,81 +0,0 @@ -const parser = require('./helpers/parser') -const utils = require('./helpers/utils') -const file = require('./helpers/file') -const log = require('./helpers/log') - -let globalBuffer = [] - -async function main() { - log.start() - - let files = await file.list() - if (!files.length) log.print(`No files is selected\n`) - files = files.filter(file => file !== 'channels/unsorted.m3u') - for (const file of files) { - log.print(`\nProcessing '${file}'...`) - await parser - .parsePlaylist(file) - .then(addToGlobalBuffer) - .then(removeDuplicates) - .then(p => p.save()) - } - - if (files.length) { - log.print(`\nProcessing 'channels/unsorted.m3u'...`) - await parser - .parsePlaylist('channels/unsorted.m3u') - .then(removeDuplicates) - .then(removeGlobalDuplicates) - .then(p => p.save()) - } - - log.print('\n') - log.finish() -} - -async function addToGlobalBuffer(playlist) { - playlist.channels.forEach(channel => { - const url = utils.removeProtocol(channel.url) - globalBuffer.push(url) - }) - - return playlist -} - -async function removeDuplicates(playlist) { - const buffer = [] - const channels = playlist.channels.filter(channel => { - const sameUrl = buffer.find(item => { - return utils.removeProtocol(item.url) === utils.removeProtocol(channel.url) - }) - if (sameUrl) return false - - buffer.push(channel) - return true - }) - - if (playlist.channels.length !== channels.length) { - log.print('updated') - playlist.channels = channels - playlist.updated = true - } - - return playlist -} - -async function removeGlobalDuplicates(playlist) { - const channels = playlist.channels.filter(channel => { - const url = utils.removeProtocol(channel.url) - return !globalBuffer.includes(url) - }) - - if (channels.length !== playlist.channels.length) { - log.print('updated') - playlist.channels = channels - playlist.updated = true - } - - return playlist -} - -main() diff --git a/scripts/sort.js b/scripts/sort.js deleted file mode 100644 index c199bd5d1..000000000 --- a/scripts/sort.js +++ /dev/null @@ -1,41 +0,0 @@ -const parser = require('./helpers/parser') -const utils = require('./helpers/utils') -const file = require('./helpers/file') -const log = require('./helpers/log') - -async function main() { - log.start() - - let files = await file.list() - if (!files.length) log.print(`No files is selected\n`) - files = files.filter(file => file !== 'channels/unsorted.m3u') - for (const file of files) { - log.print(`\nProcessing '${file}'...`) - await parser - .parsePlaylist(file) - .then(sortChannels) - .then(p => p.save()) - } - - log.print('\n') - log.finish() -} - -async function sortChannels(playlist) { - let channels = [...playlist.channels] - channels = utils.sortBy( - channels, - ['name', 'status', 'resolution.height', 'url'], - ['asc', 'asc', 'desc', 'asc'] - ) - - if (JSON.stringify(channels) !== JSON.stringify(playlist.channels)) { - log.print('updated') - playlist.channels = channels - playlist.updated = true - } - - return playlist -} - -main() diff --git a/scripts/update-readme.js b/scripts/update-readme.js deleted file mode 100644 index 5fa324039..000000000 --- a/scripts/update-readme.js +++ /dev/null @@ -1,142 +0,0 @@ -const utils = require('./helpers/utils') -const file = require('./helpers/file') -const log = require('./helpers/log') -const db = require('./helpers/db') - -async function main() { - log.start() - await loadDatabase() - generateCategoriesTable() - generateCountriesTable() - generateLanguagesTable() - generateReadme() - log.finish() -} - -async function loadDatabase() { - log.print('Loading database...\n') - await db.load() -} - -function generateCategoriesTable() { - log.print('Generating categories table...\n') - - const categories = [] - for (const category of [...db.categories.all(), { name: 'Other', id: 'other' }]) { - categories.push({ - category: category.name, - channels: db.channels.forCategory(category).removeOffline().removeDuplicates().count(), - playlist: `https://iptv-org.github.io/iptv/categories/${category.id}.m3u` - }) - } - - const table = generateTable(categories, { - columns: [ - { name: 'Category', align: 'left' }, - { name: 'Channels', align: 'right' }, - { name: 'Playlist', align: 'left' } - ] - }) - - file.create('./.readme/_categories.md', table) -} - -function generateCountriesTable() { - log.print('Generating countries table...\n') - - const countries = [] - for (const country of [ - ...db.countries.sortBy(['name']).all(), - { name: 'Undefined', code: 'undefined' } - ]) { - let flag = utils.code2flag(country.code) - const prefix = flag ? `${flag} ` : '' - countries.push({ - country: prefix + country.name, - channels: db.channels - .forCountry(country) - .removeOffline() - .removeDuplicates() - .removeNSFW() - .count(), - playlist: `https://iptv-org.github.io/iptv/countries/${country.code}.m3u` - }) - } - - const table = generateTable(countries, { - columns: [ - { name: 'Country', align: 'left' }, - { name: 'Channels', align: 'right' }, - { name: 'Playlist', align: 'left', nowrap: true } - ] - }) - - file.create('./.readme/_countries.md', table) -} - -function generateLanguagesTable() { - log.print('Generating languages table...\n') - const languages = [] - - for (const language of [ - ...db.languages.sortBy(['name']).all(), - { name: 'Undefined', code: 'undefined' } - ]) { - languages.push({ - language: language.name, - channels: db.channels - .forLanguage(language) - .removeOffline() - .removeDuplicates() - .removeNSFW() - .count(), - playlist: `https://iptv-org.github.io/iptv/languages/${language.code}.m3u` - }) - } - - const table = generateTable(languages, { - columns: [ - { name: 'Language', align: 'left' }, - { name: 'Channels', align: 'right' }, - { name: 'Playlist', align: 'left' } - ] - }) - - file.create('./.readme/_languages.md', table) -} - -function generateTable(data, options) { - let output = '\n' - - output += '\t\n\t\t' - for (let column of options.columns) { - output += `` - } - output += '\n\t\n' - - output += '\t\n' - for (let item of data) { - output += '\t\t' - let i = 0 - for (let prop in item) { - const column = options.columns[i] - let nowrap = column.nowrap - let align = column.align - output += `` - i++ - } - output += '\n' - } - output += '\t\n' - - output += '
${column.name}
${item[prop]}
' - - return output -} - -function generateReadme() { - log.print('Generating README.md...\n') - file.compileMarkdown('.readme/config.json') -} - -main()