iptv-database/scripts/db/update.ts

199 lines
6.2 KiB
TypeScript
Raw Normal View History

2023-10-16 07:45:12 -04:00
import { CSV, IssueLoader, CSVParser, IDCreator, Issue, IssueData } from '../core'
import { Channel, Blocked } from '../models'
2023-10-06 22:14:21 -04:00
import { DATA_DIR } from '../constants'
import { Storage, Collection } from '@freearhey/core'
let blocklist = new Collection()
let channels = new Collection()
const processedIssues = new Collection()
async function main() {
2023-10-07 19:13:58 -04:00
const idCreator = new IDCreator()
2023-10-06 22:14:21 -04:00
const dataStorage = new Storage(DATA_DIR)
const parser = new CSVParser()
const _channels = await dataStorage.load('channels.csv')
channels = (await parser.parse(_channels)).map(data => new Channel(data))
const _blocklist = await dataStorage.load('blocklist.csv')
blocklist = (await parser.parse(_blocklist)).map(data => new Blocked(data))
const loader = new IssueLoader()
await removeChannels({ loader })
2023-10-07 19:13:58 -04:00
await editChannels({ loader, idCreator })
await addChannels({ loader, idCreator })
2023-10-06 22:14:21 -04:00
await blockChannels({ loader })
await unblockChannels({ loader })
2023-10-06 22:57:36 -04:00
channels = sortBy(channels, 'id')
2023-10-06 22:14:21 -04:00
const channelsOutput = new CSV({ items: channels }).toString()
await dataStorage.save('channels.csv', channelsOutput)
2023-10-06 22:57:36 -04:00
blocklist = sortBy(blocklist, 'channel')
2023-10-06 22:14:21 -04:00
const blocklistOutput = new CSV({ items: blocklist }).toString()
await dataStorage.save('blocklist.csv', blocklistOutput)
const output = processedIssues.map((issue: Issue) => `closes #${issue.number}`).join(', ')
process.stdout.write(`OUTPUT=${output}`)
}
main()
2023-10-06 22:57:36 -04:00
function sortBy(channels: Collection, key: string) {
const items = channels.all().sort((a, b) => {
const normA = a[key].toLowerCase()
const normB = b[key].toLowerCase()
if (normA < normB) return -1
if (normA > normB) return 1
return 0
})
return new Collection(items)
}
2023-10-06 22:14:21 -04:00
async function removeChannels({ loader }: { loader: IssueLoader }) {
const issues = await loader.load({ labels: ['channels:remove,approved'] })
issues.forEach((issue: Issue) => {
if (issue.data.missing('channel_id')) return
2023-10-16 07:45:12 -04:00
const found = channels.first(
(channel: Channel) => channel.id === issue.data.getString('channel_id')
)
2023-10-06 22:14:21 -04:00
if (!found) return
channels.remove((channel: Channel) => channel.id === found.id)
processedIssues.push(issue)
})
}
2023-10-07 19:13:58 -04:00
async function editChannels({ loader, idCreator }: { loader: IssueLoader; idCreator: IDCreator }) {
2023-10-06 22:14:21 -04:00
const issues = await loader.load({ labels: ['channels:edit,approved'] })
issues.forEach((issue: Issue) => {
2023-10-16 07:45:12 -04:00
const data: IssueData = issue.data
2023-10-06 22:14:21 -04:00
if (data.missing('channel_id')) return
const found: Channel = channels.first(
2023-10-16 07:45:12 -04:00
(channel: Channel) => channel.id === data.getString('channel_id')
2023-10-06 22:14:21 -04:00
)
if (!found) return
let channelId = found.id
if (data.has('name') || data.has('country')) {
2023-10-16 07:45:12 -04:00
const name = data.getString('name') || found.name
const country = data.getString('country') || found.country
if (name && country) {
channelId = idCreator.create(name, country)
}
2023-10-06 22:14:21 -04:00
}
2023-10-12 02:07:13 -04:00
const updated = new Channel({
2023-10-06 22:14:21 -04:00
id: channelId,
2023-10-16 07:45:12 -04:00
name: data.getString('name'),
alt_names: data.getArray('alt_names'),
network: data.getString('network'),
owners: data.getArray('owners'),
country: data.getString('country'),
subdivision: data.getString('subdivision'),
city: data.getString('city'),
broadcast_area: data.getArray('broadcast_area'),
languages: data.getArray('languages'),
categories: data.getArray('categories'),
is_nsfw: data.getBoolean('is_nsfw'),
launched: data.getString('launched'),
closed: data.getString('closed'),
replaced_by: data.getString('replaced_by'),
website: data.getString('website'),
logo: data.getString('logo')
2023-10-06 22:14:21 -04:00
})
2023-10-12 02:07:13 -04:00
found.merge(updated)
2023-10-06 22:14:21 -04:00
processedIssues.push(issue)
})
}
2023-10-07 19:13:58 -04:00
async function addChannels({ loader, idCreator }: { loader: IssueLoader; idCreator: IDCreator }) {
2023-10-06 22:14:21 -04:00
const issues = await loader.load({ labels: ['channels:add,approved'] })
issues.forEach((issue: Issue) => {
2023-10-16 07:45:12 -04:00
const data: IssueData = issue.data
const name = data.getString('name')
const country = data.getString('country')
2023-10-06 22:14:21 -04:00
2023-10-16 07:45:12 -04:00
if (!name || !country) return
const channelId = idCreator.create(name, country)
2023-10-06 22:14:21 -04:00
const found: Channel = channels.first((channel: Channel) => channel.id === channelId)
if (found) return
channels.push(
new Channel({
id: channelId,
2023-10-16 07:45:12 -04:00
name: data.getString('name'),
alt_names: data.getArray('alt_names'),
network: data.getString('network'),
owners: data.getArray('owners'),
country: data.getString('country'),
subdivision: data.getString('subdivision'),
city: data.getString('city'),
broadcast_area: data.getArray('broadcast_area'),
languages: data.getArray('languages'),
categories: data.getArray('categories'),
is_nsfw: data.getBoolean('is_nsfw'),
launched: data.getString('launched'),
closed: data.getString('closed'),
replaced_by: data.getString('replaced_by'),
website: data.getString('website'),
logo: data.getString('logo')
2023-10-06 22:14:21 -04:00
})
)
processedIssues.push(issue)
})
}
async function unblockChannels({ loader }: { loader: IssueLoader }) {
const issues = await loader.load({ labels: ['blocklist:remove,approved'] })
issues.forEach((issue: Issue) => {
const data = issue.data
if (data.missing('channel_id')) return
const found: Blocked = blocklist.first(
2023-10-16 07:45:12 -04:00
(blocked: Blocked) => blocked.channel === data.getString('channel_id')
2023-10-06 22:14:21 -04:00
)
if (!found) return
blocklist.remove((blocked: Blocked) => blocked.channel === found.channel)
processedIssues.push(issue)
})
}
async function blockChannels({ loader }: { loader: IssueLoader }) {
const issues = await loader.load({ labels: ['blocklist:add,approved'] })
issues.forEach((issue: Issue) => {
const data = issue.data
if (data.missing('channel_id')) return
const found: Blocked = blocklist.first(
2023-10-16 07:45:12 -04:00
(blocked: Blocked) => blocked.channel === data.getString('channel_id')
2023-10-06 22:14:21 -04:00
)
if (found) return
2023-10-16 07:45:12 -04:00
const channel = data.getString('channel_id')
const ref = data.getString('ref')
if (!channel || !ref) return
2023-10-06 22:14:21 -04:00
blocklist.push(
new Blocked({
2023-10-16 07:45:12 -04:00
channel,
ref
2023-10-06 22:14:21 -04:00
})
)
processedIssues.push(issue)
})
}