Merge branch 'master' of https://github.com/AntiPontifex/database

.github/ISSUE_TEMPLATE/__channels_add.yml

@@ -13,7 +13,7 @@ body:
     id: name
     attributes:
       label: Channel Name
-      description: "Official channel name in English or call sign. May include: `a-z`, `0-9`, `space`, `-`, `!`, `:`, `&`, `.`, `+`, `'`, `/`, `»`, `#`, `%`, `°`, `$`, `@`, `?`"
+      description: "Official channel name in English or call sign. May include: `a-z`, `0-9`, `space`, `-`, `!`, `:`, `&`, `.`, `+`, `'`, `/`, `»`, `#`, `%`, `°`, `$`, `@`, `?`, `|`"
       placeholder: 'Anhui TV'
     validations:
       required: true

.github/ISSUE_TEMPLATE/__channels_edit.yml

@@ -22,7 +22,7 @@ body:
     id: name
     attributes:
       label: Channel Name
-      description: "Official channel name in English or call sign. May include: `a-z`, `0-9`, `space`, `-`, `!`, `:`, `&`, `.`, `+`, `'`, `/`, `»`, `#`, `%`, `°`, `$`, `@`, `?`"
+      description: "Official channel name in English or call sign. May include: `a-z`, `0-9`, `space`, `-`, `!`, `:`, `&`, `.`, `+`, `'`, `/`, `»`, `#`, `%`, `°`, `$`, `@`, `?`, `|`"
       placeholder: 'Anhui TV'
 
   - type: input
@@ -96,8 +96,6 @@ body:
       options:
         - 'FALSE'
         - 'TRUE'
-    validations:
-      required: true
 
   - type: input
     id: launched

CONTRIBUTING.md

@@ -11,9 +11,9 @@
 ### channels
 
 | Field          | Description                                                                                                                                                                                                                                  | Required | Example                        |
-| -------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | ------------------------------ |
+| -------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | ------------------------------ | ---------- |
 | id             | Unique channel ID derived from the `name` and `country` separated by dot. May only contain Latin letters, numbers and dot.                                                                                                                   | Required | `AnhuiTV.cn`                   |
-| name           | Official channel name in English or call sign. May include: `a-z`, `0-9`, `space`, `-`, `!`, `:`, `&`, `.`, `+`, `'`, `/`, `»`, `#`, `%`, `°`, `$`, `@`, `?`.                                                                                | Required | `Anhui TV`                     |
+| name           | Official channel name in English or call sign. May include: `a-z`, `0-9`, `space`, `-`, `!`, `:`, `&`, `.`, `+`, `'`, `/`, `»`, `#`, `%`, `°`, `$`, `@`, `?`, `                                                                              | `.       | Required                       | `Anhui TV` |
 | alt_names      | List of alternative channel names separated by `;`. May contain any characters except `,` and `"`.                                                                                                                                           | Optional | `安徽卫视;AHTV`                |
 | network        | Network of which this channel is a part. May contain any characters except `,` and `"`.                                                                                                                                                      | Optional | `Anhui`                        |
 | owners         | List of channel owners separated by `;`. May contain any characters except `,` and `"`.                                                                                                                                                      | Optional | `China Central Television`     |

scripts/core/index.ts

@@ -3,3 +3,5 @@ export * from './issueParser'
 export * from './issueLoader'
 export * from './csvParser'
 export * from './idCreator'
+export * from './issueData'
+export * from './issue'

scripts/core/issue.ts

@@ -1,15 +1,15 @@
-import { Dictionary } from '@freearhey/core'
+import { IssueData } from './'
 
 type IssueProps = {
   number: number
   labels: string[]
-  data: Dictionary
+  data: IssueData
 }
 
 export class Issue {
   number: number
   labels: string[]
-  data: Dictionary
+  data: IssueData
 
   constructor({ number, labels, data }: IssueProps) {
     this.number = number

scripts/core/issueData.ts

@@ -0,0 +1,40 @@
+import { Dictionary } from '@freearhey/core'
+
+export class IssueData {
+  _data: Dictionary
+  constructor(data: Dictionary) {
+    this._data = data
+  }
+
+  has(key: string): boolean {
+    return this._data.has(key)
+  }
+
+  missing(key: string): boolean {
+    return this._data.missing(key) || this._data.get(key) === undefined
+  }
+
+  getBoolean(key: string): boolean | undefined {
+    return this.missing(key) ? undefined : this._data.get(key)
+  }
+
+  getString(key: string): string | undefined {
+    const deleteSymbol = '~'
+
+    return this.missing(key)
+      ? undefined
+      : this._data.get(key) === deleteSymbol
+      ? ''
+      : this._data.get(key)
+  }
+
+  getArray(key: string): string[] | undefined {
+    const deleteSymbol = '~'
+
+    return this.missing(key)
+      ? undefined
+      : this._data.get(key) === deleteSymbol
+      ? []
+      : this._data.get(key).split(';')
+  }
+}

scripts/core/issueParser.ts

@@ -1,5 +1,5 @@
 import { Dictionary } from '@freearhey/core'
-import { Issue } from '../models'
+import { IssueData, Issue } from '../core'
 
 const FIELDS = new Dictionary({
   'Channel ID': 'channel_id',
@@ -52,7 +52,8 @@ export class IssueParser {
       if (!_label || !_value) return data
 
       const id: string = FIELDS.get(_label)
-      const value: string = _value === '_No response_' || _value === 'None' ? '' : _value
+      const value: string | undefined =
+        _value === '_No response_' || _value === 'None' ? undefined : _value
 
       if (!id) return
 
@@ -61,6 +62,6 @@ export class IssueParser {
 
     const labels = issue.labels.map(label => label.name)
 
-    return new Issue({ number: issue.number, labels, data })
+    return new Issue({ number: issue.number, labels, data: new IssueData(data) })
   }
 }

scripts/db/update.ts

@@ -1,5 +1,5 @@
-import { CSV, IssueLoader, CSVParser, IDCreator } from '../core'
+import { CSV, IssueLoader, CSVParser, IDCreator, Issue, IssueData } from '../core'
-import { Channel, Blocked, Issue } from '../models'
+import { Channel, Blocked } from '../models'
 import { DATA_DIR } from '../constants'
 import { Storage, Collection } from '@freearhey/core'
 
@@ -57,7 +57,9 @@ async function removeChannels({ loader }: { loader: IssueLoader }) {
   issues.forEach((issue: Issue) => {
     if (issue.data.missing('channel_id')) return
 
-    const found = channels.first((channel: Channel) => channel.id === issue.data.get('channel_id'))
+    const found = channels.first(
+      (channel: Channel) => channel.id === issue.data.getString('channel_id')
+    )
     if (!found) return
 
     channels.remove((channel: Channel) => channel.id === found.id)
@@ -69,55 +71,41 @@ async function removeChannels({ loader }: { loader: IssueLoader }) {
 async function editChannels({ loader, idCreator }: { loader: IssueLoader; idCreator: IDCreator }) {
   const issues = await loader.load({ labels: ['channels:edit,approved'] })
   issues.forEach((issue: Issue) => {
-    const data = issue.data
+    const data: IssueData = issue.data
     if (data.missing('channel_id')) return
 
     const found: Channel = channels.first(
-      (channel: Channel) => channel.id === data.get('channel_id')
+      (channel: Channel) => channel.id === data.getString('channel_id')
     )
     if (!found) return
 
     let channelId = found.id
     if (data.has('name') || data.has('country')) {
-      const name = data.get('name') || found.name
+      const name = data.getString('name') || found.name
-      const country = data.get('country') || found.country
+      const country = data.getString('country') || found.country
+      if (name && country) {
         channelId = idCreator.create(name, country)
       }
+    }
 
-    const deleteSymbol = '~'
     const updated = new Channel({
       id: channelId,
-      name: data.get('name') !== deleteSymbol ? data.get('name') : '',
+      name: data.getString('name'),
-      alt_names:
+      alt_names: data.getArray('alt_names'),
-        data.has('alt_names') && data.get('alt_names') !== deleteSymbol
+      network: data.getString('network'),
-          ? data.get('alt_names').split(';')
+      owners: data.getArray('owners'),
-          : [],
+      country: data.getString('country'),
-      network: data.get('network') !== deleteSymbol ? data.get('network') : '',
+      subdivision: data.getString('subdivision'),
-      owners:
+      city: data.getString('city'),
-        data.has('owners') && data.get('owners') !== deleteSymbol
+      broadcast_area: data.getArray('broadcast_area'),
-          ? data.get('owners').split(';')
+      languages: data.getArray('languages'),
-          : [],
+      categories: data.getArray('categories'),
-      country: data.get('country') !== deleteSymbol ? data.get('country') : '',
+      is_nsfw: data.getBoolean('is_nsfw'),
-      subdivision: data.get('subdivision') !== deleteSymbol ? data.get('subdivision') : '',
+      launched: data.getString('launched'),
-      city: data.get('city') !== deleteSymbol ? data.get('city') : '',
+      closed: data.getString('closed'),
-      broadcast_area:
+      replaced_by: data.getString('replaced_by'),
-        data.has('broadcast_area') && data.get('broadcast_area') !== deleteSymbol
+      website: data.getString('website'),
-          ? data.get('broadcast_area').split(';')
+      logo: data.getString('logo')
-          : [],
-      languages:
-        data.has('languages') && data.get('languages') !== deleteSymbol
-          ? data.get('languages').split(';')
-          : [],
-      categories:
-        data.has('categories') && data.get('categories') !== deleteSymbol
-          ? data.get('categories').split(';')
-          : [],
-      is_nsfw: data.has('is_nsfw') ? data.get('is_nsfw') === 'TRUE' : false,
-      launched: data.get('launched') !== deleteSymbol ? data.get('launched') : '',
-      closed: data.get('closed') !== deleteSymbol ? data.get('closed') : '',
-      replaced_by: data.get('replaced_by') !== deleteSymbol ? data.get('replaced_by') : '',
-      website: data.get('website') !== deleteSymbol ? data.get('website') : '',
-      logo: data.get('logo') !== deleteSymbol ? data.get('logo') : ''
     })
 
     found.merge(updated)
@@ -129,10 +117,13 @@ async function editChannels({ loader, idCreator }: { loader: IssueLoader; idCrea
 async function addChannels({ loader, idCreator }: { loader: IssueLoader; idCreator: IDCreator }) {
   const issues = await loader.load({ labels: ['channels:add,approved'] })
   issues.forEach((issue: Issue) => {
-    const data = issue.data
+    const data: IssueData = issue.data
-    if (data.missing('name') || data.missing('country')) return
+    const name = data.getString('name')
+    const country = data.getString('country')
 
-    const channelId = idCreator.create(data.get('name'), data.get('country'))
+    if (!name || !country) return
+
+    const channelId = idCreator.create(name, country)
 
     const found: Channel = channels.first((channel: Channel) => channel.id === channelId)
     if (found) return
@@ -140,22 +131,22 @@ async function addChannels({ loader, idCreator }: { loader: IssueLoader; idCreat
     channels.push(
       new Channel({
         id: channelId,
-        name: data.get('name'),
+        name: data.getString('name'),
-        alt_names: data.get('alt_names'),
+        alt_names: data.getArray('alt_names'),
-        network: data.get('network'),
+        network: data.getString('network'),
-        owners: data.get('owners'),
+        owners: data.getArray('owners'),
-        country: data.get('country'),
+        country: data.getString('country'),
-        subdivision: data.get('subdivision'),
+        subdivision: data.getString('subdivision'),
-        city: data.get('city'),
+        city: data.getString('city'),
-        broadcast_area: data.get('broadcast_area'),
+        broadcast_area: data.getArray('broadcast_area'),
-        languages: data.get('languages'),
+        languages: data.getArray('languages'),
-        categories: data.get('categories'),
+        categories: data.getArray('categories'),
-        is_nsfw: data.get('is_nsfw'),
+        is_nsfw: data.getBoolean('is_nsfw'),
-        launched: data.get('launched'),
+        launched: data.getString('launched'),
-        closed: data.get('closed'),
+        closed: data.getString('closed'),
-        replaced_by: data.get('replaced_by'),
+        replaced_by: data.getString('replaced_by'),
-        website: data.get('website'),
+        website: data.getString('website'),
-        logo: data.get('logo')
+        logo: data.getString('logo')
       })
     )
 
@@ -170,7 +161,7 @@ async function unblockChannels({ loader }: { loader: IssueLoader }) {
     if (data.missing('channel_id')) return
 
     const found: Blocked = blocklist.first(
-      (blocked: Blocked) => blocked.channel === data.get('channel_id')
+      (blocked: Blocked) => blocked.channel === data.getString('channel_id')
     )
     if (!found) return
 
@@ -187,14 +178,18 @@ async function blockChannels({ loader }: { loader: IssueLoader }) {
     if (data.missing('channel_id')) return
 
     const found: Blocked = blocklist.first(
-      (blocked: Blocked) => blocked.channel === data.get('channel_id')
+      (blocked: Blocked) => blocked.channel === data.getString('channel_id')
     )
     if (found) return
 
+    const channel = data.getString('channel_id')
+    const ref = data.getString('ref')
+    if (!channel || !ref) return
+
     blocklist.push(
       new Blocked({
-        channel: data.get('channel_id'),
+        channel,
-        ref: data.get('ref')
+        ref
       })
     )
 

scripts/models/channel.ts

@@ -1,41 +1,41 @@
 type ChannelProps = {
   id: string
-  name: string
+  name?: string
-  alt_names: string[]
+  alt_names?: string[]
-  network: string
+  network?: string
-  owners: string[]
+  owners?: string[]
-  country: string
+  country?: string
-  subdivision: string
+  subdivision?: string
-  city: string
+  city?: string
-  broadcast_area: string[]
+  broadcast_area?: string[]
-  languages: string[]
+  languages?: string[]
-  categories: string[]
+  categories?: string[]
-  is_nsfw: boolean
+  is_nsfw?: boolean
-  launched: string
+  launched?: string
-  closed: string
+  closed?: string
-  replaced_by: string
+  replaced_by?: string
-  website: string
+  website?: string
-  logo: string
+  logo?: string
 }
 
 export class Channel {
   id: string
-  name: string
+  name?: string
-  alt_names: string[]
+  alt_names?: string[]
-  network: string
+  network?: string
-  owners: string[]
+  owners?: string[]
-  country: string
+  country?: string
-  subdivision: string
+  subdivision?: string
-  city: string
+  city?: string
-  broadcast_area: string[]
+  broadcast_area?: string[]
-  languages: string[]
+  languages?: string[]
-  categories: string[]
+  categories?: string[]
-  is_nsfw: boolean
+  is_nsfw?: boolean
-  launched: string
+  launched?: string
-  closed: string
+  closed?: string
-  replaced_by: string
+  replaced_by?: string
-  website: string
+  website?: string
-  logo: string
+  logo?: string
 
   constructor({
     id,
@@ -82,10 +82,9 @@ export class Channel {
   }
 
   merge(channel: Channel) {
-    const data: { [key: string]: string | string[] | boolean } = channel.data()
+    const data: { [key: string]: string | string[] | boolean | undefined } = channel.data()
     for (const prop in data) {
       if (data[prop] === undefined) continue
-      if (Array.isArray(data[prop]) && !(data[prop] as string[]).length) continue
       this[prop] = data[prop]
     }
   }

scripts/models/index.ts

@@ -1,3 +1,2 @@
 export * from './channel'
-export * from './issue'
 export * from './blocked'

scripts/schemes/channels.js

@@ -7,7 +7,7 @@ module.exports = {
     .regex(/^[A-Za-z0-9]+\.[a-z]{2}$/)
     .required(),
   name: Joi.string()
-    .regex(/^[a-z0-9-!:&.+'/»#%°$@?\s]+$/i)
+    .regex(/^[a-z0-9-!:&.+'/»#%°$@?|\s]+$/i)
     .required(),
   alt_names: Joi.array().items(
     Joi.string()

tests/__data__/expected/api/channels.json

@@ -1 +1 @@
-[{"id":"002RadioTV.do","name":"002 Radio TV","alt_names":[],"network":null,"owners":[],"country":"DO","subdivision":null,"city":null,"broadcast_area":["c/DO"],"languages":["spa"],"categories":["general"],"is_nsfw":false,"launched":null,"closed":null,"replaced_by":null,"website":"https://www.002radio.com/","logo":"https://i.imgur.com/7oNe8xj.png"},{"id":"BeijingSatelliteTV.cn","name":"Beijing Satellite TV","alt_names":["北京卫视"],"network":null,"owners":[],"country":"CN","subdivision":null,"city":"Beijing","broadcast_area":["c/CN"],"languages":["zho"],"categories":["general"],"is_nsfw":false,"launched":"1979-05-16","closed":null,"replaced_by":null,"website":"https://www.brtn.cn/btv/","logo":"https://i.imgur.com/vsktAez.png"},{"id":"M5.hu","name":"M5","alt_names":[],"network":null,"owners":[],"country":"HU","subdivision":null,"city":null,"broadcast_area":["c/HU"],"languages":["hun"],"categories":[],"is_nsfw":false,"launched":null,"closed":"2001-01-01","replaced_by":null,"website":"https://www.mediaklikk.hu/m5/","logo":"https://i.imgur.com/y21wFd0.png"}]
+[{"id":"002RadioTV.do","name":"002 Radio TV","alt_names":[],"network":null,"owners":[],"country":"DO","subdivision":null,"city":null,"broadcast_area":["c/DO"],"languages":["spa"],"categories":["general"],"is_nsfw":false,"launched":null,"closed":null,"replaced_by":null,"website":"https://www.002radio.com/","logo":"https://i.imgur.com/7oNe8xj.png"},{"id":"BeijingSatelliteTV.cn","name":"Beijing Satellite TV","alt_names":["北京卫视"],"network":null,"owners":[],"country":"CN","subdivision":null,"city":"Beijing","broadcast_area":["c/CN"],"languages":["zho"],"categories":["general"],"is_nsfw":false,"launched":"1979-05-16","closed":null,"replaced_by":null,"website":"https://www.brtn.cn/btv/","logo":"https://i.imgur.com/vsktAez.png"},{"id":"M5.hu","name":"M5","alt_names":[],"network":null,"owners":[],"country":"HU","subdivision":null,"city":null,"broadcast_area":["c/HU"],"languages":["hun"],"categories":["auto"],"is_nsfw":true,"launched":null,"closed":"2001-01-01","replaced_by":null,"website":"https://www.mediaklikk.hu/m5/","logo":"https://i.imgur.com/y21wFd0.png"}]

tests/__data__/expected/data/channels.csv

@@ -1,6 +1,6 @@
 id,name,alt_names,network,owners,country,subdivision,city,broadcast_area,languages,categories,is_nsfw,launched,closed,replaced_by,website,logo
 beINMoviesTurk.tr,beIN Movies Turk,beIN Movies Türk,BBC,Gazprom Media,TR,US-CA,London,c/TR,tur,movies,FALSE,1979-05-16,1980-05-16,M5.hu,http://www.digiturk.com.tr/,https://i.imgur.com/nw8Sa2z.png
-M5.hu,M5,,,Duna Médiaszolgáltató Nonprofit Zrt.,HU,,,c/HU,hun,,FALSE,,,,https://www.mediaklikk.hu/m5/,https://i.imgur.com/y21wFd0.png
+M5.hu,M5,,,Duna Médiaszolgáltató Nonprofit Zrt.,HU,,,c/HU,hun,,TRUE,,,,https://www.mediaklikk.hu/m5/,https://i.imgur.com/y21wFd0.png
 WenzhouEconomicandEducation.cn,Wenzhou Economic and Education,,,,CN,,Wenzhou,c/CN,zho,science,FALSE,,,,,https://www.tvchinese.net/uploads/tv/wzjjkj.jpg
 YiwuBusinessChannel.cn,Yiwu Business Channel,,,,CN,,,c/CN,zho,business,FALSE,,,,,https://www.tvchinese.net/uploads/tv/yiwutv.jpg
 YiwuNewsIntegratedChannel.cn,Yiwu News Integrated Channel,,,,CN,,,c/CN,zho,news,FALSE,,,,,https://www.tvchinese.net/uploads/tv/yiwutv.jpg

tests/__data__/input/data/channels.csv

@@ -1,4 +1,4 @@
 id,name,alt_names,network,owners,country,subdivision,city,broadcast_area,languages,categories,is_nsfw,launched,closed,replaced_by,website,logo
 002RadioTV.do,002 Radio TV,,,,DO,,,c/DO,spa,general,FALSE,,,,https://www.002radio.com/,https://i.imgur.com/7oNe8xj.png
 BeijingSatelliteTV.cn,Beijing Satellite TV,北京卫视,,,CN,,Beijing,c/CN,zho,general,FALSE,1979-05-16,,,https://www.brtn.cn/btv/,https://i.imgur.com/vsktAez.png
-M5.hu,M5,,,,HU,,,c/HU,hun,,FALSE,,2001-01-01,,https://www.mediaklikk.hu/m5/,https://i.imgur.com/y21wFd0.png
+M5.hu,M5,,,,HU,,,c/HU,hun,auto,TRUE,,2001-01-01,,https://www.mediaklikk.hu/m5/,https://i.imgur.com/y21wFd0.png

tests/__data__/input/issues/channels_edit_approved.js

@@ -61,7 +61,7 @@ module.exports = [
     closed_at: null,
     author_association: 'CONTRIBUTOR',
     active_lock_reason: null,
-    body: '### Channel ID (required)\n\nM5.hu\n\n### Channel Name\n\n_No response_\n\n### Alternative Names\n\n_No response_\n\n### Network\n\n_No response_\n\n### Owners\n\nDuna Médiaszolgáltató Nonprofit Zrt.\n\n### Country\n\n_No response_\n\n### Subdivision\n\n_No response_\n\n### City\n\n_No response_\n\n### Broadcast Area\n\n_No response_\n\n### Languages\n\n_No response_\n\n### Categories\n\n_No response_\n\n### NSFW\n\nFALSE\n\n### Launched\n\n_No response_\n\n### Closed\n\n~\n\n### Replaced By\n\n_No response_\n\n### Website\n\n_No response_\n\n### Logo\n\n_No response_\n\n### Notes\n\n_No response_',
+    body: '### Channel ID (required)\n\nM5.hu\n\n### Channel Name\n\n_No response_\n\n### Alternative Names\n\n_No response_\n\n### Network\n\n_No response_\n\n### Owners\n\nDuna Médiaszolgáltató Nonprofit Zrt.\n\n### Country\n\n_No response_\n\n### Subdivision\n\n_No response_\n\n### City\n\n_No response_\n\n### Broadcast Area\n\n_No response_\n\n### Languages\n\n_No response_\n\n### Categories\n\n~\n\n### NSFW\n\nNone\n\n### Launched\n\n_No response_\n\n### Closed\n\n~\n\n### Replaced By\n\n_No response_\n\n### Website\n\n_No response_\n\n### Logo\n\n_No response_\n\n### Notes\n\n_No response_',
     reactions: {
       url: 'https://api.github.com/repos/iptv-org/database/issues/5901/reactions',
       total_count: 0,

tests/__data__/input/validate/valid_data/channels.csv

@@ -1,2 +1,3 @@
 id,name,alt_names,network,owners,country,subdivision,city,broadcast_area,languages,categories,is_nsfw,launched,closed,replaced_by,website,logo
 PeoplesWeather.do,People°s Weather,,,,DO,,,c/DO,spa,,FALSE,,,,,https://i.imgur.com/7oNe8xj.png
+KSTVKids.ua,KS TV | Kids,,,,UA,,,c/UA,ukr,,FALSE,,,,,https://i.imgur.com/CmRfj78.png

tests/__data__/input/validate/valid_data/countries.csv

@@ -1,3 +1,4 @@
 name,code,languages,flag
 Andorra,AD,cat,🇦🇩
 Dominican Republic,DO,spa,🇩🇴
+Ukraine,UA,ukr,🇺🇦

tests/__data__/input/validate/valid_data/languages.csv

@@ -1,3 +1,4 @@
 code,name
 cat,Catalan
 spa,Spanish
+ukr,Ukrainian

tests/db/validate.test.ts

@@ -79,8 +79,12 @@ describe('db:validate', () => {
   })
 
   it('does not show an error if all data are correct', () => {
+    try {
       execSync('DATA_DIR=tests/__data__/input/validate/valid_data npm run db:validate', {
         encoding: 'utf8'
       })
+    } catch (error) {
+      console.log((error as ExecError).stdout)
+    }
   })
 })