Merge branch 'master' of https://github.com/AntiPontifex/database

package-lock.json

@@ -6,7 +6,7 @@
     "": {
       "name": "@iptv-org/database",
       "dependencies": {
-        "@freearhey/core": "^0.2.1",
+        "@freearhey/core": "^0.2.2",
         "@joi/date": "^2.1.0",
         "@json2csv/formatters": "^7.0.3",
         "@json2csv/node": "^7.0.3",
@@ -798,9 +798,9 @@
       }
     },
     "node_modules/@freearhey/core": {
-      "version": "0.2.1",
-      "resolved": "https://registry.npmjs.org/@freearhey/core/-/core-0.2.1.tgz",
-      "integrity": "sha512-kEdIxZClykKhGpgyCSlkwuVuSCCAWr3J5YvOUMJQDPgVAYvT5VbD8MYKPm+OwNi9T4HFmF6qqY90qwKJPoOXCA==",
+      "version": "0.2.2",
+      "resolved": "https://registry.npmjs.org/@freearhey/core/-/core-0.2.2.tgz",
+      "integrity": "sha512-8eikSh+alXohY/0Qv8lls+6BG738vOqxGuLvMjwBasbwuz8S/Abr7fUAH65dGjPAsB3HMaDhjK5c6kcVz5GrcA==",
       "dependencies": {
         "@types/fs-extra": "^11.0.2",
         "@types/lodash": "^4.14.198",
@@ -6552,9 +6552,9 @@
       "integrity": "sha512-NCC3zz2+nvYd+Ckfh87rA47zfu2QsQpvc6k1yzTk+b9KzRj0wkGa8LSoGOXN6Zv4lRf/EIoZ80biDh9HOI+RNQ=="
     },
     "@freearhey/core": {
-      "version": "0.2.1",
-      "resolved": "https://registry.npmjs.org/@freearhey/core/-/core-0.2.1.tgz",
-      "integrity": "sha512-kEdIxZClykKhGpgyCSlkwuVuSCCAWr3J5YvOUMJQDPgVAYvT5VbD8MYKPm+OwNi9T4HFmF6qqY90qwKJPoOXCA==",
+      "version": "0.2.2",
+      "resolved": "https://registry.npmjs.org/@freearhey/core/-/core-0.2.2.tgz",
+      "integrity": "sha512-8eikSh+alXohY/0Qv8lls+6BG738vOqxGuLvMjwBasbwuz8S/Abr7fUAH65dGjPAsB3HMaDhjK5c6kcVz5GrcA==",
       "requires": {
         "@types/fs-extra": "^11.0.2",
         "@types/lodash": "^4.14.198",

package.json

@@ -22,7 +22,7 @@
     "testRegex": "tests/(.*?/)?.*test.(js|ts)$"
   },
   "dependencies": {
-    "@freearhey/core": "^0.2.1",
+    "@freearhey/core": "^0.2.2",
     "@joi/date": "^2.1.0",
     "@json2csv/formatters": "^7.0.3",
     "@json2csv/node": "^7.0.3",

scripts/db/update.ts

@@ -84,19 +84,19 @@ async function editChannels({ loader, idCreator }: { loader: IssueLoader; idCrea
       channelId = idCreator.create(name, country)
     }
 
-    found.update({
+    const updated = new Channel({
       id: channelId,
       name: data.get('name'),
-      alt_names: data.get('alt_names'),
+      alt_names: data.has('alt_names') ? data.get('alt_names').split(';') : [],
       network: data.get('network'),
-      owners: data.get('owners'),
+      owners: data.has('owners') ? data.get('owners').split(';') : [],
       country: data.get('country'),
       subdivision: data.get('subdivision'),
       city: data.get('city'),
-      broadcast_area: data.get('broadcast_area'),
-      languages: data.get('languages'),
-      categories: data.get('categories'),
-      is_nsfw: data.get('is_nsfw'),
+      broadcast_area: data.has('broadcast_area') ? data.get('broadcast_area').split(';') : [],
+      languages: data.has('languages') ? data.get('languages').split(';') : [],
+      categories: data.has('categories') ? data.get('categories').split(';') : [],
+      is_nsfw: data.has('is_nsfw') ? data.get('is_nsfw') === 'TRUE' : false,
       launched: data.get('launched'),
       closed: data.get('closed'),
       replaced_by: data.get('replaced_by'),
@@ -104,6 +104,8 @@ async function editChannels({ loader, idCreator }: { loader: IssueLoader; idCrea
       logo: data.get('logo')
     })
 
+    found.merge(updated)
+
     processedIssues.push(issue)
   })
 }

scripts/db/validate.ts

@@ -129,11 +129,11 @@ async function main() {
     }
 
     const schema = Joi.object(schemes[filename])
-    rows.forEach((row: string | string[] | boolean, i: number) => {
+    rows.all().forEach((row: { [key: string]: string }, i: number) => {
       const { error } = schema.validate(row, { abortEarly: false })
       if (error) {
         error.details.forEach(detail => {
-          fileErrors.push({ line: i + 2, message: detail.message })
+          fileErrors.push({ line: i + 2, row, message: detail.message })
         })
       }
     })
@@ -142,7 +142,8 @@ async function main() {
       logger.info(`\n${chalk.underline(filepath)}`)
       fileErrors.forEach(err => {
         const position = err.line.toString().padEnd(6, ' ')
-        logger.info(` ${chalk.gray(position)} ${err.message}`)
+        const id = err.row && err.row.id ? ` ${err.row.id}:` : ''
+        logger.info(` ${chalk.gray(position)}${id} ${err.message}`)
       })
       globalErrors = globalErrors.concat(fileErrors)
     }

scripts/models/channel.ts

@@ -75,11 +75,18 @@ export class Channel {
     this.logo = logo
   }
 
-  update(data: { [key: string]: string }) {
-    for (const key in data) {
-      if (this[key] && data[key]) {
-        this[key] = data[key]
+  data() {
+    const { ...object } = this
+
+    return object
   }
+
+  merge(channel: Channel) {
+    const data: { [key: string]: string | string[] | boolean } = channel.data()
+    for (const prop in data) {
+      if (data[prop] === undefined) continue
+      if (Array.isArray(data[prop]) && !data[prop].length) continue
+      this[prop] = data[prop]
     }
   }
 }

tests/__data__/expected/data/channels.csv

@@ -1,5 +1,5 @@
 id,name,alt_names,network,owners,country,subdivision,city,broadcast_area,languages,categories,is_nsfw,launched,closed,replaced_by,website,logo
-beINMoviesTurk.tr,beIN Movies Turk,beIN Movies Türk,,,TR,,Beijing,c/TR,tur,movies,FALSE,1979-05-16,,,http://www.digiturk.com.tr/,https://i.imgur.com/nw8Sa2z.png
+beINMoviesTurk.tr,beIN Movies Turk,beIN Movies Türk,BBC,Gazprom Media,TR,US-CA,London,c/TR,tur,movies,FALSE,1979-05-16,1980-05-16,M5.hu,http://www.digiturk.com.tr/,https://i.imgur.com/nw8Sa2z.png
 M5.hu,M5,,,Duna Médiaszolgáltató Nonprofit Zrt.,HU,,,c/HU,hun,,FALSE,,,,https://www.mediaklikk.hu/m5/,https://i.imgur.com/y21wFd0.png
 WenzhouEconomicandEducation.cn,Wenzhou Economic and Education,,,,CN,,Wenzhou,c/CN,zho,science,FALSE,,,,,https://www.tvchinese.net/uploads/tv/wzjjkj.jpg
 YiwuBusinessChannel.cn,Yiwu Business Channel,,,,CN,,,c/CN,zho,business,FALSE,,,,,https://www.tvchinese.net/uploads/tv/yiwutv.jpg

tests/__data__/input/issues/channels_edit_approved.js

@@ -140,7 +140,7 @@ module.exports = [
     closed_at: null,
     author_association: 'CONTRIBUTOR',
     active_lock_reason: null,
-    body: '### Channel ID (required)\n\nBeijingSatelliteTV.cn\n\n### Channel Name\n\nbeIN Movies Turk\n\n### Alternative Names\n\nbeIN Movies Türk\n\n### Network\n\n_No response_\n\n### Owners\n\n_No response_\n\n### Country\n\nTR\n\n### Subdivision\n\n_No response_\n\n### City\n\n_No response_\n\n### Broadcast Area\n\nc/TR\n\n### Languages\n\ntur\n\n### Categories\n\nmovies\n\n### NSFW\n\nFALSE\n\n### Launched\n\n1979-05-16\n\n### Closed\n\n_No response_\n\n### Replaced By\n\n_No response_\n\n### Website\n\nhttp://www.digiturk.com.tr/\n\n### Logo\n\nhttps://i.imgur.com/nw8Sa2z.png\n\n### Notes\n\n_No response_',
+    body: '### Channel ID (required)\n\nBeijingSatelliteTV.cn\n\n### Channel Name\n\nbeIN Movies Turk\n\n### Alternative Names\n\nbeIN Movies Türk\n\n### Network\n\nBBC\n\n### Owners\n\nGazprom Media\n\n### Country\n\nTR\n\n### Subdivision\n\nUS-CA\n\n### City\n\nLondon\n\n### Broadcast Area\n\nc/TR\n\n### Languages\n\ntur\n\n### Categories\n\nmovies\n\n### NSFW\nTRUE\n\n### Launched\n\n1979-05-16\n\n### Closed\n\n1980-05-16\n\n### Replaced By\n\nM5.hu\n\n### Website\n\nhttp://www.digiturk.com.tr/\n\n### Logo\n\nhttps://i.imgur.com/nw8Sa2z.png\n\n### Notes\n\n_No response_',
     reactions: {
       url: 'https://api.github.com/repos/iptv-org/database/issues/5701/reactions',
       total_count: 0,

tests/__data__/input/validate/invalid_value/channels.csv

@@ -1,2 +1,2 @@
 id,name,alt_names,network,owners,country,subdivision,city,broadcast_area,languages,categories,is_nsfw,launched,closed,replaced_by,website,logo
-002RadioTV.do,002 Radio TV,,,,DO,,,c/DO,spa,,FALSE,,,,https://www.002radio.com/,https://i.imgur.com/7oNe8xj.png
+002RadioTV.do,002 Radio TV,,,,DO,,,c/DO,spa,,FALSE,,,,ttps://www.002radio.com/,https://i.imgur.com/7oNe8xj.png

tests/db/validate.test.ts

@@ -68,8 +68,13 @@ describe('db:validate', () => {
       process.exit(1)
     } catch (error) {
       expect((error as ExecError).status).toBe(1)
-      expect((error as ExecError).stdout).toContain('"aaa.us" is missing in the channels.csv')
-      expect((error as ExecError).stdout).toContain('1 error(s)')
+      expect((error as ExecError).stdout).toContain(
+        '2      "aaa.us" is missing in the channels.csv'
+      )
+      expect((error as ExecError).stdout).toContain(
+        '2      002RadioTV.do: "website" must be a valid uri with a scheme matching the http|https pattern'
+      )
+      expect((error as ExecError).stdout).toContain('2 error(s)')
     }
   })
 

yarn.lock

@@ -341,10 +341,10 @@
   resolved "https://registry.npmjs.org/@eslint/js/-/js-8.50.0.tgz"
   integrity sha512-NCC3zz2+nvYd+Ckfh87rA47zfu2QsQpvc6k1yzTk+b9KzRj0wkGa8LSoGOXN6Zv4lRf/EIoZ80biDh9HOI+RNQ==
 
-"@freearhey/core@^0.2.1":
-  version "0.2.1"
-  resolved "https://registry.npmjs.org/@freearhey/core/-/core-0.2.1.tgz"
-  integrity sha512-kEdIxZClykKhGpgyCSlkwuVuSCCAWr3J5YvOUMJQDPgVAYvT5VbD8MYKPm+OwNi9T4HFmF6qqY90qwKJPoOXCA==
+"@freearhey/core@^0.2.2":
+  version "0.2.2"
+  resolved "https://registry.npmjs.org/@freearhey/core/-/core-0.2.2.tgz"
+  integrity sha512-8eikSh+alXohY/0Qv8lls+6BG738vOqxGuLvMjwBasbwuz8S/Abr7fUAH65dGjPAsB3HMaDhjK5c6kcVz5GrcA==
   dependencies:
     "@types/fs-extra" "^11.0.2"
     "@types/lodash" "^4.14.198"