Merge pull request #15105 from iptv-org/freearhey-patch-4

Patch 2024.12.2

.husky/pre-commit

@@ -0,0 +1 @@
+npm run db:validate

data/blocklist.csv

@@ -441,7 +441,6 @@ Eurosport1.de,https://github.com/iptv-org/iptv/issues/1831
 Eurosport1.dk,https://github.com/iptv-org/iptv/issues/1831
 Eurosport1.es,https://github.com/iptv-org/iptv/issues/1831
 Eurosport1.fr,https://github.com/iptv-org/iptv/issues/1831
-Eurosport1.fr,https://github.com/iptv-org/iptv/issues/1831
 Eurosport1.gr,https://github.com/iptv-org/iptv/issues/1831
 Eurosport1.hu,https://github.com/iptv-org/iptv/issues/1831
 Eurosport1.it,https://github.com/iptv-org/iptv/issues/1831
@@ -457,7 +456,6 @@ Eurosport2.de,https://github.com/iptv-org/iptv/issues/1831
 Eurosport2.dk,https://github.com/iptv-org/iptv/issues/1831
 Eurosport2.es,https://github.com/iptv-org/iptv/issues/1831
 Eurosport2.fr,https://github.com/iptv-org/iptv/issues/1831
-Eurosport2.fr,https://github.com/iptv-org/iptv/issues/1831
 Eurosport2.gr,https://github.com/iptv-org/iptv/issues/1831
 Eurosport2.hu,https://github.com/iptv-org/iptv/issues/1831
 Eurosport2.it,https://github.com/iptv-org/iptv/issues/1831
@@ -517,7 +515,6 @@ FAPTVTeens.ru,https://github.com/iptv-org/iptv/issues/15723
 FAPTVTrans.ru,https://github.com/iptv-org/iptv/issues/15723
 FashionTVMidnightSecrets.fr,https://github.com/iptv-org/iptv/issues/15723
 FastTV.us,https://github.com/iptv-org/iptv/issues/1831
-FastTV.us,https://github.com/iptv-org/iptv/issues/1831
 Fatafeat.ae,https://github.com/iptv-org/iptv/issues/1831
 FEM.no,https://github.com/iptv-org/iptv/issues/1831
 Flamingo.jp,https://github.com/iptv-org/iptv/issues/15723
@@ -825,7 +822,6 @@ SKYHIGHSEXVR.ru,https://github.com/iptv-org/iptv/issues/15723
 SkyNews.uk,https://github.com/iptv-org/iptv/issues/7314
 SkyNewsHD.uk,https://github.com/iptv-org/iptv/issues/7314
 SkySport.it,https://github.com/github/dmca/blob/master/2020/09/2020-09-16-dfl.md
-SkySport.it,https://github.com/github/dmca/blob/master/2020/09/2020-09-16-dfl.md
 SkySport1.de,https://github.com/github/dmca/blob/master/2020/09/2020-09-16-dfl.md
 SkySport10.de,https://github.com/github/dmca/blob/master/2020/09/2020-09-16-dfl.md
 SkySport10.it,https://github.com/github/dmca/blob/master/2020/09/2020-09-16-dfl.md
@@ -968,7 +964,6 @@ TAPSports.ph,https://github.com/github/dmca/blob/master/2020/09/2020-09-16-dfl.m
 TBSEast.us,https://github.com/iptv-org/iptv/issues/16839
 TBSWest.us,https://github.com/iptv-org/iptv/issues/16839
 TCMEast.us,https://github.com/iptv-org/iptv/issues/16839
-TCMEast.us,https://github.com/iptv-org/iptv/issues/16839
 TCMWest.us,https://github.com/iptv-org/iptv/issues/16839
 TelevisionX.uk,https://github.com/iptv-org/iptv/issues/15723
 TheGrioTV.us,https://github.com/iptv-org/iptv/issues/9729

package-lock.json

@@ -6,6 +6,8 @@
     "": {
       "name": "@iptv-org/database",
       "dependencies": {
+        "@eslint/eslintrc": "^3.2.0",
+        "@eslint/js": "^9.16.0",
         "@freearhey/core": "^0.2.2",
         "@joi/date": "^2.1.0",
         "@json2csv/formatters": "^7.0.3",
@@ -15,6 +17,7 @@
         "@octokit/plugin-paginate-rest": "^6.0.0",
         "@octokit/plugin-rest-endpoint-methods": "^7.1.3",
         "@types/jest": "^29.5.5",
+        "@types/joi": "^17.2.3",
         "@typescript-eslint/eslint-plugin": "^8.17.0",
         "chalk": "^4.1.2",
         "commander": "^9.0.0",
@@ -22,18 +25,13 @@
         "eslint": "^9.16.0",
         "eslint-config-prettier": "^9.0.0",
         "fs-extra": "^11.2.0",
+        "globals": "^15.13.0",
+        "husky": "^9.1.7",
         "jest": "^29.7.0",
         "joi": "^17.13.3",
         "ts-jest": "^29.1.1",
         "tsx": "^4.10.5"
       },
-      "devDependencies": {
-        "@eslint/eslintrc": "^3.2.0",
-        "@eslint/js": "^9.16.0",
-        "@types/joi": "^17.2.3",
-        "globals": "^15.13.0",
-        "pre-commit": "^1.0.10"
-      },
       "engines": {
         "node": ">=18.0.0 <=22.12.0"
       }
@@ -2184,7 +2182,6 @@
       "resolved": "https://registry.npmjs.org/@types/joi/-/joi-17.2.3.tgz",
       "integrity": "sha512-dGjs/lhrWOa+eO0HwgxCSnDm5eMGCsXuvLglMghJq32F6q5LyyNuXb41DHzrg501CKNOSSAHmfB7FDGeUnDmzw==",
       "deprecated": "This is a stub types definition. joi provides its own type definitions, so you do not need this installed.",
-      "dev": true,
       "dependencies": {
         "joi": "*"
       }
@@ -3790,7 +3787,6 @@
       "version": "15.13.0",
       "resolved": "https://registry.npmjs.org/globals/-/globals-15.13.0.tgz",
       "integrity": "sha512-49TewVEz0UxZjr1WYYsWpPrhyC/B/pA8Bq0fUmet2n+eR7yn0IvNzNaoBwnK6mdkzcN+se7Ez9zUgULTz2QH4g==",
-      "dev": true,
       "engines": {
         "node": ">=18"
       },
@@ -3840,6 +3836,20 @@
         "node": ">=10.17.0"
       }
     },
+    "node_modules/husky": {
+      "version": "9.1.7",
+      "resolved": "https://registry.npmjs.org/husky/-/husky-9.1.7.tgz",
+      "integrity": "sha512-5gs5ytaNjBrh5Ow3zrvdUUY+0VxIuWVL4i9irt6friV+BqdCfmV11CQTWMiBYWHbXhco+J1kHfTOUkePhCDvMA==",
+      "bin": {
+        "husky": "bin.js"
+      },
+      "engines": {
+        "node": ">=18"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/typicode"
+      }
+    },
     "node_modules/ignore": {
       "version": "5.3.2",
       "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz",
@@ -5329,16 +5339,6 @@
         "node": ">=8"
       }
     },
-    "node_modules/pre-commit": {
-      "version": "1.0.10",
-      "resolved": "https://registry.npmjs.org/pre-commit/-/pre-commit-1.0.10.tgz",
-      "integrity": "sha512-lpS8vLmOuAHSH8v+9GLSyYMplk9oZYLr3caRbsx9BVJzXb+/bI1l45JXAa0x/io732ZwgW2nhV8EeKjqVm7y4Q==",
-      "dev": true,
-      "hasInstallScript": true,
-      "dependencies": {
-        "shelljs": "0.5.x"
-      }
-    },
     "node_modules/prelude-ls": {
       "version": "1.2.1",
       "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz",
@@ -5540,18 +5540,6 @@
         "node": ">=10"
       }
     },
-    "node_modules/shelljs": {
-      "version": "0.5.3",
-      "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.5.3.tgz",
-      "integrity": "sha512-C2FisSSW8S6TIYHHiMHN0NqzdjWfTekdMpA2FJTbRWnQMLO1RRIXEB9eVZYOlofYmjZA7fY3ChoFu09MeI3wlQ==",
-      "dev": true,
-      "bin": {
-        "shjs": "bin/shjs"
-      },
-      "engines": {
-        "node": ">=0.8.0"
-      }
-    },
     "node_modules/signal-exit": {
       "version": "3.0.7",
       "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz",
@@ -7667,7 +7655,6 @@
       "version": "17.2.3",
       "resolved": "https://registry.npmjs.org/@types/joi/-/joi-17.2.3.tgz",
       "integrity": "sha512-dGjs/lhrWOa+eO0HwgxCSnDm5eMGCsXuvLglMghJq32F6q5LyyNuXb41DHzrg501CKNOSSAHmfB7FDGeUnDmzw==",
-      "dev": true,
       "requires": {
         "joi": "*"
       }
@@ -8756,8 +8743,7 @@
     "globals": {
       "version": "15.13.0",
       "resolved": "https://registry.npmjs.org/globals/-/globals-15.13.0.tgz",
-      "integrity": "sha512-49TewVEz0UxZjr1WYYsWpPrhyC/B/pA8Bq0fUmet2n+eR7yn0IvNzNaoBwnK6mdkzcN+se7Ez9zUgULTz2QH4g==",
+      "integrity": "sha512-49TewVEz0UxZjr1WYYsWpPrhyC/B/pA8Bq0fUmet2n+eR7yn0IvNzNaoBwnK6mdkzcN+se7Ez9zUgULTz2QH4g=="
-      "dev": true
     },
     "graceful-fs": {
       "version": "4.2.9",
@@ -8792,6 +8778,11 @@
       "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-2.1.0.tgz",
       "integrity": "sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw=="
     },
+    "husky": {
+      "version": "9.1.7",
+      "resolved": "https://registry.npmjs.org/husky/-/husky-9.1.7.tgz",
+      "integrity": "sha512-5gs5ytaNjBrh5Ow3zrvdUUY+0VxIuWVL4i9irt6friV+BqdCfmV11CQTWMiBYWHbXhco+J1kHfTOUkePhCDvMA=="
+    },
     "ignore": {
       "version": "5.3.2",
       "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz",
@@ -9891,15 +9882,6 @@
         }
       }
     },
-    "pre-commit": {
-      "version": "1.0.10",
-      "resolved": "https://registry.npmjs.org/pre-commit/-/pre-commit-1.0.10.tgz",
-      "integrity": "sha512-lpS8vLmOuAHSH8v+9GLSyYMplk9oZYLr3caRbsx9BVJzXb+/bI1l45JXAa0x/io732ZwgW2nhV8EeKjqVm7y4Q==",
-      "dev": true,
-      "requires": {
-        "shelljs": "0.5.x"
-      }
-    },
     "prelude-ls": {
       "version": "1.2.1",
       "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz",
@@ -10014,12 +9996,6 @@
       "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz",
       "integrity": "sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A=="
     },
-    "shelljs": {
-      "version": "0.5.3",
-      "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.5.3.tgz",
-      "integrity": "sha512-C2FisSSW8S6TIYHHiMHN0NqzdjWfTekdMpA2FJTbRWnQMLO1RRIXEB9eVZYOlofYmjZA7fY3ChoFu09MeI3wlQ==",
-      "dev": true
-    },
     "signal-exit": {
       "version": "3.0.7",
       "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz",

package.json

@@ -8,14 +8,12 @@
     "db:export": "tsx scripts/db/export.ts",
     "db:update": "tsx scripts/db/update.ts",
     "lint": "npx eslint \"{scripts,tests}/**/*.{ts,js}\"",
-    "test": "jest --runInBand"
+    "test": "jest --runInBand",
+    "prepare": "husky"
   },
   "engines": {
     "node": ">=18.0.0 <=22.12.0"
   },
-  "pre-commit": [
-    "db:validate"
-  ],
   "private": true,
   "author": "Arhey",
   "jest": {
@@ -25,6 +23,8 @@
     "testRegex": "tests/(.*?/)?.*test.(js|ts)$"
   },
   "dependencies": {
+    "@eslint/eslintrc": "^3.2.0",
+    "@eslint/js": "^9.16.0",
     "@freearhey/core": "^0.2.2",
     "@joi/date": "^2.1.0",
     "@json2csv/formatters": "^7.0.3",
@@ -34,6 +34,7 @@
     "@octokit/plugin-paginate-rest": "^6.0.0",
     "@octokit/plugin-rest-endpoint-methods": "^7.1.3",
     "@types/jest": "^29.5.5",
+    "@types/joi": "^17.2.3",
     "@typescript-eslint/eslint-plugin": "^8.17.0",
     "chalk": "^4.1.2",
     "commander": "^9.0.0",
@@ -41,16 +42,11 @@
     "eslint": "^9.16.0",
     "eslint-config-prettier": "^9.0.0",
     "fs-extra": "^11.2.0",
+    "globals": "^15.13.0",
+    "husky": "^9.1.7",
     "jest": "^29.7.0",
     "joi": "^17.13.3",
     "ts-jest": "^29.1.1",
     "tsx": "^4.10.5"
-  },
-  "devDependencies": {
-    "@eslint/eslintrc": "^3.2.0",
-    "@eslint/js": "^9.16.0",
-    "@types/joi": "^17.2.3",
-    "globals": "^15.13.0",
-    "pre-commit": "^1.0.10"
   }
 }

scripts/db/validate.ts

@@ -43,7 +43,7 @@ async function main() {
     let grouped
     switch (filename) {
       case 'blocklist':
-        grouped = data.keyBy(item => item.channel)
+        grouped = data.keyBy(item => item.channel + item.ref)
         break
       case 'categories':
       case 'channels':
@@ -70,7 +70,7 @@ async function main() {
     let fileErrors = new Collection()
     switch (filename) {
       case 'channels':
-        fileErrors = fileErrors.concat(findDuplicatesBy(rowsCopy, 'id'))
+        fileErrors = fileErrors.concat(findDuplicatesBy(rowsCopy, ['id']))
         for (const [i, row] of rowsCopy.entries()) {
           fileErrors = fileErrors.concat(validateChannelId(row, i))
           fileErrors = fileErrors.concat(validateChannelBroadcastArea(row, i))
@@ -92,12 +92,13 @@ async function main() {
         }
         break
       case 'blocklist':
+        fileErrors = fileErrors.concat(findDuplicatesBy(rowsCopy, ['channel', 'ref']))
         for (const [i, row] of rowsCopy.entries()) {
           fileErrors = fileErrors.concat(validateChannel(row.channel, i))
         }
         break
       case 'countries':
-        fileErrors = fileErrors.concat(findDuplicatesBy(rowsCopy, 'code'))
+        fileErrors = fileErrors.concat(findDuplicatesBy(rowsCopy, ['code']))
         for (const [i, row] of rowsCopy.entries()) {
           fileErrors = fileErrors.concat(
             checkValue(i, row, 'code', 'languages', buffer.get('languages'))
@@ -105,7 +106,7 @@ async function main() {
         }
         break
       case 'subdivisions':
-        fileErrors = fileErrors.concat(findDuplicatesBy(rowsCopy, 'code'))
+        fileErrors = fileErrors.concat(findDuplicatesBy(rowsCopy, ['code']))
         for (const [i, row] of rowsCopy.entries()) {
           fileErrors = fileErrors.concat(
             checkValue(i, row, 'code', 'country', buffer.get('countries'))
@@ -113,7 +114,7 @@ async function main() {
         }
         break
       case 'regions':
-        fileErrors = fileErrors.concat(findDuplicatesBy(rowsCopy, 'code'))
+        fileErrors = fileErrors.concat(findDuplicatesBy(rowsCopy, ['code']))
         for (const [i, row] of rowsCopy.entries()) {
           fileErrors = fileErrors.concat(
             checkValue(i, row, 'code', 'countries', buffer.get('countries'))
@@ -121,10 +122,10 @@ async function main() {
         }
         break
       case 'categories':
-        fileErrors = fileErrors.concat(findDuplicatesBy(rowsCopy, 'id'))
+        fileErrors = fileErrors.concat(findDuplicatesBy(rowsCopy, ['id']))
         break
       case 'languages':
-        fileErrors = fileErrors.concat(findDuplicatesBy(rowsCopy, 'code'))
+        fileErrors = fileErrors.concat(findDuplicatesBy(rowsCopy, ['code']))
         break
     }
 
@@ -195,16 +196,17 @@ function validateChannel(channelId: string, i: number) {
   return errors
 }
 
-function findDuplicatesBy(rows: { [key: string]: string }[], key: string) {
+function findDuplicatesBy(rows: { [key: string]: string }[], keys: string[]) {
   const errors = new Collection()
   const buffer = new Dictionary()
 
   rows.forEach((row, i) => {
-    const normId = row[key].toLowerCase()
+    const normId = keys.map(key => row[key].toLowerCase()).join()
     if (buffer.has(normId)) {
+      const fieldsList = keys.map(key => `${key} "${row[key]}"`).join(' and ')
       errors.push({
         line: i + 2,
-        message: `entry with the ${key} "${row[key]}" already exists`
+        message: `entry with the ${fieldsList} already exists`
       })
     }
 

tests/__data__/input/validate/duplicate/blocklist.csv

@@ -0,0 +1,3 @@
+channel,ref
+002RadioTV.do,eee
+002RadioTV.do,eee

tests/__data__/input/validate/duplicate/channels.csv

@@ -0,0 +1,2 @@
+id,name,alt_names,network,owners,country,subdivision,city,broadcast_area,languages,categories,is_nsfw,launched,closed,replaced_by,website,logo
+002RadioTV.do,002 Radio TV,,,,DO,,,c/DO,,,FALSE,,,,,https://i.imgur.com/7oNe8xj.png

tests/__data__/input/validate/duplicate/countries.csv

@@ -0,0 +1,3 @@
+name,code,languages,flag
+Andorra,AD,cat,🇦🇩
+Dominican Republic,DO,spa,🇩🇴

tests/__data__/input/validate/duplicate/languages.csv

@@ -0,0 +1,3 @@
+code,name
+cat,Catalan
+spa,Spanish

tests/__data__/input/validate/duplicate/subdivisions.csv

@@ -0,0 +1,2 @@
+country,name,code
+AD,Andorra la Vella,AD-07

tests/db/validate.test.ts

@@ -43,6 +43,9 @@ describe('db:validate', () => {
     } catch (error) {
       expect((error as ExecError).status).toBe(1)
       expect((error as ExecError).stdout).toContain('entry with the id "aaa" already exists')
+      expect((error as ExecError).stdout).toContain(
+        'entry with the channel "002RadioTV.do" and ref "eee" already exists'
+      )
     }
   })
 

yarn.lock

@@ -2010,6 +2010,11 @@ human-signals@^2.1.0:
   resolved "https://registry.npmjs.org/human-signals/-/human-signals-2.1.0.tgz"
   integrity sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==
 
+husky@^9.1.7:
+  version "9.1.7"
+  resolved "https://registry.npmjs.org/husky/-/husky-9.1.7.tgz"
+  integrity sha512-5gs5ytaNjBrh5Ow3zrvdUUY+0VxIuWVL4i9irt6friV+BqdCfmV11CQTWMiBYWHbXhco+J1kHfTOUkePhCDvMA==
+
 ignore@^5.2.0, ignore@^5.3.1:
   version "5.3.2"
   resolved "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz"
@@ -3012,13 +3017,6 @@ pkg-dir@^4.2.0:
   dependencies:
     find-up "^4.0.0"
 
-pre-commit@^1.0.10:
-  version "1.0.10"
-  resolved "https://registry.npmjs.org/pre-commit/-/pre-commit-1.0.10.tgz"
-  integrity sha512-lpS8vLmOuAHSH8v+9GLSyYMplk9oZYLr3caRbsx9BVJzXb+/bI1l45JXAa0x/io732ZwgW2nhV8EeKjqVm7y4Q==
-  dependencies:
-    shelljs "0.5.x"
-
 prelude-ls@^1.2.1:
   version "1.2.1"
   resolved "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz"
@@ -3141,11 +3139,6 @@ shebang-regex@^3.0.0:
   resolved "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz"
   integrity sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==
 
-shelljs@0.5.x:
-  version "0.5.3"
-  resolved "https://registry.npmjs.org/shelljs/-/shelljs-0.5.3.tgz"
-  integrity sha512-C2FisSSW8S6TIYHHiMHN0NqzdjWfTekdMpA2FJTbRWnQMLO1RRIXEB9eVZYOlofYmjZA7fY3ChoFu09MeI3wlQ==
-
 signal-exit@^3.0.3, signal-exit@^3.0.7:
   version "3.0.7"
   resolved "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz"