diff --git a/.husky/pre-commit b/.husky/pre-commit new file mode 100644 index 00000000..fb967d4e --- /dev/null +++ b/.husky/pre-commit @@ -0,0 +1 @@ +npm run db:validate diff --git a/data/blocklist.csv b/data/blocklist.csv index 1d48cb8e..23789e73 100644 --- a/data/blocklist.csv +++ b/data/blocklist.csv @@ -441,7 +441,6 @@ Eurosport1.de,https://github.com/iptv-org/iptv/issues/1831 Eurosport1.dk,https://github.com/iptv-org/iptv/issues/1831 Eurosport1.es,https://github.com/iptv-org/iptv/issues/1831 Eurosport1.fr,https://github.com/iptv-org/iptv/issues/1831 -Eurosport1.fr,https://github.com/iptv-org/iptv/issues/1831 Eurosport1.gr,https://github.com/iptv-org/iptv/issues/1831 Eurosport1.hu,https://github.com/iptv-org/iptv/issues/1831 Eurosport1.it,https://github.com/iptv-org/iptv/issues/1831 @@ -457,7 +456,6 @@ Eurosport2.de,https://github.com/iptv-org/iptv/issues/1831 Eurosport2.dk,https://github.com/iptv-org/iptv/issues/1831 Eurosport2.es,https://github.com/iptv-org/iptv/issues/1831 Eurosport2.fr,https://github.com/iptv-org/iptv/issues/1831 -Eurosport2.fr,https://github.com/iptv-org/iptv/issues/1831 Eurosport2.gr,https://github.com/iptv-org/iptv/issues/1831 Eurosport2.hu,https://github.com/iptv-org/iptv/issues/1831 Eurosport2.it,https://github.com/iptv-org/iptv/issues/1831 @@ -517,7 +515,6 @@ FAPTVTeens.ru,https://github.com/iptv-org/iptv/issues/15723 FAPTVTrans.ru,https://github.com/iptv-org/iptv/issues/15723 FashionTVMidnightSecrets.fr,https://github.com/iptv-org/iptv/issues/15723 FastTV.us,https://github.com/iptv-org/iptv/issues/1831 -FastTV.us,https://github.com/iptv-org/iptv/issues/1831 Fatafeat.ae,https://github.com/iptv-org/iptv/issues/1831 FEM.no,https://github.com/iptv-org/iptv/issues/1831 Flamingo.jp,https://github.com/iptv-org/iptv/issues/15723 @@ -825,7 +822,6 @@ SKYHIGHSEXVR.ru,https://github.com/iptv-org/iptv/issues/15723 SkyNews.uk,https://github.com/iptv-org/iptv/issues/7314 SkyNewsHD.uk,https://github.com/iptv-org/iptv/issues/7314 SkySport.it,https://github.com/github/dmca/blob/master/2020/09/2020-09-16-dfl.md -SkySport.it,https://github.com/github/dmca/blob/master/2020/09/2020-09-16-dfl.md SkySport1.de,https://github.com/github/dmca/blob/master/2020/09/2020-09-16-dfl.md SkySport10.de,https://github.com/github/dmca/blob/master/2020/09/2020-09-16-dfl.md SkySport10.it,https://github.com/github/dmca/blob/master/2020/09/2020-09-16-dfl.md @@ -968,7 +964,6 @@ TAPSports.ph,https://github.com/github/dmca/blob/master/2020/09/2020-09-16-dfl.m TBSEast.us,https://github.com/iptv-org/iptv/issues/16839 TBSWest.us,https://github.com/iptv-org/iptv/issues/16839 TCMEast.us,https://github.com/iptv-org/iptv/issues/16839 -TCMEast.us,https://github.com/iptv-org/iptv/issues/16839 TCMWest.us,https://github.com/iptv-org/iptv/issues/16839 TelevisionX.uk,https://github.com/iptv-org/iptv/issues/15723 TheGrioTV.us,https://github.com/iptv-org/iptv/issues/9729 diff --git a/package-lock.json b/package-lock.json index 8b45da90..3587a667 100644 --- a/package-lock.json +++ b/package-lock.json @@ -6,6 +6,8 @@ "": { "name": "@iptv-org/database", "dependencies": { + "@eslint/eslintrc": "^3.2.0", + "@eslint/js": "^9.16.0", "@freearhey/core": "^0.2.2", "@joi/date": "^2.1.0", "@json2csv/formatters": "^7.0.3", @@ -15,6 +17,7 @@ "@octokit/plugin-paginate-rest": "^6.0.0", "@octokit/plugin-rest-endpoint-methods": "^7.1.3", "@types/jest": "^29.5.5", + "@types/joi": "^17.2.3", "@typescript-eslint/eslint-plugin": "^8.17.0", "chalk": "^4.1.2", "commander": "^9.0.0", @@ -22,18 +25,13 @@ "eslint": "^9.16.0", "eslint-config-prettier": "^9.0.0", "fs-extra": "^11.2.0", + "globals": "^15.13.0", + "husky": "^9.1.7", "jest": "^29.7.0", "joi": "^17.13.3", "ts-jest": "^29.1.1", "tsx": "^4.10.5" }, - "devDependencies": { - "@eslint/eslintrc": "^3.2.0", - "@eslint/js": "^9.16.0", - "@types/joi": "^17.2.3", - "globals": "^15.13.0", - "pre-commit": "^1.0.10" - }, "engines": { "node": ">=18.0.0 <=22.12.0" } @@ -2184,7 +2182,6 @@ "resolved": "https://registry.npmjs.org/@types/joi/-/joi-17.2.3.tgz", "integrity": "sha512-dGjs/lhrWOa+eO0HwgxCSnDm5eMGCsXuvLglMghJq32F6q5LyyNuXb41DHzrg501CKNOSSAHmfB7FDGeUnDmzw==", "deprecated": "This is a stub types definition. joi provides its own type definitions, so you do not need this installed.", - "dev": true, "dependencies": { "joi": "*" } @@ -3790,7 +3787,6 @@ "version": "15.13.0", "resolved": "https://registry.npmjs.org/globals/-/globals-15.13.0.tgz", "integrity": "sha512-49TewVEz0UxZjr1WYYsWpPrhyC/B/pA8Bq0fUmet2n+eR7yn0IvNzNaoBwnK6mdkzcN+se7Ez9zUgULTz2QH4g==", - "dev": true, "engines": { "node": ">=18" }, @@ -3840,6 +3836,20 @@ "node": ">=10.17.0" } }, + "node_modules/husky": { + "version": "9.1.7", + "resolved": "https://registry.npmjs.org/husky/-/husky-9.1.7.tgz", + "integrity": "sha512-5gs5ytaNjBrh5Ow3zrvdUUY+0VxIuWVL4i9irt6friV+BqdCfmV11CQTWMiBYWHbXhco+J1kHfTOUkePhCDvMA==", + "bin": { + "husky": "bin.js" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/typicode" + } + }, "node_modules/ignore": { "version": "5.3.2", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", @@ -5329,16 +5339,6 @@ "node": ">=8" } }, - "node_modules/pre-commit": { - "version": "1.0.10", - "resolved": "https://registry.npmjs.org/pre-commit/-/pre-commit-1.0.10.tgz", - "integrity": "sha512-lpS8vLmOuAHSH8v+9GLSyYMplk9oZYLr3caRbsx9BVJzXb+/bI1l45JXAa0x/io732ZwgW2nhV8EeKjqVm7y4Q==", - "dev": true, - "hasInstallScript": true, - "dependencies": { - "shelljs": "0.5.x" - } - }, "node_modules/prelude-ls": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", @@ -5540,18 +5540,6 @@ "node": ">=10" } }, - "node_modules/shelljs": { - "version": "0.5.3", - "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.5.3.tgz", - "integrity": "sha512-C2FisSSW8S6TIYHHiMHN0NqzdjWfTekdMpA2FJTbRWnQMLO1RRIXEB9eVZYOlofYmjZA7fY3ChoFu09MeI3wlQ==", - "dev": true, - "bin": { - "shjs": "bin/shjs" - }, - "engines": { - "node": ">=0.8.0" - } - }, "node_modules/signal-exit": { "version": "3.0.7", "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", @@ -7667,7 +7655,6 @@ "version": "17.2.3", "resolved": "https://registry.npmjs.org/@types/joi/-/joi-17.2.3.tgz", "integrity": "sha512-dGjs/lhrWOa+eO0HwgxCSnDm5eMGCsXuvLglMghJq32F6q5LyyNuXb41DHzrg501CKNOSSAHmfB7FDGeUnDmzw==", - "dev": true, "requires": { "joi": "*" } @@ -8756,8 +8743,7 @@ "globals": { "version": "15.13.0", "resolved": "https://registry.npmjs.org/globals/-/globals-15.13.0.tgz", - "integrity": "sha512-49TewVEz0UxZjr1WYYsWpPrhyC/B/pA8Bq0fUmet2n+eR7yn0IvNzNaoBwnK6mdkzcN+se7Ez9zUgULTz2QH4g==", - "dev": true + "integrity": "sha512-49TewVEz0UxZjr1WYYsWpPrhyC/B/pA8Bq0fUmet2n+eR7yn0IvNzNaoBwnK6mdkzcN+se7Ez9zUgULTz2QH4g==" }, "graceful-fs": { "version": "4.2.9", @@ -8792,6 +8778,11 @@ "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-2.1.0.tgz", "integrity": "sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==" }, + "husky": { + "version": "9.1.7", + "resolved": "https://registry.npmjs.org/husky/-/husky-9.1.7.tgz", + "integrity": "sha512-5gs5ytaNjBrh5Ow3zrvdUUY+0VxIuWVL4i9irt6friV+BqdCfmV11CQTWMiBYWHbXhco+J1kHfTOUkePhCDvMA==" + }, "ignore": { "version": "5.3.2", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", @@ -9891,15 +9882,6 @@ } } }, - "pre-commit": { - "version": "1.0.10", - "resolved": "https://registry.npmjs.org/pre-commit/-/pre-commit-1.0.10.tgz", - "integrity": "sha512-lpS8vLmOuAHSH8v+9GLSyYMplk9oZYLr3caRbsx9BVJzXb+/bI1l45JXAa0x/io732ZwgW2nhV8EeKjqVm7y4Q==", - "dev": true, - "requires": { - "shelljs": "0.5.x" - } - }, "prelude-ls": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", @@ -10014,12 +9996,6 @@ "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz", "integrity": "sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==" }, - "shelljs": { - "version": "0.5.3", - "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.5.3.tgz", - "integrity": "sha512-C2FisSSW8S6TIYHHiMHN0NqzdjWfTekdMpA2FJTbRWnQMLO1RRIXEB9eVZYOlofYmjZA7fY3ChoFu09MeI3wlQ==", - "dev": true - }, "signal-exit": { "version": "3.0.7", "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", diff --git a/package.json b/package.json index 4de3cfb7..2fa1f69f 100644 --- a/package.json +++ b/package.json @@ -8,14 +8,12 @@ "db:export": "tsx scripts/db/export.ts", "db:update": "tsx scripts/db/update.ts", "lint": "npx eslint \"{scripts,tests}/**/*.{ts,js}\"", - "test": "jest --runInBand" + "test": "jest --runInBand", + "prepare": "husky" }, "engines": { "node": ">=18.0.0 <=22.12.0" }, - "pre-commit": [ - "db:validate" - ], "private": true, "author": "Arhey", "jest": { @@ -25,6 +23,8 @@ "testRegex": "tests/(.*?/)?.*test.(js|ts)$" }, "dependencies": { + "@eslint/eslintrc": "^3.2.0", + "@eslint/js": "^9.16.0", "@freearhey/core": "^0.2.2", "@joi/date": "^2.1.0", "@json2csv/formatters": "^7.0.3", @@ -34,6 +34,7 @@ "@octokit/plugin-paginate-rest": "^6.0.0", "@octokit/plugin-rest-endpoint-methods": "^7.1.3", "@types/jest": "^29.5.5", + "@types/joi": "^17.2.3", "@typescript-eslint/eslint-plugin": "^8.17.0", "chalk": "^4.1.2", "commander": "^9.0.0", @@ -41,16 +42,11 @@ "eslint": "^9.16.0", "eslint-config-prettier": "^9.0.0", "fs-extra": "^11.2.0", + "globals": "^15.13.0", + "husky": "^9.1.7", "jest": "^29.7.0", "joi": "^17.13.3", "ts-jest": "^29.1.1", "tsx": "^4.10.5" - }, - "devDependencies": { - "@eslint/eslintrc": "^3.2.0", - "@eslint/js": "^9.16.0", - "@types/joi": "^17.2.3", - "globals": "^15.13.0", - "pre-commit": "^1.0.10" } } diff --git a/scripts/db/validate.ts b/scripts/db/validate.ts index 05013b15..afb61ab8 100644 --- a/scripts/db/validate.ts +++ b/scripts/db/validate.ts @@ -43,7 +43,7 @@ async function main() { let grouped switch (filename) { case 'blocklist': - grouped = data.keyBy(item => item.channel) + grouped = data.keyBy(item => item.channel + item.ref) break case 'categories': case 'channels': @@ -70,7 +70,7 @@ async function main() { let fileErrors = new Collection() switch (filename) { case 'channels': - fileErrors = fileErrors.concat(findDuplicatesBy(rowsCopy, 'id')) + fileErrors = fileErrors.concat(findDuplicatesBy(rowsCopy, ['id'])) for (const [i, row] of rowsCopy.entries()) { fileErrors = fileErrors.concat(validateChannelId(row, i)) fileErrors = fileErrors.concat(validateChannelBroadcastArea(row, i)) @@ -92,12 +92,13 @@ async function main() { } break case 'blocklist': + fileErrors = fileErrors.concat(findDuplicatesBy(rowsCopy, ['channel', 'ref'])) for (const [i, row] of rowsCopy.entries()) { fileErrors = fileErrors.concat(validateChannel(row.channel, i)) } break case 'countries': - fileErrors = fileErrors.concat(findDuplicatesBy(rowsCopy, 'code')) + fileErrors = fileErrors.concat(findDuplicatesBy(rowsCopy, ['code'])) for (const [i, row] of rowsCopy.entries()) { fileErrors = fileErrors.concat( checkValue(i, row, 'code', 'languages', buffer.get('languages')) @@ -105,7 +106,7 @@ async function main() { } break case 'subdivisions': - fileErrors = fileErrors.concat(findDuplicatesBy(rowsCopy, 'code')) + fileErrors = fileErrors.concat(findDuplicatesBy(rowsCopy, ['code'])) for (const [i, row] of rowsCopy.entries()) { fileErrors = fileErrors.concat( checkValue(i, row, 'code', 'country', buffer.get('countries')) @@ -113,7 +114,7 @@ async function main() { } break case 'regions': - fileErrors = fileErrors.concat(findDuplicatesBy(rowsCopy, 'code')) + fileErrors = fileErrors.concat(findDuplicatesBy(rowsCopy, ['code'])) for (const [i, row] of rowsCopy.entries()) { fileErrors = fileErrors.concat( checkValue(i, row, 'code', 'countries', buffer.get('countries')) @@ -121,10 +122,10 @@ async function main() { } break case 'categories': - fileErrors = fileErrors.concat(findDuplicatesBy(rowsCopy, 'id')) + fileErrors = fileErrors.concat(findDuplicatesBy(rowsCopy, ['id'])) break case 'languages': - fileErrors = fileErrors.concat(findDuplicatesBy(rowsCopy, 'code')) + fileErrors = fileErrors.concat(findDuplicatesBy(rowsCopy, ['code'])) break } @@ -195,16 +196,17 @@ function validateChannel(channelId: string, i: number) { return errors } -function findDuplicatesBy(rows: { [key: string]: string }[], key: string) { +function findDuplicatesBy(rows: { [key: string]: string }[], keys: string[]) { const errors = new Collection() const buffer = new Dictionary() rows.forEach((row, i) => { - const normId = row[key].toLowerCase() + const normId = keys.map(key => row[key].toLowerCase()).join() if (buffer.has(normId)) { + const fieldsList = keys.map(key => `${key} "${row[key]}"`).join(' and ') errors.push({ line: i + 2, - message: `entry with the ${key} "${row[key]}" already exists` + message: `entry with the ${fieldsList} already exists` }) } diff --git a/tests/__data__/input/validate/duplicate/blocklist.csv b/tests/__data__/input/validate/duplicate/blocklist.csv new file mode 100644 index 00000000..dcfba40d --- /dev/null +++ b/tests/__data__/input/validate/duplicate/blocklist.csv @@ -0,0 +1,3 @@ +channel,ref +002RadioTV.do,eee +002RadioTV.do,eee \ No newline at end of file diff --git a/tests/__data__/input/validate/duplicate/channels.csv b/tests/__data__/input/validate/duplicate/channels.csv new file mode 100644 index 00000000..22a83c3a --- /dev/null +++ b/tests/__data__/input/validate/duplicate/channels.csv @@ -0,0 +1,2 @@ +id,name,alt_names,network,owners,country,subdivision,city,broadcast_area,languages,categories,is_nsfw,launched,closed,replaced_by,website,logo +002RadioTV.do,002 Radio TV,,,,DO,,,c/DO,,,FALSE,,,,,https://i.imgur.com/7oNe8xj.png \ No newline at end of file diff --git a/tests/__data__/input/validate/duplicate/countries.csv b/tests/__data__/input/validate/duplicate/countries.csv new file mode 100644 index 00000000..8a1c05e5 --- /dev/null +++ b/tests/__data__/input/validate/duplicate/countries.csv @@ -0,0 +1,3 @@ +name,code,languages,flag +Andorra,AD,cat,🇦🇩 +Dominican Republic,DO,spa,🇩🇴 \ No newline at end of file diff --git a/tests/__data__/input/validate/duplicate/languages.csv b/tests/__data__/input/validate/duplicate/languages.csv new file mode 100644 index 00000000..7e3eb4b4 --- /dev/null +++ b/tests/__data__/input/validate/duplicate/languages.csv @@ -0,0 +1,3 @@ +code,name +cat,Catalan +spa,Spanish \ No newline at end of file diff --git a/tests/__data__/input/validate/duplicate/subdivisions.csv b/tests/__data__/input/validate/duplicate/subdivisions.csv new file mode 100644 index 00000000..9498b609 --- /dev/null +++ b/tests/__data__/input/validate/duplicate/subdivisions.csv @@ -0,0 +1,2 @@ +country,name,code +AD,Andorra la Vella,AD-07 \ No newline at end of file diff --git a/tests/db/validate.test.ts b/tests/db/validate.test.ts index 495a9b70..f60d3eda 100644 --- a/tests/db/validate.test.ts +++ b/tests/db/validate.test.ts @@ -43,6 +43,9 @@ describe('db:validate', () => { } catch (error) { expect((error as ExecError).status).toBe(1) expect((error as ExecError).stdout).toContain('entry with the id "aaa" already exists') + expect((error as ExecError).stdout).toContain( + 'entry with the channel "002RadioTV.do" and ref "eee" already exists' + ) } }) diff --git a/yarn.lock b/yarn.lock index 27f14153..91a49e56 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2010,6 +2010,11 @@ human-signals@^2.1.0: resolved "https://registry.npmjs.org/human-signals/-/human-signals-2.1.0.tgz" integrity sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw== +husky@^9.1.7: + version "9.1.7" + resolved "https://registry.npmjs.org/husky/-/husky-9.1.7.tgz" + integrity sha512-5gs5ytaNjBrh5Ow3zrvdUUY+0VxIuWVL4i9irt6friV+BqdCfmV11CQTWMiBYWHbXhco+J1kHfTOUkePhCDvMA== + ignore@^5.2.0, ignore@^5.3.1: version "5.3.2" resolved "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz" @@ -3012,13 +3017,6 @@ pkg-dir@^4.2.0: dependencies: find-up "^4.0.0" -pre-commit@^1.0.10: - version "1.0.10" - resolved "https://registry.npmjs.org/pre-commit/-/pre-commit-1.0.10.tgz" - integrity sha512-lpS8vLmOuAHSH8v+9GLSyYMplk9oZYLr3caRbsx9BVJzXb+/bI1l45JXAa0x/io732ZwgW2nhV8EeKjqVm7y4Q== - dependencies: - shelljs "0.5.x" - prelude-ls@^1.2.1: version "1.2.1" resolved "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz" @@ -3141,11 +3139,6 @@ shebang-regex@^3.0.0: resolved "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz" integrity sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A== -shelljs@0.5.x: - version "0.5.3" - resolved "https://registry.npmjs.org/shelljs/-/shelljs-0.5.3.tgz" - integrity sha512-C2FisSSW8S6TIYHHiMHN0NqzdjWfTekdMpA2FJTbRWnQMLO1RRIXEB9eVZYOlofYmjZA7fY3ChoFu09MeI3wlQ== - signal-exit@^3.0.3, signal-exit@^3.0.7: version "3.0.7" resolved "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz"