kevin/iptv-database
1
0
Fork 0
mirror of https://github.com/iptv-org/database.git synced 2024-11-22 02:44:10 -05:00
Code Issues Projects Releases Packages Wiki Activity

Update validate.js

Browse source
This commit is contained in:
Aleksandr Statciuk 2022-04-09 15:55:35 +03:00
parent b211bf904d
commit f92d80b236
1 changed files with 50 additions and 44 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

82
scripts/db/validate.js
View file

@ -7,11 +7,7 @@ const _ = require('lodash')
program.argument('[filepath]', 'Path to file to validate').parse(process.argv) program.argument('[filepath]', 'Path to file to validate').parse(process.argv)
async function main() { const allFiles = [
let globalErrors = []
const files = program.args.length
? program.args
: [
'data/blocklist.csv', 'data/blocklist.csv',
'data/categories.csv', 'data/categories.csv',
'data/channels.csv', 'data/channels.csv',
@ -20,7 +16,13 @@ async function main() {
'data/regions.csv', 'data/regions.csv',
'data/subdivisions.csv' 'data/subdivisions.csv'
] ]
for (const filepath of files) {
let db = {}
async function main() {
let globalErrors = []
for (let filepath of allFiles) {
if (!filepath.endsWith('.csv')) continue if (!filepath.endsWith('.csv')) continue
const eol = await file.eol(filepath) const eol = await file.eol(filepath)
@ -31,41 +33,45 @@ async function main() {
return handleError(`empty lines at the end of file not allowed (${filepath})`) return handleError(`empty lines at the end of file not allowed (${filepath})`)
const filename = file.getFilename(filepath) const filename = file.getFilename(filepath)
if (!schemes[filename]) return handleError(`"${filename}" scheme is missing`) let data = await csv
const rows = await csv
.fromString(csvString) .fromString(csvString)
.catch(err => handleError(`${err.message} (${filepath})`)) .catch(err => handleError(`${err.message} (${filepath})`))
switch (filename) {
case 'blocklist':
data = _.keyBy(data, 'channel')
break
case 'categories':
case 'channels':
data = _.keyBy(data, 'id')
break
default:
data = _.keyBy(data, 'code')
break
}
db[filename] = data
}
const toCheck = program.args.length ? program.args : allFiles
for (const filepath of toCheck) {
const filename = file.getFilename(filepath)
if (!schemes[filename]) return handleError(`"${filename}" scheme is missing`)
const rows = Object.values(db[filename])
let fileErrors = [] let fileErrors = []
if (filename === 'channels') { if (filename === 'channels') {
if (/\"/.test(csvString)) return handleError(`\" character is not allowed (${filepath})`)
fileErrors = fileErrors.concat(findDuplicatesById(rows)) fileErrors = fileErrors.concat(findDuplicatesById(rows))
let categories = await csv
.fromFile('data/categories.csv')
.catch(err => handleError(err.message))
categories = _.keyBy(categories, 'id')
let languages = await csv
.fromFile('data/languages.csv')
.catch(err => handleError(err.message))
languages = _.keyBy(languages, 'code')
let countries = await csv
.fromFile('data/countries.csv')
.catch(err => handleError(err.message))
countries = _.keyBy(countries, 'code')
for (const [i, row] of rows.entries()) { for (const [i, row] of rows.entries()) {
fileErrors = fileErrors.concat(await validateChannelCategories(row, i, categories)) fileErrors = fileErrors.concat(await validateChannelCategories(row, i))
fileErrors = fileErrors.concat(await validateChannelLanguages(row, i, languages)) fileErrors = fileErrors.concat(await validateChannelLanguages(row, i))
fileErrors = fileErrors.concat(await validateChannelCountry(row, i, countries)) fileErrors = fileErrors.concat(await validateChannelCountry(row, i))
} }
} else if (filename === 'blocklist') { } else if (filename === 'blocklist') {
let channels = await csv.fromFile('data/channels.csv').catch(err => handleError(err.message))
channels = _.keyBy(channels, 'id')
for (const [i, row] of rows.entries()) { for (const [i, row] of rows.entries()) {
fileErrors = fileErrors.concat(await validateChannelId(row, i, channels)) fileErrors = fileErrors.concat(await validateChannelId(row, i))
} }
} }
@ -115,10 +121,10 @@ function findDuplicatesById(data) {
return errors return errors
} }
async function validateChannelCategories(row, i, categories) { async function validateChannelCategories(row, i) {
const errors = [] const errors = []
row.categories.forEach(category => { row.categories.forEach(category => {
if (!categories[category]) { if (!db.categories[category]) {
errors.push({ errors.push({
line: i + 2, line: i + 2,
message: `"${row.id}" has the wrong category "${category}"` message: `"${row.id}" has the wrong category "${category}"`
@ -129,9 +135,9 @@ async function validateChannelCategories(row, i, categories) {
return errors return errors
} }
async function validateChannelCountry(row, i, countries) { async function validateChannelCountry(row, i) {
const errors = [] const errors = []
if (!countries[row.country]) { if (!db.countries[row.country]) {
errors.push({ errors.push({
line: i + 2, line: i + 2,
message: `"${row.id}" has the wrong country "${row.country}"` message: `"${row.id}" has the wrong country "${row.country}"`
@ -141,10 +147,10 @@ async function validateChannelCountry(row, i, countries) {
return errors return errors
} }
async function validateChannelLanguages(row, i, languages) { async function validateChannelLanguages(row, i) {
const errors = [] const errors = []
row.languages.forEach(language => { row.languages.forEach(language => {
if (!languages[language]) { if (!db.languages[language]) {
errors.push({ errors.push({
line: i + 2, line: i + 2,
message: `"${row.id}" has the wrong language "${language}"` message: `"${row.id}" has the wrong language "${language}"`
@ -155,9 +161,9 @@ async function validateChannelLanguages(row, i, languages) {
return errors return errors
} }
async function validateChannelId(row, i, channels) { async function validateChannelId(row, i) {
const errors = [] const errors = []
if (!channels[row.channel]) { if (!db.channels[row.channel]) {
errors.push({ errors.push({
line: i + 2, line: i + 2,
message: `"${row.channel}" is missing in the channels.csv` message: `"${row.channel}" is missing in the channels.csv`