kevin/iptv-database
1
0
Fork 0
mirror of https://github.com/iptv-org/database.git synced 2024-12-22 16:00:24 -05:00
Code Issues Projects Releases Packages Wiki Activity

Update validate.js

Browse source
This commit is contained in:
Aleksandr Statciuk 2022-04-09 15:11:13 +03:00
parent 6fa99c38a7
commit 8495d3c847
1 changed files with 81 additions and 85 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

166
scripts/db/validate.js
View file

@ -8,7 +8,7 @@ const _ = require('lodash')
program.argument('[filepath]', 'Path to file to validate').parse(process.argv)
async function main() {
let errors = []
let globalErrors = []
const files = program.args.length
? program.args
: [
@ -24,93 +24,32 @@ async function main() {
if (!filepath.endsWith('.csv')) continue
const eol = await file.eol(filepath)
if (eol !== 'CRLF') {
logger.error(chalk.red(`\nError: file must have line endings with CRLF (${filepath})`))
process.exit(1)
}
if (eol !== 'CRLF') return handleError(`file must have line endings with CRLF (${filepath})`)
const csvString = await file.read(filepath)
if (/\s+$/.test(csvString)) {
logger.error(chalk.red(`\nError: empty lines at the end of file not allowed (${filepath})`))
process.exit(1)
}
if (/\s+$/.test(csvString))
return handleError(`empty lines at the end of file not allowed (${filepath})`)
const filename = file.getFilename(filepath)
if (!schemes[filename]) {
logger.error(chalk.red(`\nError: "${filename}" scheme is missing`))
process.exit(1)
}
if (!schemes[filename]) return handleError(`"${filename}" scheme is missing`)
const data = await csv.fromString(csvString).catch(err => {
logger.error(chalk.red(`\n${err.message} (${filepath})`))
process.exit(1)
})
const rows = await csv
.fromString(csvString)
.catch(err => handleError(`${err.message} (${filepath})`))
let fileErrors = []
if (filename === 'channels') {
if (/\"/.test(csvString)) {
logger.error(chalk.red(`\nError: \" character is not allowed (${filepath})`))
process.exit(1)
}
if (/\"/.test(csvString)) return handleError(`\" character is not allowed (${filepath})`)
fileErrors = fileErrors.concat(findDuplicatesById(data))
let categories = await csv.fromFile('data/categories.csv').catch(err => {
logger.error(chalk.red(`\nError: ${err.message}`))
process.exit(1)
})
if (categories.length) {
categories = _.keyBy(categories, 'id')
data.forEach((row, i) => {
row.categories.forEach(category => {
if (!categories[category]) {
fileErrors.push({
line: i + 2,
message: `"${row.id}" has the wrong category "${category}"`
})
}
})
})
}
let languages = await csv.fromFile('data/languages.csv').catch(err => {
logger.error(chalk.red(`\nError: ${err.message}`))
process.exit(1)
})
if (languages.length) {
languages = _.keyBy(languages, 'code')
data.forEach((row, i) => {
row.languages.forEach(language => {
if (!languages[language]) {
fileErrors.push({
line: i + 2,
message: `"${row.id}" has the wrong language "${language}"`
})
}
})
})
}
fileErrors = fileErrors.concat(findDuplicatesById(rows))
fileErrors = fileErrors.concat(await validateChannelCategories(rows))
fileErrors = fileErrors.concat(await validateChannelLanguages(rows))
} else if (filename === 'blocklist') {
let channels = await csv.fromFile('data/channels.csv').catch(err => {
logger.error(chalk.red(`\nError: ${err.message}`))
process.exit(1)
})
channels = channels.map(c => c.id)
data.forEach((row, i) => {
if (channels.length && !channels.includes(row.channel)) {
fileErrors.push({
line: i + 2,
message: `"${row.channel}" is missing in the channels.csv`
})
}
})
fileErrors = fileErrors.concat(await validateChannelId(rows))
}
const schema = Joi.object(schemes[filename])
data.forEach((row, i) => {
rows.forEach((row, i) => {
const { error } = schema.validate(row, { abortEarly: false })
if (error) {
error.details.forEach(detail => {
@ -123,16 +62,13 @@ async function main() {
logger.info(`\n${chalk.underline(filepath)}`)
fileErrors.forEach(err => {
const position = err.line.toString().padEnd(6, ' ')
logger.error(` ${chalk.gray(position)} ${err.message}`)
logger.info(` ${chalk.gray(position)} ${err.message}`)
})
errors = errors.concat(fileErrors)
globalErrors = globalErrors.concat(fileErrors)
}
}
if (errors.length) {
logger.error(chalk.red(`\n${errors.length} error(s)`))
process.exit(1)
}
if (globalErrors.length) return handleError(`${globalErrors.length} error(s)`)
}
main()
@ -158,8 +94,68 @@ function findDuplicatesById(data) {
return errors
}
function intersection(array1, array2) {
return array1.filter(function (n) {
return array2.indexOf(n) !== -1
})
async function validateChannelCategories(rows) {
let categories = await csv.fromFile('data/categories.csv').catch(err => handleError(err.message))
const errors = []
if (categories.length) {
categories = _.keyBy(categories, 'id')
rows.forEach((row, i) => {
row.categories.forEach(category => {
if (!categories[category]) {
errors.push({
line: i + 2,
message: `"${row.id}" has the wrong category "${category}"`
})
}
})
})
}
return errors
}
async function validateChannelLanguages(rows) {
let languages = await csv.fromFile('data/languages.csv').catch(err => handleError(err.message))
const errors = []
if (languages.length) {
languages = _.keyBy(languages, 'code')
rows.forEach((row, i) => {
row.languages.forEach(language => {
if (!languages[language]) {
errors.push({
line: i + 2,
message: `"${row.id}" has the wrong language "${language}"`
})
}
})
})
}
return errors
}
async function validateChannelId(rows) {
let channels = await csv.fromFile('data/channels.csv').catch(err => handleError(err.message))
const errors = []
if (channels.length) {
channels = _.keyBy(channels, 'id')
rows.forEach((row, i) => {
if (!channels[row.channel]) {
errors.push({
line: i + 2,
message: `"${row.channel}" is missing in the channels.csv`
})
}
})
}
return errors
}
function handleError(message) {
logger.error(chalk.red(`\n${message}`))
process.exit(1)
}