iptv-database/scripts/db/validate.ts

258 lines
7.7 KiB
TypeScript
Raw Normal View History

2023-10-06 22:14:21 -04:00
import { Collection, Storage, File, Dictionary, Logger } from '@freearhey/core'
import { DATA_DIR } from '../constants'
import { program } from 'commander'
import Joi from 'joi'
2023-10-07 19:13:58 -04:00
import { CSVParser, IDCreator } from '../core'
2023-10-06 22:14:21 -04:00
import chalk from 'chalk'
program.argument('[filepath]', 'Path to file to validate').parse(process.argv)
const logger = new Logger()
const buffer = new Dictionary()
const files = new Dictionary()
const schemes: { [key: string]: object } = require('../schemes')
async function main() {
const dataStorage = new Storage(DATA_DIR)
const _files = await dataStorage.list('*.csv')
let globalErrors = new Collection()
const parser = new CSVParser()
for (const filepath of _files) {
const file = new File(filepath)
if (file.extension() !== 'csv') continue
const csv = await dataStorage.load(file.basename())
const rows = csv.split(/\r\n/)
const headers = rows[0].split(',')
for (const [i, line] of rows.entries()) {
2023-10-23 15:24:23 -04:00
if (!line.trim()) continue
2023-10-06 22:14:21 -04:00
if (line.indexOf('\n') > -1)
return handleError(
`Error: row ${i + 1} has the wrong line ending character, should be CRLF (${filepath})`
)
if (line.split(/,(?=(?:(?:[^"]*"){2})*[^"]*$)/).length !== headers.length)
return handleError(`Error: row ${i + 1} has the wrong number of columns (${filepath})`)
}
const data = await parser.parse(csv)
const filename = file.name()
let grouped
switch (filename) {
case 'blocklist':
grouped = data.keyBy(item => item.channel)
break
case 'categories':
case 'channels':
grouped = data.keyBy(item => item.id)
break
default:
grouped = data.keyBy(item => item.code)
break
}
buffer.set(filename, grouped)
files.set(filename, data)
}
const filesToCheck = program.args.length ? program.args : _files
for (const filepath of filesToCheck) {
const file = new File(filepath)
const filename = file.name()
if (!schemes[filename]) return handleError(`Error: "${filename}" scheme is missing`)
const rows: Collection = files.get(filename)
const rowsCopy = JSON.parse(JSON.stringify(rows.all()))
let fileErrors = new Collection()
switch (filename) {
case 'channels':
fileErrors = fileErrors.concat(findDuplicatesBy(rowsCopy, 'id'))
for (const [i, row] of rowsCopy.entries()) {
fileErrors = fileErrors.concat(validateChannelId(row, i))
fileErrors = fileErrors.concat(validateChannelBroadcastArea(row, i))
fileErrors = fileErrors.concat(
checkValue(i, row, 'id', 'subdivision', buffer.get('subdivisions'))
)
fileErrors = fileErrors.concat(
checkValue(i, row, 'id', 'categories', buffer.get('categories'))
)
fileErrors = fileErrors.concat(
checkValue(i, row, 'id', 'replaced_by', buffer.get('channels'))
)
fileErrors = fileErrors.concat(
checkValue(i, row, 'id', 'languages', buffer.get('languages'))
)
fileErrors = fileErrors.concat(
checkValue(i, row, 'id', 'country', buffer.get('countries'))
)
}
break
case 'blocklist':
for (const [i, row] of rowsCopy.entries()) {
fileErrors = fileErrors.concat(validateChannel(row.channel, i))
}
break
case 'countries':
fileErrors = fileErrors.concat(findDuplicatesBy(rowsCopy, 'code'))
for (const [i, row] of rowsCopy.entries()) {
fileErrors = fileErrors.concat(
checkValue(i, row, 'code', 'languages', buffer.get('languages'))
)
}
break
case 'subdivisions':
fileErrors = fileErrors.concat(findDuplicatesBy(rowsCopy, 'code'))
for (const [i, row] of rowsCopy.entries()) {
fileErrors = fileErrors.concat(
checkValue(i, row, 'code', 'country', buffer.get('countries'))
)
}
break
case 'regions':
fileErrors = fileErrors.concat(findDuplicatesBy(rowsCopy, 'code'))
for (const [i, row] of rowsCopy.entries()) {
fileErrors = fileErrors.concat(
checkValue(i, row, 'code', 'countries', buffer.get('countries'))
)
}
break
case 'categories':
fileErrors = fileErrors.concat(findDuplicatesBy(rowsCopy, 'id'))
break
case 'languages':
fileErrors = fileErrors.concat(findDuplicatesBy(rowsCopy, 'code'))
break
}
const schema = Joi.object(schemes[filename])
2023-10-11 11:16:24 -04:00
rows.all().forEach((row: { [key: string]: string }, i: number) => {
2023-10-06 22:14:21 -04:00
const { error } = schema.validate(row, { abortEarly: false })
if (error) {
error.details.forEach(detail => {
2023-10-11 11:16:24 -04:00
fileErrors.push({ line: i + 2, row, message: detail.message })
2023-10-06 22:14:21 -04:00
})
}
})
if (fileErrors.count()) {
logger.info(`\n${chalk.underline(filepath)}`)
fileErrors.forEach(err => {
const position = err.line.toString().padEnd(6, ' ')
2023-10-11 11:16:24 -04:00
const id = err.row && err.row.id ? ` ${err.row.id}:` : ''
logger.info(` ${chalk.gray(position)}${id} ${err.message}`)
2023-10-06 22:14:21 -04:00
})
globalErrors = globalErrors.concat(fileErrors)
}
}
if (globalErrors.count()) return handleError(`${globalErrors.count()} error(s)`)
}
main()
function checkValue(
i: number,
row: { [key: string]: string[] | string | boolean },
key: string,
field: string,
collection: Collection
) {
const errors = new Collection()
let values: string[] = []
if (Array.isArray(row[field])) {
values = row[field] as string[]
} else if (typeof row[field] === 'string') {
values = new Array(row[field]) as string[]
}
values.forEach((value: string) => {
if (collection.missing(value)) {
errors.push({
line: i + 2,
message: `"${row[key]}" has an invalid ${field} "${value}"`
})
}
})
return errors
}
function validateChannel(channelId: string, i: number) {
const errors = new Collection()
const channels = buffer.get('channels')
if (channels.missing(channelId)) {
errors.push({
line: i + 2,
message: `"${channelId}" is missing in the channels.csv`
})
}
return errors
}
function findDuplicatesBy(rows: { [key: string]: string }[], key: string) {
const errors = new Collection()
const buffer = new Dictionary()
rows.forEach((row, i) => {
const normId = row[key].toLowerCase()
if (buffer.has(normId)) {
errors.push({
line: i + 2,
message: `entry with the ${key} "${row[key]}" already exists`
})
}
buffer.set(normId, true)
})
return errors
}
function validateChannelId(row: { [key: string]: string }, i: number) {
const errors = new Collection()
2023-10-07 19:13:58 -04:00
const expectedId = new IDCreator().create(row.name, row.country)
2023-10-06 22:14:21 -04:00
2023-10-07 19:13:58 -04:00
if (expectedId !== row.id) {
2023-10-06 22:14:21 -04:00
errors.push({
line: i + 2,
message: `"${row.id}" must be derived from the channel name "${row.name}" and the country code "${row.country}"`
})
}
return errors
}
function validateChannelBroadcastArea(row: { [key: string]: string[] }, i: number) {
const errors = new Collection()
const regions = buffer.get('regions')
const countries = buffer.get('countries')
const subdivisions = buffer.get('subdivisions')
row.broadcast_area.forEach((areaCode: string) => {
const [type, code] = areaCode.split('/')
if (
(type === 'r' && regions.missing(code)) ||
(type === 'c' && countries.missing(code)) ||
(type === 's' && subdivisions.missing(code))
) {
errors.push({
line: i + 2,
message: `"${row.id}" has the wrong broadcast_area "${areaCode}"`
})
}
})
return errors
}
function handleError(message: string) {
logger.error(chalk.red(message))
process.exit(1)
}