const { logger, file, csv } = require('../core') const { program } = require('commander') const schemes = require('./schemes') const chalk = require('chalk') const Joi = require('joi') const _ = require('lodash') program.argument('[filepath]', 'Path to file to validate').parse(process.argv) async function main() { let globalErrors = [] const files = program.args.length ? program.args : [ 'data/blocklist.csv', 'data/categories.csv', 'data/channels.csv', 'data/countries.csv', 'data/languages.csv', 'data/regions.csv', 'data/subdivisions.csv' ] for (const filepath of files) { if (!filepath.endsWith('.csv')) continue const eol = await file.eol(filepath) if (eol !== 'CRLF') return handleError(`file must have line endings with CRLF (${filepath})`) const csvString = await file.read(filepath) if (/\s+$/.test(csvString)) return handleError(`empty lines at the end of file not allowed (${filepath})`) const filename = file.getFilename(filepath) if (!schemes[filename]) return handleError(`"${filename}" scheme is missing`) const rows = await csv .fromString(csvString) .catch(err => handleError(`${err.message} (${filepath})`)) let fileErrors = [] if (filename === 'channels') { if (/\"/.test(csvString)) return handleError(`\" character is not allowed (${filepath})`) fileErrors = fileErrors.concat(findDuplicatesById(rows)) fileErrors = fileErrors.concat(await validateChannelCategories(rows)) fileErrors = fileErrors.concat(await validateChannelLanguages(rows)) } else if (filename === 'blocklist') { fileErrors = fileErrors.concat(await validateChannelId(rows)) } const schema = Joi.object(schemes[filename]) rows.forEach((row, i) => { const { error } = schema.validate(row, { abortEarly: false }) if (error) { error.details.forEach(detail => { fileErrors.push({ line: i + 2, message: detail.message }) }) } }) if (fileErrors.length) { logger.info(`\n${chalk.underline(filepath)}`) fileErrors.forEach(err => { const position = err.line.toString().padEnd(6, ' ') logger.info(` ${chalk.gray(position)} ${err.message}`) }) globalErrors = globalErrors.concat(fileErrors) } } if (globalErrors.length) return handleError(`${globalErrors.length} error(s)`) } main() function findDuplicatesById(data) { data = data.map(i => { i.id = i.id.toLowerCase() return i }) const errors = [] const schema = Joi.array().unique((a, b) => a.id === b.id) const { error } = schema.validate(data, { abortEarly: false }) if (error) { error.details.forEach(detail => { errors.push({ line: detail.context.pos + 2, message: `Entry with the id "${detail.context.value.id}" already exists` }) }) } return errors } async function validateChannelCategories(rows) { let categories = await csv.fromFile('data/categories.csv').catch(err => handleError(err.message)) const errors = [] if (categories.length) { categories = _.keyBy(categories, 'id') rows.forEach((row, i) => { row.categories.forEach(category => { if (!categories[category]) { errors.push({ line: i + 2, message: `"${row.id}" has the wrong category "${category}"` }) } }) }) } return errors } async function validateChannelLanguages(rows) { let languages = await csv.fromFile('data/languages.csv').catch(err => handleError(err.message)) const errors = [] if (languages.length) { languages = _.keyBy(languages, 'code') rows.forEach((row, i) => { row.languages.forEach(language => { if (!languages[language]) { errors.push({ line: i + 2, message: `"${row.id}" has the wrong language "${language}"` }) } }) }) } return errors } async function validateChannelId(rows) { let channels = await csv.fromFile('data/channels.csv').catch(err => handleError(err.message)) const errors = [] if (channels.length) { channels = _.keyBy(channels, 'id') rows.forEach((row, i) => { if (!channels[row.channel]) { errors.push({ line: i + 2, message: `"${row.channel}" is missing in the channels.csv` }) } }) } return errors } function handleError(message) { logger.error(chalk.red(`\n${message}`)) process.exit(1) }