Merge branch 'replace-eol-char'

data/languages.csv

@@ -7891,4 +7891,4 @@ zyj,Youjiang Zhuang
 zyn,Yongnan Zhuang
 zyp,Zyphe Chin
 zza,Dimili
-zzj,Zuojiang Zhuang
+zzj,Zuojiang Zhuang

scripts/core/csv.js

@@ -1,4 +1,6 @@
 const csv2json = require('csvtojson')
+const chalk = require('chalk')
+const logger = require('./logger')
 const fs = require('mz/fs')
 const {
 	Parser,
@@ -10,6 +12,7 @@ const csv2jsonOptions = {
 	checkColumn: true,
 	trim: true,
 	delimiter: ',',
+	eol: '\r\n',
 	colParser: {
 		countries: listParser,
 		languages: listParser,
@@ -32,10 +35,14 @@ const json2csv = new Parser({
 
 const csv = {}
 
-csv.load = async function (filepath) {
+csv.fromFile = async function (filepath) {
 	return csv2json(csv2jsonOptions).fromFile(filepath)
 }
 
+csv.fromString = async function (filepath) {
+	return csv2json(csv2jsonOptions).fromString(filepath)
+}
+
 csv.save = async function (filepath, data) {
 	const string = json2csv.parse(data)
 

scripts/core/file.js

@@ -1,6 +1,7 @@
 const path = require('path')
 const glob = require('glob')
 const fs = require('mz/fs')
+const crlf = require('crlf')
 
 const file = {}
 
@@ -65,4 +66,13 @@ file.basename = function (filepath) {
   return path.basename(filepath)
 }
 
+file.eol = function (filepath) {
+  return new Promise((resolve, reject) => {
+    crlf.get(filepath, null, function (err, endingType) {
+      if (err) reject(err)
+      resolve(endingType)
+    })
+  })
+}
+
 module.exports = file

scripts/db/export.js

@@ -8,7 +8,7 @@ async function main() {
 	const files = await file.list(`${DATA_DIR}/*.csv`)
 	for (const filepath of files) {
 		const filename = file.getFilename(filepath)
-		const json = await csv.load(filepath).catch(err => {
+		const json = await csv.fromFile(filepath).catch(err => {
 			logger.error(chalk.red(`\n${err.message} (${filepath})`))
 			process.exit(1)
 		})

scripts/db/validate.js

@@ -21,7 +21,20 @@ async function main() {
 		  ]
 	for (const filepath of files) {
 		if (!filepath.endsWith('.csv')) continue
-		const data = await csv.load(filepath).catch(err => {
+
+		const eol = await file.eol(filepath)
+		if (eol !== 'CRLF') {
+			logger.error(chalk.red(`\nError: file must have line endings with CRLF (${filepath})`))
+			process.exit(1)
+		}
+
+		const csvString = await file.read(filepath)
+		if (/\s+$/.test(csvString)) {
+			logger.error(chalk.red(`\nError: empty lines at the end of file not allowed (${filepath})`))
+			process.exit(1)
+		}
+
+		const data = await csv.fromString(csvString).catch(err => {
 			logger.error(chalk.red(`\n${err.message} (${filepath})`))
 			process.exit(1)
 		})