mirror of
https://github.com/iptv-org/database.git
synced 2024-11-26 12:31:27 -05:00
Merge branch 'master' of https://github.com/AntiPontifex/database
This commit is contained in:
AntiPontifex
commit
16817e09c9
70 changed files with 12913 additions and 5599 deletions
39
.eslintrc.json
Normal file
39
.eslintrc.json
Normal file
|
|
@ -0,0 +1,39 @@
|
|||
{
|
||||
"env": {
|
||||
"node": true,
|
||||
"es2021": true,
|
||||
"jest": true
|
||||
},
|
||||
"extends": [
|
||||
"eslint:recommended",
|
||||
"plugin:@typescript-eslint/recommended",
|
||||
"prettier"
|
||||
],
|
||||
"parser": "@typescript-eslint/parser",
|
||||
"parserOptions": {
|
||||
"ecmaVersion": "latest",
|
||||
"sourceType": "module"
|
||||
},
|
||||
"plugins": [
|
||||
"@typescript-eslint"
|
||||
],
|
||||
"rules": {
|
||||
"@typescript-eslint/no-var-requires": "off",
|
||||
"no-case-declarations": "off",
|
||||
"linebreak-style": [
|
||||
"error",
|
||||
"unix"
|
||||
],
|
||||
"quotes": [
|
||||
"error",
|
||||
"single",
|
||||
{
|
||||
"avoidEscape": true
|
||||
}
|
||||
],
|
||||
"semi": [
|
||||
"error",
|
||||
"never"
|
||||
]
|
||||
}
|
||||
}
|
||||
13
.github/CODE_OF_CONDUCT.md
vendored
Normal file
13
.github/CODE_OF_CONDUCT.md
vendored
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
# Contributor Code of Conduct
|
||||
|
||||
As contributors and maintainers of this project, we pledge to respect all people who contribute through reporting issues, posting feature requests, updating documentation, submitting pull requests or patches, and other activities.
|
||||
|
||||
We are committed to making participation in this project a harassment-free experience for everyone, regardless of level of experience, gender, gender identity and expression, sexual orientation, disability, personal appearance, body size, race, ethnicity, age, or religion.
|
||||
|
||||
Examples of unacceptable behavior by participants include the use of sexual language or imagery, derogatory comments or personal attacks, trolling, public or private harassment, insults, or other unprofessional conduct.
|
||||
|
||||
Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct. Project maintainers who do not follow the Code of Conduct may be removed from the project team.
|
||||
|
||||
Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by opening an issue or contacting one or more of the project maintainers.
|
||||
|
||||
This Code of Conduct is adapted from the [Contributor Covenant](http:contributor-covenant.org), version 1.0.0, available at https://www.contributor-covenant.org/version/1/0/0/code-of-conduct.html
|
||||
21
.github/workflows/check.yml
vendored
21
.github/workflows/check.yml
vendored
|
|
@ -7,15 +7,20 @@ jobs:
|
|||
check:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/checkout@v3
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- id: files
|
||||
uses: tj-actions/changed-files@v12.2
|
||||
fetch-depth: 2
|
||||
- uses: actions/setup-node@v3
|
||||
if: ${{ !env.ACT }}
|
||||
with:
|
||||
files: 'data'
|
||||
node-version: 18
|
||||
cache: 'npm'
|
||||
- uses: tj-actions/changed-files@v35
|
||||
id: files
|
||||
with:
|
||||
files: data/*.csv
|
||||
- name: install dependencies
|
||||
run: npm install
|
||||
- name: validate
|
||||
if: steps.files.outputs.any_changed == 'true'
|
||||
run: |
|
||||
npm install
|
||||
npm run db:validate -- ${{ steps.files.outputs.all_changed_files }}
|
||||
run: npm run db:validate -- ${{ steps.files.outputs.all_changed_files }}
|
||||
|
|
|
|||
15
.github/workflows/deploy.yml
vendored
15
.github/workflows/deploy.yml
vendored
|
|
@ -8,10 +8,17 @@ jobs:
|
|||
update:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- run: npm install
|
||||
- run: npm run db:export
|
||||
- uses: tibdex/github-app-token@v1
|
||||
- uses: actions/checkout@v3
|
||||
- uses: actions/setup-node@v3
|
||||
if: ${{ !env.ACT }}
|
||||
with:
|
||||
node-version: 18
|
||||
cache: 'npm'
|
||||
- name: install dependencies
|
||||
run: npm install
|
||||
- name: export data to ./api
|
||||
run: npm run db:export
|
||||
- uses: tibdex/github-app-token@v1.8.2
|
||||
if: ${{ !env.ACT }}
|
||||
id: create-app-token
|
||||
with:
|
||||
|
|
|
|||
35
.github/workflows/update.yml
vendored
35
.github/workflows/update.yml
vendored
|
|
@ -7,25 +7,40 @@ jobs:
|
|||
main:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: tibdex/github-app-token@v1
|
||||
- uses: actions/checkout@v3
|
||||
- uses: tibdex/github-app-token@v1.8.2
|
||||
if: ${{ !env.ACT }}
|
||||
id: create-app-token
|
||||
with:
|
||||
app_id: ${{ secrets.APP_ID }}
|
||||
private_key: ${{ secrets.APP_PRIVATE_KEY }}
|
||||
- uses: actions/checkout@v3
|
||||
if: ${{ !env.ACT }}
|
||||
with:
|
||||
token: ${{ steps.create-app-token.outputs.token }}
|
||||
- run: npm install
|
||||
- run: npm run db:update --silent >> $GITHUB_OUTPUT
|
||||
id: issue-process
|
||||
- name: Validate Changes
|
||||
- uses: actions/setup-node@v3
|
||||
if: ${{ !env.ACT }}
|
||||
with:
|
||||
node-version: 18
|
||||
cache: 'npm'
|
||||
- name: install dependencies
|
||||
run: npm install
|
||||
- name: update data
|
||||
id: db-update
|
||||
run: npm run db:update --silent >> $GITHUB_OUTPUT
|
||||
- name: validate changes
|
||||
run: npm run db:validate
|
||||
- name: Commit Changes
|
||||
if: steps.issue-process.outputs.OUTPUT != 0
|
||||
- name: setup git
|
||||
run: |
|
||||
git config user.name "iptv-bot[bot]"
|
||||
git config user.email "84861620+iptv-bot[bot]@users.noreply.github.com"
|
||||
git add data/channels.csv
|
||||
git commit -m "[Bot] Update channels.csv" -m "Committed by [iptv-bot](https://github.com/apps/iptv-bot) via [update](https://github.com/iptv-org/database/actions/runs/${{ github.run_id }}) workflow." -m "${{ steps.issue-process.outputs.OUTPUT }}" --no-verify
|
||||
- run: git status
|
||||
- name: commit changes
|
||||
if: steps.db-update.outputs.OUTPUT != 0
|
||||
run: |
|
||||
git add data/*.csv
|
||||
git status
|
||||
git push
|
||||
git commit -m "[Bot] Update data" -m "Committed by [iptv-bot](https://github.com/apps/iptv-bot) via [update](https://github.com/iptv-org/database/actions/runs/${{ github.run_id }}) workflow." -m "${{ steps.db-update.outputs.OUTPUT }}" --no-verify
|
||||
- name: push all changes to the repository
|
||||
if: ${{ !env.ACT && github.ref == 'refs/heads/master' }}
|
||||
run: git push
|
||||
|
|
|
|||
10
.prettierrc.js
Normal file
10
.prettierrc.js
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
module.exports = {
|
||||
tabWidth: 2,
|
||||
useTabs: false,
|
||||
endOfLine: 'lf',
|
||||
semi: false,
|
||||
singleQuote: true,
|
||||
printWidth: 100,
|
||||
trailingComma: 'none',
|
||||
arrowParens: 'avoid'
|
||||
}
|
||||
|
|
@ -1,41 +1,26 @@
|
|||
# Contributing Guide
|
||||
|
||||
### How to add a channel to the database or edit its description?
|
||||
|
||||
1. Download the repository to your computer. The easiest way to do this is via [GitHub Desktop](https://desktop.github.com/).
|
||||
2. Open [data/channels.csv](data/channels.csv) file in one of the spreadsheet editors (such as [Google Sheets](https://www.google.com/sheets/about/), [LibreOffice](https://www.libreoffice.org/discover/libreoffice/), ...).
|
||||
3. Make the necessary changes and save the file.
|
||||
4. Make a [pull request](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/about-pull-requests) with all changes. This can also be done via [GitHub Desktop](https://desktop.github.com/).
|
||||
|
||||
**IMPORTANT:** Since different programs process CSV files differently before publishing an edited file, please make sure that:
|
||||
|
||||
- no extra columns (commas) were added to the file
|
||||
- only [CRLF](https://developer.mozilla.org/en-US/docs/Glossary/CRLF) is used to indicate the end of a line
|
||||
- no empty lines at the end of the file
|
||||
- [Data Scheme](#data-scheme)
|
||||
- [Channel Logo Guidelines](#channel-logo-guidelines)
|
||||
- [Project Structure](#project-structure)
|
||||
- [Scripts](#scripts)
|
||||
- [Workflows](#workflows)
|
||||
|
||||
## Data Scheme
|
||||
|
||||
- [channels](#channels)
|
||||
- [categories](#categories)
|
||||
- [countries](#countries)
|
||||
- [languages](#languages)
|
||||
- [regions](#regions)
|
||||
- [subdivisions](#subdivisions)
|
||||
- [blocklist](#blocklist)
|
||||
|
||||
### channels
|
||||
|
||||
| Field | Description | Required | Example |
|
||||
| -------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | ------------------------------ |
|
||||
| id | Unique channel ID derived from the `name` and `country` separated by dot. May only contain Latin letters, numbers and dot. | Required | `AnhuiTV.cn` |
|
||||
| name | Official channel name in English or call sign. May include: `a-z`, `0-9`, `space`, `-`, `!`, `:`, `&`, `.`, `+`, `'`, `/`, `»`, `#`, `%`, `°`, `$`, `@`, `?`. | Required | `Anhui TV` |
|
||||
| alt_names | List of alternative channel names separated by `;`. May contain any characters except `,` and `"`. | Optional | `安徽卫视` |
|
||||
| alt_names | List of alternative channel names separated by `;`. May contain any characters except `,` and `"`. | Optional | `安徽卫视;AHTV` |
|
||||
| network | Network of which this channel is a part. May contain any characters except `,` and `"`. | Optional | `Anhui` |
|
||||
| owners | List of channel owners separated by `;`. May contain any characters except `,` and `"`. | Optional | `China Central Television` |
|
||||
| country | Country code from which the channel is transmitted. A list of all supported countries and their codes can be found in [data/countries.csv](data/countries.csv) | Required | `CN` |
|
||||
| subdivision | Code of the subdivision (e.g., provinces or states) from which the broadcast is transmitted. A list of all supported subdivisions and their codes can be found in [data/subdivisions.csv](data/subdivisions.csv). | Optional | `CN-AH` |
|
||||
| city | The name of the city in English from which the channel is broadcast. May contain any characters except `,` and `"`. | Optional | `Hefei` |
|
||||
| broadcast_area | List of codes describing the broadcasting area of the channel separated by `;`. Any combination of `r/<region_code>`, `c/<country_code>`, `s/<subdivision_code>`. | Required | `c/CN;r/EUR` |
|
||||
| broadcast_area | List of codes describing the broadcasting area of the channel separated by `;`. Any combination of `r/<region_code>`, `c/<country_code>`, `s/<subdivision_code>`. | Required | `c/CN;r/ASIA` |
|
||||
| languages | List of languages in which the channel is broadcast separated by `;`. A list of all supported languages and their codes can be found in [data/languages.csv](data/languages.csv). | Required | `zho;eng` |
|
||||
| categories | List of categories to which this channel belongs separated by `;`. A list of all supported categories can be found in [data/categories.csv](data/categories.csv). | Optional | `animation;kids` |
|
||||
| is_nsfw | Indicates whether the channel broadcasts adult content (`TRUE` or `FALSE`). | Required | `FALSE` |
|
||||
|
|
@ -118,3 +103,44 @@ Since finding a suitable logo for the channel is not always possible, this list
|
|||
That way it's much easier to scale the logo later.
|
||||
|
||||
<img src="https://user-images.githubusercontent.com/7253922/235551815-b9925ac5-85ac-458a-bf0b-a9d57eb2547d.png" width="600"/>
|
||||
|
||||
## Project Structure
|
||||
|
||||
- `.github/`
|
||||
- `ISSUE_TEMPLATE/`: issue templates for the repository.
|
||||
- `workflows`: contains [GitHub actions](https://docs.github.com/en/actions/quickstart) workflows.
|
||||
- `CODE_OF_CONDUCT.md`: rules you shouldn't break if you don't want to get banned.
|
||||
- `.readme/`
|
||||
- `preview.png`: image displayed in the `README.md`.
|
||||
- `data/`: contains all data.
|
||||
- `scripts/`: contains all scripts used in the repository.
|
||||
- `tests/`: contains tests to check the scripts.
|
||||
- `CONTRIBUTING.md`: file you are currently reading.
|
||||
- `README.md`: project description displayed on the home page.
|
||||
|
||||
## Scripts
|
||||
|
||||
These scripts are created to automate routine processes in the repository and make it a bit easier to maintain.
|
||||
|
||||
For scripts to work, you must have [Node.js](https://nodejs.org/en) installed on your computer.
|
||||
|
||||
To run scripts use the `npm run <script-name>` command.
|
||||
|
||||
- `act:check`: allows to run the [check](https://github.com/iptv-org/iptv/blob/master/.github/workflows/check.yml) workflow locally. Depends on [nektos/act](https://github.com/nektos/act).
|
||||
- `act:update`: allows to run the [update](https://github.com/iptv-org/iptv/blob/master/.github/workflows/update.yml) workflow locally. Depends on [nektos/act](https://github.com/nektos/act).
|
||||
- `act:deploy`: allows to run the [deploy](https://github.com/iptv-org/iptv/blob/master/.github/workflows/deploy.yml) workflow locally. Depends on [nektos/act](https://github.com/nektos/act).
|
||||
- `db:validate`: checks the integrity of data.
|
||||
- `db:export`: saves all data in JSON format to the `/.api` folder.
|
||||
- `db:update`: triggers a data update using approved requests from issues.
|
||||
- `lint`: сhecks the scripts for syntax errors.
|
||||
- `test`: runs a test of all the scripts described above.
|
||||
|
||||
## Workflows
|
||||
|
||||
To automate the run of the scripts described above, we use the [GitHub Actions workflows](https://docs.github.com/en/actions/using-workflows).
|
||||
|
||||
Each workflow includes its own set of scripts that can be run either manually or in response to an event.
|
||||
|
||||
- `check`: runs the `db:validate` script when a new pull request appears, and blocks the merge if it detects an error in it.
|
||||
- `update`: sequentially runs `db:update` and `db:validate` scripts and commits all the changes if successful.
|
||||
- `deploy`: after each update of the [master](https://github.com/iptv-org/database/branches) branch runs the script `db:export` and then publishes the resulting files to the [iptv-org/api](https://github.com/iptv-org/api) repository.
|
||||
|
|
|
|||
24
README.md
24
README.md
|
|
@ -6,9 +6,31 @@ User editable database for TV channels.
|
|||
|
||||
All data is stored in the [/data](data) folder as [CSV](https://en.wikipedia.org/wiki/Comma-separated_values) (Comma-separated values) files and can be edited with any spreadsheet editor (such as [Google Sheets](https://www.google.com/sheets/about/), [LibreOffice](https://www.libreoffice.org/discover/libreoffice/), ...).
|
||||
|
||||
## API
|
||||
|
||||
All data is also available through the API, documentation for which can be found in the [iptv-org/api](https://github.com/iptv-org/api) repository.
|
||||
|
||||
## Resources
|
||||
|
||||
Links to other useful IPTV-related resources can be found in the [iptv-org/awesome-iptv](https://github.com/iptv-org/awesome-iptv) repository.
|
||||
|
||||
## Discussions
|
||||
|
||||
If you have a question or an idea, you can post it in the [Discussions](https://github.com/orgs/iptv-org/discussions) tab.
|
||||
|
||||
## Contribution
|
||||
|
||||
Please make sure to read the [Contributing Guide](CONTRIBUTING.md) before sending an issue or making a pull request.
|
||||
Please make sure to read the [Contributing Guide](https://github.com/iptv-org/epg/blob/master/CONTRIBUTING.md) before sending [issue](https://github.com/iptv-org/epg/issues) or a [pull request](https://github.com/iptv-org/epg/pulls).
|
||||
|
||||
And thank you to everyone who has already contributed!
|
||||
|
||||
### Backers
|
||||
|
||||
<a href="https://opencollective.com/iptv-org"><img src="https://opencollective.com/iptv-org/backers.svg?width=890" /></a>
|
||||
|
||||
### Contributors
|
||||
|
||||
<a href="https://github.com/iptv-org/epg/graphs/contributors"><img src="https://opencollective.com/iptv-org/contributors.svg?width=890" /></a>
|
||||
|
||||
## License
|
||||
|
||||
|
|
|
|||
|
|
@ -333,6 +333,7 @@ HGTVBrazil.br,https://github.com/iptv-org/iptv/issues/1831
|
|||
HGTVCanada.ca,https://github.com/iptv-org/iptv/issues/1831
|
||||
HGTVEast.us,https://github.com/iptv-org/iptv/issues/1831
|
||||
HGTVGermany.us,https://github.com/iptv-org/iptv/issues/1831
|
||||
HGTVHungary.hu,https://github.com/iptv-org/iptv/issues/1831
|
||||
HGTVItaly.it,https://github.com/iptv-org/iptv/issues/1831
|
||||
HGTVNewZealand.nz,https://github.com/iptv-org/iptv/issues/1831
|
||||
HGTVPanregional.us,https://github.com/iptv-org/iptv/issues/1831
|
||||
|
|
@ -393,6 +394,7 @@ Motortrend.us,https://github.com/iptv-org/iptv/issues/1831
|
|||
N1BosniaHerzegovina.ba,https://github.com/iptv-org/iptv/issues/6486
|
||||
N1Croatia.hr,https://github.com/iptv-org/iptv/issues/6486
|
||||
N1Serbia.rs,https://github.com/iptv-org/iptv/issues/6486
|
||||
Nova.bg,https://github.com/iptv-org/iptv/issues/6486
|
||||
Novacinema1.gr,https://github.com/iptv-org/iptv/issues/6486
|
||||
Novacinema2.gr,https://github.com/iptv-org/iptv/issues/6486
|
||||
Novacinema3.gr,https://github.com/iptv-org/iptv/issues/6486
|
||||
|
|
@ -415,7 +417,6 @@ Novasports4.gr,https://github.com/github/dmca/blob/master/2020/09/2020-09-16-dfl
|
|||
Novasports4.gr,https://github.com/iptv-org/iptv/issues/6486
|
||||
Novasports5.gr,https://github.com/github/dmca/blob/master/2020/09/2020-09-16-dfl.md
|
||||
Novasports6.gr,https://github.com/github/dmca/blob/master/2020/09/2020-09-16-dfl.md
|
||||
Nova.bg,https://github.com/iptv-org/iptv/issues/6486
|
||||
NovaTV.hr,https://github.com/iptv-org/iptv/issues/6486
|
||||
Nove.it,https://github.com/iptv-org/iptv/issues/1831
|
||||
One.il,https://github.com/github/dmca/blob/master/2020/09/2020-09-16-dfl.md
|
||||
|
|
|
|||
|
File diff suppressed because it is too large
Load diff
11137
package-lock.json
generated
11137
package-lock.json
generated
File diff suppressed because it is too large
Load diff
49
package.json
49
package.json
|
|
@ -2,46 +2,45 @@
|
|||
"name": "@iptv-org/database",
|
||||
"scripts": {
|
||||
"act:check": "act pull_request -W .github/workflows/check.yml",
|
||||
"act:update": "act workflow_dispatch -W .github/workflows/update.yml",
|
||||
"act:deploy": "act push -W .github/workflows/deploy.yml",
|
||||
"db:validate": "node scripts/db/validate.js",
|
||||
"db:export": "node scripts/db/export.js",
|
||||
"db:update": "node scripts/db/update.js"
|
||||
"db:validate": "ts-node scripts/db/validate.ts",
|
||||
"db:export": "ts-node scripts/db/export.ts",
|
||||
"db:update": "ts-node scripts/db/update.ts",
|
||||
"lint": "npx eslint ./scripts/**/*.ts ./scripts/**/*.js ./tests/**/*.ts",
|
||||
"test": "jest --runInBand"
|
||||
},
|
||||
"pre-commit": [
|
||||
"db:validate"
|
||||
],
|
||||
"private": true,
|
||||
"author": "Arhey",
|
||||
"jest": {
|
||||
"transform": {
|
||||
"^.+\\.(ts|js)$": "ts-jest"
|
||||
},
|
||||
"testRegex": "tests/(.*?/)?.*test.(js|ts)$"
|
||||
},
|
||||
"dependencies": {
|
||||
"@freearhey/core": "^0.2.1",
|
||||
"@joi/date": "^2.1.0",
|
||||
"@json2csv/formatters": "^7.0.3",
|
||||
"@json2csv/node": "^7.0.3",
|
||||
"@json2csv/transforms": "^7.0.3",
|
||||
"@octokit/core": "^4.2.0",
|
||||
"@octokit/plugin-paginate-rest": "^6.0.0",
|
||||
"autocrop-js": "^0.2.0",
|
||||
"axios": "^0.25.0",
|
||||
"@octokit/plugin-rest-endpoint-methods": "^7.1.3",
|
||||
"@types/jest": "^29.5.5",
|
||||
"@typescript-eslint/eslint-plugin": "^6.7.4",
|
||||
"chalk": "^4.1.2",
|
||||
"cheerio": "^1.0.0-rc.10",
|
||||
"cli-progress": "^3.11.2",
|
||||
"commander": "^9.0.0",
|
||||
"crlf": "^1.1.1",
|
||||
"crypto": "^1.0.1",
|
||||
"csvtojson": "^2.0.10",
|
||||
"dayjs": "^1.11.0",
|
||||
"form-data": "^4.0.0",
|
||||
"glob": "^7.2.0",
|
||||
"iso-639-2": "^3.0.1",
|
||||
"eslint": "^8.50.0",
|
||||
"eslint-config-prettier": "^9.0.0",
|
||||
"jest": "^29.7.0",
|
||||
"joi": "^17.6.0",
|
||||
"json2csv": "^6.0.0-alpha.0",
|
||||
"lodash": "^4.17.21",
|
||||
"mz": "^2.7.0",
|
||||
"node-cleanup": "^2.1.2",
|
||||
"pre-commit": "^1.2.2",
|
||||
"probe-image-size": "^7.2.3",
|
||||
"sharp": "^0.31.1",
|
||||
"signale": "^1.4.0",
|
||||
"slugify": "^1.6.5",
|
||||
"transliteration": "^2.2.0",
|
||||
"wikijs": "^6.3.3",
|
||||
"wtf_wikipedia": "^10.0.0",
|
||||
"wtf-plugin-image": "^1.0.0"
|
||||
"ts-jest": "^29.1.1",
|
||||
"ts-node": "^10.9.1"
|
||||
}
|
||||
}
|
||||
|
|
|
|||
5
scripts/constants.ts
Normal file
5
scripts/constants.ts
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
export const OWNER = 'iptv-org'
|
||||
export const REPO = 'database'
|
||||
export const DATA_DIR = process.env.DATA_DIR || './data'
|
||||
export const API_DIR = process.env.API_DIR || './.api'
|
||||
export const TESTING = process.env.NODE_ENV === 'test' ? true : false
|
||||
|
|
@ -1,105 +0,0 @@
|
|||
const csv2json = require('csvtojson')
|
||||
const chalk = require('chalk')
|
||||
const logger = require('./logger')
|
||||
const fs = require('mz/fs')
|
||||
const {
|
||||
Parser,
|
||||
transforms: { flatten },
|
||||
formatters: { stringQuoteOnlyIfNecessary }
|
||||
} = require('json2csv')
|
||||
|
||||
const csv2jsonOptions = {
|
||||
checkColumn: true,
|
||||
trim: true,
|
||||
delimiter: ',',
|
||||
eol: '\r\n',
|
||||
colParser: {
|
||||
alt_names: listParser,
|
||||
network: nullable,
|
||||
owners: listParser,
|
||||
subdivision: nullable,
|
||||
city: nullable,
|
||||
broadcast_area: listParser,
|
||||
languages: listParser,
|
||||
categories: listParser,
|
||||
is_nsfw: boolParser,
|
||||
launched: nullable,
|
||||
closed: nullable,
|
||||
replaced_by: nullable,
|
||||
website: nullable,
|
||||
logo: nullable,
|
||||
countries: listParser
|
||||
}
|
||||
}
|
||||
|
||||
const json2csv = new Parser({
|
||||
transforms: [flattenArray, formatBool],
|
||||
formatters: {
|
||||
string: stringQuoteOnlyIfNecessary()
|
||||
},
|
||||
eol: '\r\n'
|
||||
})
|
||||
|
||||
const csv = {}
|
||||
|
||||
csv.fromFile = async function (filepath) {
|
||||
return csv2json(csv2jsonOptions).fromFile(filepath)
|
||||
}
|
||||
|
||||
csv.fromString = async function (filepath) {
|
||||
return csv2json(csv2jsonOptions).fromString(filepath)
|
||||
}
|
||||
|
||||
csv.save = async function (filepath, data) {
|
||||
const string = json2csv.parse(data)
|
||||
|
||||
return fs.writeFile(filepath, string)
|
||||
}
|
||||
|
||||
csv.saveSync = function (filepath, data) {
|
||||
const string = json2csv.parse(data)
|
||||
|
||||
return fs.writeFileSync(filepath, string)
|
||||
}
|
||||
|
||||
module.exports = csv
|
||||
|
||||
function flattenArray(item) {
|
||||
for (let prop in item) {
|
||||
const value = item[prop]
|
||||
item[prop] = Array.isArray(value) ? value.join(';') : value
|
||||
}
|
||||
|
||||
return item
|
||||
}
|
||||
|
||||
function formatBool(item) {
|
||||
for (let prop in item) {
|
||||
if (item[prop] === false) {
|
||||
item[prop] = 'FALSE'
|
||||
} else if (item[prop] === true) {
|
||||
item[prop] = 'TRUE'
|
||||
}
|
||||
}
|
||||
|
||||
return item
|
||||
}
|
||||
|
||||
function listParser(value) {
|
||||
return value.split(';').filter(i => i)
|
||||
}
|
||||
|
||||
function boolParser(value) {
|
||||
switch (value) {
|
||||
case 'TRUE':
|
||||
return true
|
||||
case 'FALSE':
|
||||
return false
|
||||
default:
|
||||
return value
|
||||
}
|
||||
}
|
||||
|
||||
function nullable(value) {
|
||||
return value === '' ? null : value
|
||||
}
|
||||
44
scripts/core/csv.ts
Normal file
44
scripts/core/csv.ts
Normal file
|
|
@ -0,0 +1,44 @@
|
|||
import { Collection } from '@freearhey/core'
|
||||
import { Parser } from '@json2csv/plainjs'
|
||||
import { stringQuoteOnlyIfNecessary } from '@json2csv/formatters'
|
||||
|
||||
export class CSV {
|
||||
items: Collection
|
||||
|
||||
constructor({ items }: { items: Collection }) {
|
||||
this.items = items
|
||||
}
|
||||
|
||||
toString(): string {
|
||||
const parser = new Parser({
|
||||
transforms: [flattenArray, formatBool],
|
||||
formatters: {
|
||||
string: stringQuoteOnlyIfNecessary()
|
||||
},
|
||||
eol: '\r\n'
|
||||
})
|
||||
|
||||
return parser.parse(this.items.all())
|
||||
}
|
||||
}
|
||||
|
||||
function flattenArray(item: { [key: string]: string[] | string | boolean }) {
|
||||
for (const prop in item) {
|
||||
const value = item[prop]
|
||||
item[prop] = Array.isArray(value) ? value.join(';') : value
|
||||
}
|
||||
|
||||
return item
|
||||
}
|
||||
|
||||
function formatBool(item: { [key: string]: string[] | string | boolean }) {
|
||||
for (const prop in item) {
|
||||
if (item[prop] === false) {
|
||||
item[prop] = 'FALSE'
|
||||
} else if (item[prop] === true) {
|
||||
item[prop] = 'TRUE'
|
||||
}
|
||||
}
|
||||
|
||||
return item
|
||||
}
|
||||
53
scripts/core/csvParser.ts
Normal file
53
scripts/core/csvParser.ts
Normal file
|
|
@ -0,0 +1,53 @@
|
|||
import { Collection } from '@freearhey/core'
|
||||
import csv2json from 'csvtojson'
|
||||
|
||||
const opts = {
|
||||
checkColumn: true,
|
||||
trim: true,
|
||||
delimiter: ',',
|
||||
eol: '\r\n',
|
||||
colParser: {
|
||||
alt_names: listParser,
|
||||
network: nullable,
|
||||
owners: listParser,
|
||||
subdivision: nullable,
|
||||
city: nullable,
|
||||
broadcast_area: listParser,
|
||||
languages: listParser,
|
||||
categories: listParser,
|
||||
is_nsfw: boolParser,
|
||||
launched: nullable,
|
||||
closed: nullable,
|
||||
replaced_by: nullable,
|
||||
website: nullable,
|
||||
logo: nullable,
|
||||
countries: listParser
|
||||
}
|
||||
}
|
||||
|
||||
export class CSVParser {
|
||||
async parse(data: string): Promise<Collection> {
|
||||
const items = await csv2json(opts).fromString(data)
|
||||
|
||||
return new Collection(items)
|
||||
}
|
||||
}
|
||||
|
||||
function listParser(value: string) {
|
||||
return value.split(';').filter(i => i)
|
||||
}
|
||||
|
||||
function boolParser(value: string) {
|
||||
switch (value) {
|
||||
case 'TRUE':
|
||||
return true
|
||||
case 'FALSE':
|
||||
return false
|
||||
default:
|
||||
return value
|
||||
}
|
||||
}
|
||||
|
||||
function nullable(value: string) {
|
||||
return value === '' ? null : value
|
||||
}
|
||||
|
|
@ -1,78 +0,0 @@
|
|||
const path = require('path')
|
||||
const glob = require('glob')
|
||||
const fs = require('mz/fs')
|
||||
const crlf = require('crlf')
|
||||
|
||||
const file = {}
|
||||
|
||||
file.list = function (pattern) {
|
||||
return new Promise(resolve => {
|
||||
glob(pattern, function (err, files) {
|
||||
resolve(files)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
file.getFilename = function (filepath) {
|
||||
return path.parse(filepath).name
|
||||
}
|
||||
|
||||
file.createDir = async function (dir) {
|
||||
if (await file.exists(dir)) return
|
||||
|
||||
return fs.mkdir(dir, { recursive: true }).catch(console.error)
|
||||
}
|
||||
|
||||
file.exists = function (filepath) {
|
||||
return fs.exists(path.resolve(filepath))
|
||||
}
|
||||
|
||||
file.read = function (filepath) {
|
||||
return fs.readFile(path.resolve(filepath), { encoding: 'utf8' }).catch(console.error)
|
||||
}
|
||||
|
||||
file.append = function (filepath, data) {
|
||||
return fs.appendFile(path.resolve(filepath), data).catch(console.error)
|
||||
}
|
||||
|
||||
file.create = function (filepath, data = '') {
|
||||
filepath = path.resolve(filepath)
|
||||
const dir = path.dirname(filepath)
|
||||
|
||||
return file
|
||||
.createDir(dir)
|
||||
.then(() => file.write(filepath, data))
|
||||
.catch(console.error)
|
||||
}
|
||||
|
||||
file.write = function (filepath, data = '') {
|
||||
return fs.writeFile(path.resolve(filepath), data, { encoding: 'utf8' }).catch(console.error)
|
||||
}
|
||||
|
||||
file.clear = async function (filepath) {
|
||||
if (await file.exists(filepath)) return file.write(filepath, '')
|
||||
return true
|
||||
}
|
||||
|
||||
file.resolve = function (filepath) {
|
||||
return path.resolve(filepath)
|
||||
}
|
||||
|
||||
file.dirname = function (filepath) {
|
||||
return path.dirname(filepath)
|
||||
}
|
||||
|
||||
file.basename = function (filepath) {
|
||||
return path.basename(filepath)
|
||||
}
|
||||
|
||||
file.eol = function (filepath) {
|
||||
return new Promise((resolve, reject) => {
|
||||
crlf.get(filepath, null, function (err, endingType) {
|
||||
if (err) reject(err)
|
||||
resolve(endingType)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
module.exports = file
|
||||
17
scripts/core/idCreator.ts
Normal file
17
scripts/core/idCreator.ts
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
export class IDCreator {
|
||||
create(name: string, country: string): string {
|
||||
const slug = normalize(name)
|
||||
const code = country.toLowerCase()
|
||||
|
||||
return `${slug}.${code}`
|
||||
}
|
||||
}
|
||||
|
||||
function normalize(name: string) {
|
||||
return name
|
||||
.replace(/^@/gi, 'At')
|
||||
.replace(/^&/i, 'And')
|
||||
.replace(/\+/gi, 'Plus')
|
||||
.replace(/\s-(\d)/gi, ' Minus$1')
|
||||
.replace(/[^a-z\d]+/gi, '')
|
||||
}
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
exports.csv = require('./csv')
|
||||
exports.file = require('./file')
|
||||
exports.logger = require('./logger')
|
||||
5
scripts/core/index.ts
Normal file
5
scripts/core/index.ts
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
export * from './csv'
|
||||
export * from './issueParser'
|
||||
export * from './issueLoader'
|
||||
export * from './csvParser'
|
||||
export * from './idCreator'
|
||||
49
scripts/core/issueLoader.ts
Normal file
49
scripts/core/issueLoader.ts
Normal file
|
|
@ -0,0 +1,49 @@
|
|||
import { Collection } from '@freearhey/core'
|
||||
import { restEndpointMethods } from '@octokit/plugin-rest-endpoint-methods'
|
||||
import { paginateRest } from '@octokit/plugin-paginate-rest'
|
||||
import { Octokit } from '@octokit/core'
|
||||
import { IssueParser } from './'
|
||||
import { TESTING, OWNER, REPO } from '../constants'
|
||||
|
||||
const CustomOctokit = Octokit.plugin(paginateRest, restEndpointMethods)
|
||||
const octokit = new CustomOctokit()
|
||||
|
||||
export class IssueLoader {
|
||||
async load({ labels }: { labels: string[] | string }) {
|
||||
labels = Array.isArray(labels) ? labels.join(',') : labels
|
||||
let issues: object[] = []
|
||||
if (TESTING) {
|
||||
switch (labels) {
|
||||
case 'channels:add,approved':
|
||||
issues = require('../../tests/__data__/input/issues/channels_add_approved.js')
|
||||
break
|
||||
case 'channels:edit,approved':
|
||||
issues = require('../../tests/__data__/input/issues/channels_edit_approved.js')
|
||||
break
|
||||
case 'channels:remove,approved':
|
||||
issues = require('../../tests/__data__/input/issues/channels_remove_approved.js')
|
||||
break
|
||||
case 'blocklist:add,approved':
|
||||
issues = require('../../tests/__data__/input/issues/blocklist_add_approved.js')
|
||||
break
|
||||
case 'blocklist:remove,approved':
|
||||
issues = require('../../tests/__data__/input/issues/blocklist_remove_approved.js')
|
||||
break
|
||||
}
|
||||
} else {
|
||||
issues = await octokit.paginate(octokit.rest.issues.listForRepo, {
|
||||
owner: OWNER,
|
||||
repo: REPO,
|
||||
per_page: 100,
|
||||
labels,
|
||||
headers: {
|
||||
'X-GitHub-Api-Version': '2022-11-28'
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
const parser = new IssueParser()
|
||||
|
||||
return new Collection(issues).map(parser.parse)
|
||||
}
|
||||
}
|
||||
66
scripts/core/issueParser.ts
Normal file
66
scripts/core/issueParser.ts
Normal file
|
|
@ -0,0 +1,66 @@
|
|||
import { Dictionary } from '@freearhey/core'
|
||||
import { Issue } from '../models'
|
||||
|
||||
const FIELDS = new Dictionary({
|
||||
'Channel ID': 'channel_id',
|
||||
'Channel ID (required)': 'channel_id',
|
||||
'Channel ID (optional)': 'channel_id',
|
||||
'Channel Name': 'name',
|
||||
'Alternative Names': 'alt_names',
|
||||
'Alternative Names (optional)': 'alt_names',
|
||||
Network: 'network',
|
||||
'Network (optional)': 'network',
|
||||
Owners: 'owners',
|
||||
'Owners (optional)': 'owners',
|
||||
Country: 'country',
|
||||
Subdivision: 'subdivision',
|
||||
'Subdivision (optional)': 'subdivision',
|
||||
City: 'city',
|
||||
'City (optional)': 'city',
|
||||
'Broadcast Area': 'broadcast_area',
|
||||
Languages: 'languages',
|
||||
Categories: 'categories',
|
||||
'Categories (optional)': 'categories',
|
||||
NSFW: 'is_nsfw',
|
||||
Launched: 'launched',
|
||||
'Launched (optional)': 'launched',
|
||||
Closed: 'closed',
|
||||
'Closed (optional)': 'closed',
|
||||
'Replaced By': 'replaced_by',
|
||||
'Replaced By (optional)': 'replaced_by',
|
||||
Website: 'website',
|
||||
'Website (optional)': 'website',
|
||||
Logo: 'logo',
|
||||
Reason: 'reason',
|
||||
Notes: 'notes',
|
||||
'Notes (optional)': 'notes',
|
||||
Reference: 'ref',
|
||||
'Reference (optional)': 'ref',
|
||||
'Reference (required)': 'ref'
|
||||
})
|
||||
|
||||
export class IssueParser {
|
||||
parse(issue: { number: number; body: string; labels: { name: string }[] }): Issue {
|
||||
const fields = issue.body.split('###')
|
||||
|
||||
const data = new Dictionary()
|
||||
fields.forEach((field: string) => {
|
||||
let [_label, , _value] = field.split(/\r?\n/)
|
||||
_label = _label ? _label.trim() : ''
|
||||
_value = _value ? _value.trim() : ''
|
||||
|
||||
if (!_label || !_value) return data
|
||||
|
||||
const id: string = FIELDS.get(_label)
|
||||
const value: string = _value === '_No response_' || _value === 'None' ? '' : _value
|
||||
|
||||
if (!id) return
|
||||
|
||||
data.set(id, value)
|
||||
})
|
||||
|
||||
const labels = issue.labels.map(label => label.name)
|
||||
|
||||
return new Issue({ number: issue.number, labels, data })
|
||||
}
|
||||
}
|
||||
|
|
@ -1,13 +0,0 @@
|
|||
const { Signale } = require('signale')
|
||||
|
||||
const options = {}
|
||||
|
||||
const logger = new Signale(options)
|
||||
|
||||
logger.config({
|
||||
displayLabel: false,
|
||||
displayScope: false,
|
||||
displayBadge: false
|
||||
})
|
||||
|
||||
module.exports = logger
|
||||
|
|
@ -1,19 +0,0 @@
|
|||
const { csv, file, logger } = require('../core')
|
||||
const chalk = require('chalk')
|
||||
|
||||
const DATA_DIR = process.env.DATA_DIR || './data'
|
||||
const OUTPUT_DIR = process.env.OUTPUT_DIR || './.api'
|
||||
|
||||
async function main() {
|
||||
const files = await file.list(`${DATA_DIR}/*.csv`)
|
||||
for (const filepath of files) {
|
||||
const filename = file.getFilename(filepath)
|
||||
const json = await csv.fromFile(filepath).catch(err => {
|
||||
logger.error(chalk.red(`\n${err.message} (${filepath})`))
|
||||
process.exit(1)
|
||||
})
|
||||
await file.create(`${OUTPUT_DIR}/${filename}.json`, JSON.stringify(json))
|
||||
}
|
||||
}
|
||||
|
||||
main()
|
||||
21
scripts/db/export.ts
Normal file
21
scripts/db/export.ts
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
import { Storage, File } from '@freearhey/core'
|
||||
import { DATA_DIR, API_DIR } from '../constants'
|
||||
import { CSVParser } from '../core'
|
||||
|
||||
async function main() {
|
||||
const dataStorage = new Storage(DATA_DIR)
|
||||
const apiStorage = new Storage(API_DIR)
|
||||
const parser = new CSVParser()
|
||||
|
||||
const files = await dataStorage.list('*.csv')
|
||||
for (const filepath of files) {
|
||||
const file = new File(filepath)
|
||||
const filename = file.name()
|
||||
const data = await dataStorage.load(file.basename())
|
||||
const items = await parser.parse(data)
|
||||
|
||||
await apiStorage.save(`${filename}.json`, items.toJSON())
|
||||
}
|
||||
}
|
||||
|
||||
main()
|
||||
|
|
@ -1,172 +0,0 @@
|
|||
const { csv, file } = require('../core')
|
||||
const channelScheme = require('../db/schemes/channels')
|
||||
const { Octokit } = require('@octokit/core')
|
||||
const { paginateRest } = require('@octokit/plugin-paginate-rest')
|
||||
const CustomOctokit = Octokit.plugin(paginateRest)
|
||||
const _ = require('lodash')
|
||||
|
||||
const octokit = new CustomOctokit()
|
||||
|
||||
const DATA_DIR = process.env.DATA_DIR || './data'
|
||||
const OWNER = 'iptv-org'
|
||||
const REPO = 'database'
|
||||
|
||||
let channels = []
|
||||
let processedIssues = []
|
||||
|
||||
async function main() {
|
||||
try {
|
||||
const filepath = `${DATA_DIR}/channels.csv`
|
||||
channels = await csv.fromFile(filepath)
|
||||
|
||||
await removeChannels()
|
||||
await editChannels()
|
||||
await addChannels()
|
||||
|
||||
channels = _.orderBy(channels, [channels => channels.id.toLowerCase()], ['asc'])
|
||||
await csv.save(filepath, channels)
|
||||
|
||||
const output = processedIssues.map(issue => `closes #${issue.number}`).join(', ')
|
||||
console.log(`OUTPUT=${output}`)
|
||||
} catch (err) {
|
||||
console.log(err.message)
|
||||
}
|
||||
}
|
||||
|
||||
main()
|
||||
|
||||
async function removeChannels() {
|
||||
const issues = await fetchIssues('channels:remove,approved')
|
||||
issues.map(parseIssue).forEach(({ issue, channel }) => {
|
||||
if (!channel) return
|
||||
|
||||
const index = _.findIndex(channels, { id: channel.id })
|
||||
if (index < 0) return
|
||||
|
||||
channels.splice(index, 1)
|
||||
|
||||
processedIssues.push(issue)
|
||||
})
|
||||
}
|
||||
|
||||
async function editChannels() {
|
||||
const issues = await fetchIssues('channels:edit,approved')
|
||||
issues.map(parseIssue).forEach(({ issue, channel }) => {
|
||||
if (!channel) return
|
||||
|
||||
const index = _.findIndex(channels, { id: channel.id })
|
||||
if (index < 0) return
|
||||
|
||||
const found = channels[index]
|
||||
|
||||
for (let prop in channel) {
|
||||
if (channel[prop] !== undefined) {
|
||||
found[prop] = channel[prop]
|
||||
}
|
||||
}
|
||||
|
||||
found.id = generateChannelId(found.name, found.country)
|
||||
|
||||
channels.splice(index, 1, found)
|
||||
|
||||
processedIssues.push(issue)
|
||||
})
|
||||
}
|
||||
|
||||
async function addChannels() {
|
||||
const issues = await fetchIssues('channels:add,approved')
|
||||
issues.map(parseIssue).forEach(({ issue, channel }) => {
|
||||
if (!channel) return
|
||||
|
||||
const found = channels.find(c => c.id === channel.id)
|
||||
if (found) return
|
||||
|
||||
channels.push(channel)
|
||||
processedIssues.push(issue)
|
||||
})
|
||||
}
|
||||
|
||||
async function fetchIssues(labels) {
|
||||
const issues = await octokit.paginate('GET /repos/{owner}/{repo}/issues', {
|
||||
owner: OWNER,
|
||||
repo: REPO,
|
||||
per_page: 100,
|
||||
labels,
|
||||
headers: {
|
||||
'X-GitHub-Api-Version': '2022-11-28'
|
||||
}
|
||||
})
|
||||
|
||||
return issues
|
||||
}
|
||||
|
||||
function parseIssue(issue) {
|
||||
const buffer = {}
|
||||
const channel = {}
|
||||
const fieldLabels = {
|
||||
'Channel ID (required)': 'id',
|
||||
'Channel Name': 'name',
|
||||
'Alternative Names': 'alt_names',
|
||||
'Alternative Names (optional)': 'alt_names',
|
||||
Network: 'network',
|
||||
'Network (optional)': 'network',
|
||||
Owners: 'owners',
|
||||
'Owners (optional)': 'owners',
|
||||
Country: 'country',
|
||||
Subdivision: 'subdivision',
|
||||
'Subdivision (optional)': 'subdivision',
|
||||
City: 'city',
|
||||
'City (optional)': 'city',
|
||||
'Broadcast Area': 'broadcast_area',
|
||||
Languages: 'languages',
|
||||
Categories: 'categories',
|
||||
'Categories (optional)': 'categories',
|
||||
NSFW: 'is_nsfw',
|
||||
Launched: 'launched',
|
||||
'Launched (optional)': 'launched',
|
||||
Closed: 'closed',
|
||||
'Closed (optional)': 'closed',
|
||||
'Replaced By': 'replaced_by',
|
||||
'Replaced By (optional)': 'replaced_by',
|
||||
Website: 'website',
|
||||
'Website (optional)': 'website',
|
||||
Logo: 'logo'
|
||||
}
|
||||
|
||||
const fields = issue.body.split('###')
|
||||
|
||||
if (!fields.length) return { issue, channel: null }
|
||||
|
||||
fields.forEach(item => {
|
||||
const [fieldLabel, , value] = item.split(/\r?\n/)
|
||||
const field = fieldLabel ? fieldLabels[fieldLabel.trim()] : null
|
||||
|
||||
if (!field) return
|
||||
|
||||
buffer[field] = value.includes('_No response_') ? undefined : value.trim()
|
||||
})
|
||||
|
||||
for (let field of Object.keys(channelScheme)) {
|
||||
channel[field] = buffer[field]
|
||||
}
|
||||
|
||||
if (!channel.id) {
|
||||
channel.id = generateChannelId(channel.name, channel.country)
|
||||
}
|
||||
|
||||
return { issue, channel }
|
||||
}
|
||||
|
||||
function generateChannelId(name, country) {
|
||||
if (name && country) {
|
||||
const slug = name
|
||||
.replace(/\+/gi, 'Plus')
|
||||
.replace(/^@/gi, 'At')
|
||||
.replace(/[^a-z\d]+/gi, '')
|
||||
country = country.toLowerCase()
|
||||
|
||||
return `${slug}.${country}`
|
||||
}
|
||||
|
||||
return null
|
||||
}
|
||||
185
scripts/db/update.ts
Normal file
185
scripts/db/update.ts
Normal file
|
|
@ -0,0 +1,185 @@
|
|||
import { CSV, IssueLoader, CSVParser, IDCreator } from '../core'
|
||||
import { Channel, Blocked, Issue } from '../models'
|
||||
import { DATA_DIR } from '../constants'
|
||||
import { Storage, Collection } from '@freearhey/core'
|
||||
|
||||
let blocklist = new Collection()
|
||||
let channels = new Collection()
|
||||
const processedIssues = new Collection()
|
||||
|
||||
async function main() {
|
||||
const idCreator = new IDCreator()
|
||||
const dataStorage = new Storage(DATA_DIR)
|
||||
const parser = new CSVParser()
|
||||
|
||||
const _channels = await dataStorage.load('channels.csv')
|
||||
channels = (await parser.parse(_channels)).map(data => new Channel(data))
|
||||
|
||||
const _blocklist = await dataStorage.load('blocklist.csv')
|
||||
blocklist = (await parser.parse(_blocklist)).map(data => new Blocked(data))
|
||||
|
||||
const loader = new IssueLoader()
|
||||
|
||||
await removeChannels({ loader })
|
||||
await editChannels({ loader, idCreator })
|
||||
await addChannels({ loader, idCreator })
|
||||
await blockChannels({ loader })
|
||||
await unblockChannels({ loader })
|
||||
|
||||
channels = sortBy(channels, 'id')
|
||||
const channelsOutput = new CSV({ items: channels }).toString()
|
||||
await dataStorage.save('channels.csv', channelsOutput)
|
||||
|
||||
blocklist = sortBy(blocklist, 'channel')
|
||||
const blocklistOutput = new CSV({ items: blocklist }).toString()
|
||||
await dataStorage.save('blocklist.csv', blocklistOutput)
|
||||
|
||||
const output = processedIssues.map((issue: Issue) => `closes #${issue.number}`).join(', ')
|
||||
process.stdout.write(`OUTPUT=${output}`)
|
||||
}
|
||||
|
||||
main()
|
||||
|
||||
function sortBy(channels: Collection, key: string) {
|
||||
const items = channels.all().sort((a, b) => {
|
||||
const normA = a[key].toLowerCase()
|
||||
const normB = b[key].toLowerCase()
|
||||
if (normA < normB) return -1
|
||||
if (normA > normB) return 1
|
||||
return 0
|
||||
})
|
||||
|
||||
return new Collection(items)
|
||||
}
|
||||
|
||||
async function removeChannels({ loader }: { loader: IssueLoader }) {
|
||||
const issues = await loader.load({ labels: ['channels:remove,approved'] })
|
||||
issues.forEach((issue: Issue) => {
|
||||
if (issue.data.missing('channel_id')) return
|
||||
|
||||
const found = channels.first((channel: Channel) => channel.id === issue.data.get('channel_id'))
|
||||
if (!found) return
|
||||
|
||||
channels.remove((channel: Channel) => channel.id === found.id)
|
||||
|
||||
processedIssues.push(issue)
|
||||
})
|
||||
}
|
||||
|
||||
async function editChannels({ loader, idCreator }: { loader: IssueLoader; idCreator: IDCreator }) {
|
||||
const issues = await loader.load({ labels: ['channels:edit,approved'] })
|
||||
issues.forEach((issue: Issue) => {
|
||||
const data = issue.data
|
||||
if (data.missing('channel_id')) return
|
||||
|
||||
const found: Channel = channels.first(
|
||||
(channel: Channel) => channel.id === data.get('channel_id')
|
||||
)
|
||||
if (!found) return
|
||||
|
||||
let channelId = found.id
|
||||
if (data.has('name') || data.has('country')) {
|
||||
const name = data.get('name') || found.name
|
||||
const country = data.get('country') || found.country
|
||||
channelId = idCreator.create(name, country)
|
||||
}
|
||||
|
||||
found.update({
|
||||
id: channelId,
|
||||
name: data.get('name'),
|
||||
alt_names: data.get('alt_names'),
|
||||
network: data.get('network'),
|
||||
owners: data.get('owners'),
|
||||
country: data.get('country'),
|
||||
subdivision: data.get('subdivision'),
|
||||
city: data.get('city'),
|
||||
broadcast_area: data.get('broadcast_area'),
|
||||
languages: data.get('languages'),
|
||||
categories: data.get('categories'),
|
||||
is_nsfw: data.get('is_nsfw'),
|
||||
launched: data.get('launched'),
|
||||
closed: data.get('closed'),
|
||||
replaced_by: data.get('replaced_by'),
|
||||
website: data.get('website'),
|
||||
logo: data.get('logo')
|
||||
})
|
||||
|
||||
processedIssues.push(issue)
|
||||
})
|
||||
}
|
||||
|
||||
async function addChannels({ loader, idCreator }: { loader: IssueLoader; idCreator: IDCreator }) {
|
||||
const issues = await loader.load({ labels: ['channels:add,approved'] })
|
||||
issues.forEach((issue: Issue) => {
|
||||
const data = issue.data
|
||||
if (data.missing('name') || data.missing('country')) return
|
||||
|
||||
const channelId = idCreator.create(data.get('name'), data.get('country'))
|
||||
|
||||
const found: Channel = channels.first((channel: Channel) => channel.id === channelId)
|
||||
if (found) return
|
||||
|
||||
channels.push(
|
||||
new Channel({
|
||||
id: channelId,
|
||||
name: data.get('name'),
|
||||
alt_names: data.get('alt_names'),
|
||||
network: data.get('network'),
|
||||
owners: data.get('owners'),
|
||||
country: data.get('country'),
|
||||
subdivision: data.get('subdivision'),
|
||||
city: data.get('city'),
|
||||
broadcast_area: data.get('broadcast_area'),
|
||||
languages: data.get('languages'),
|
||||
categories: data.get('categories'),
|
||||
is_nsfw: data.get('is_nsfw'),
|
||||
launched: data.get('launched'),
|
||||
closed: data.get('closed'),
|
||||
replaced_by: data.get('replaced_by'),
|
||||
website: data.get('website'),
|
||||
logo: data.get('logo')
|
||||
})
|
||||
)
|
||||
|
||||
processedIssues.push(issue)
|
||||
})
|
||||
}
|
||||
|
||||
async function unblockChannels({ loader }: { loader: IssueLoader }) {
|
||||
const issues = await loader.load({ labels: ['blocklist:remove,approved'] })
|
||||
issues.forEach((issue: Issue) => {
|
||||
const data = issue.data
|
||||
if (data.missing('channel_id')) return
|
||||
|
||||
const found: Blocked = blocklist.first(
|
||||
(blocked: Blocked) => blocked.channel === data.get('channel_id')
|
||||
)
|
||||
if (!found) return
|
||||
|
||||
blocklist.remove((blocked: Blocked) => blocked.channel === found.channel)
|
||||
|
||||
processedIssues.push(issue)
|
||||
})
|
||||
}
|
||||
|
||||
async function blockChannels({ loader }: { loader: IssueLoader }) {
|
||||
const issues = await loader.load({ labels: ['blocklist:add,approved'] })
|
||||
issues.forEach((issue: Issue) => {
|
||||
const data = issue.data
|
||||
if (data.missing('channel_id')) return
|
||||
|
||||
const found: Blocked = blocklist.first(
|
||||
(blocked: Blocked) => blocked.channel === data.get('channel_id')
|
||||
)
|
||||
if (found) return
|
||||
|
||||
blocklist.push(
|
||||
new Blocked({
|
||||
channel: data.get('channel_id'),
|
||||
ref: data.get('ref')
|
||||
})
|
||||
)
|
||||
|
||||
processedIssues.push(issue)
|
||||
})
|
||||
}
|
||||
|
|
@ -1,334 +0,0 @@
|
|||
const { transliterate } = require('transliteration')
|
||||
const { logger, file, csv } = require('../core')
|
||||
const { program } = require('commander')
|
||||
const schemes = require('./schemes')
|
||||
const chalk = require('chalk')
|
||||
const Joi = require('joi')
|
||||
const _ = require('lodash')
|
||||
|
||||
program.argument('[filepath]', 'Path to file to validate').parse(process.argv)
|
||||
|
||||
const allFiles = [
|
||||
'data/blocklist.csv',
|
||||
'data/categories.csv',
|
||||
'data/channels.csv',
|
||||
'data/countries.csv',
|
||||
'data/languages.csv',
|
||||
'data/regions.csv',
|
||||
'data/subdivisions.csv'
|
||||
]
|
||||
|
||||
let db = {}
|
||||
let files = {}
|
||||
|
||||
async function main() {
|
||||
let globalErrors = []
|
||||
|
||||
for (let filepath of allFiles) {
|
||||
if (!filepath.endsWith('.csv')) continue
|
||||
|
||||
const csvString = await file.read(filepath)
|
||||
if (/\s+$/.test(csvString))
|
||||
return handleError(`Error: empty lines at the end of file not allowed (${filepath})`)
|
||||
|
||||
const rows = csvString.split(/\r\n/)
|
||||
const headers = rows[0].split(',')
|
||||
for (let [i, line] of rows.entries()) {
|
||||
if (line.indexOf('\n') > -1)
|
||||
return handleError(
|
||||
`Error: row ${i + 1} has the wrong line ending character, should be CRLF (${filepath})`
|
||||
)
|
||||
if (line.split(/,(?=(?:(?:[^"]*"){2})*[^"]*$)/).length !== headers.length)
|
||||
return handleError(`Error: row ${i + 1} has the wrong number of columns (${filepath})`)
|
||||
}
|
||||
|
||||
const filename = file.getFilename(filepath)
|
||||
let data = await csv
|
||||
.fromString(csvString)
|
||||
.catch(err => handleError(`${err.message} (${filepath})`))
|
||||
|
||||
let grouped
|
||||
switch (filename) {
|
||||
case 'blocklist':
|
||||
grouped = _.keyBy(data, 'channel')
|
||||
break
|
||||
case 'categories':
|
||||
case 'channels':
|
||||
grouped = _.keyBy(data, 'id')
|
||||
break
|
||||
default:
|
||||
grouped = _.keyBy(data, 'code')
|
||||
break
|
||||
}
|
||||
|
||||
db[filename] = grouped
|
||||
files[filename] = data
|
||||
}
|
||||
|
||||
const toCheck = program.args.length ? program.args : allFiles
|
||||
for (const filepath of toCheck) {
|
||||
const filename = file.getFilename(filepath)
|
||||
if (!schemes[filename]) return handleError(`Error: "${filename}" scheme is missing`)
|
||||
|
||||
const rows = files[filename]
|
||||
const rowsCopy = JSON.parse(JSON.stringify(rows))
|
||||
|
||||
let fileErrors = []
|
||||
if (filename === 'channels') {
|
||||
fileErrors = fileErrors.concat(findDuplicatesById(rowsCopy))
|
||||
// fileErrors = fileErrors.concat(findDuplicatesByName(rowsCopy))
|
||||
for (const [i, row] of rowsCopy.entries()) {
|
||||
fileErrors = fileErrors.concat(validateChannelId(row, i))
|
||||
fileErrors = fileErrors.concat(validateChannelBroadcastArea(row, i))
|
||||
fileErrors = fileErrors.concat(validateChannelSubdivision(row, i))
|
||||
fileErrors = fileErrors.concat(validateChannelCategories(row, i))
|
||||
fileErrors = fileErrors.concat(validateChannelReplacedBy(row, i))
|
||||
fileErrors = fileErrors.concat(validateChannelLanguages(row, i))
|
||||
fileErrors = fileErrors.concat(validateChannelCountry(row, i))
|
||||
}
|
||||
} else if (filename === 'blocklist') {
|
||||
for (const [i, row] of rowsCopy.entries()) {
|
||||
fileErrors = fileErrors.concat(validateChannel(row, i))
|
||||
}
|
||||
} else if (filename === 'countries') {
|
||||
for (const [i, row] of rowsCopy.entries()) {
|
||||
fileErrors = fileErrors.concat(validateCountryLanguages(row, i))
|
||||
}
|
||||
} else if (filename === 'subdivisions') {
|
||||
for (const [i, row] of rowsCopy.entries()) {
|
||||
fileErrors = fileErrors.concat(validateSubdivisionCountry(row, i))
|
||||
}
|
||||
} else if (filename === 'regions') {
|
||||
for (const [i, row] of rowsCopy.entries()) {
|
||||
fileErrors = fileErrors.concat(validateRegionCountries(row, i))
|
||||
}
|
||||
}
|
||||
|
||||
const schema = Joi.object(schemes[filename])
|
||||
rows.forEach((row, i) => {
|
||||
const { error } = schema.validate(row, { abortEarly: false })
|
||||
if (error) {
|
||||
error.details.forEach(detail => {
|
||||
fileErrors.push({ line: i + 2, message: detail.message })
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
if (fileErrors.length) {
|
||||
logger.info(`\n${chalk.underline(filepath)}`)
|
||||
fileErrors.forEach(err => {
|
||||
const position = err.line.toString().padEnd(6, ' ')
|
||||
logger.info(` ${chalk.gray(position)} ${err.message}`)
|
||||
})
|
||||
globalErrors = globalErrors.concat(fileErrors)
|
||||
}
|
||||
}
|
||||
|
||||
if (globalErrors.length) return handleError(`${globalErrors.length} error(s)`)
|
||||
}
|
||||
|
||||
main()
|
||||
|
||||
function findDuplicatesById(rows) {
|
||||
const errors = []
|
||||
const buffer = {}
|
||||
rows.forEach((row, i) => {
|
||||
const normId = row.id.toLowerCase()
|
||||
if (buffer[normId]) {
|
||||
errors.push({
|
||||
line: i + 2,
|
||||
message: `entry with the id "${row.id}" already exists`
|
||||
})
|
||||
}
|
||||
|
||||
buffer[normId] = true
|
||||
})
|
||||
|
||||
return errors
|
||||
}
|
||||
|
||||
function findDuplicatesByName(rows) {
|
||||
const errors = []
|
||||
const buffer = {}
|
||||
rows.forEach((row, i) => {
|
||||
const normName = row.name.toLowerCase()
|
||||
if (buffer[normName]) {
|
||||
errors.push({
|
||||
line: i + 2,
|
||||
message: `entry with the name "${row.name}" already exists`
|
||||
})
|
||||
}
|
||||
|
||||
buffer[normName] = true
|
||||
})
|
||||
|
||||
return errors
|
||||
}
|
||||
|
||||
function validateChannelId(row, i) {
|
||||
const errors = []
|
||||
|
||||
let name = normalize(row.name)
|
||||
let code = row.country.toLowerCase()
|
||||
let expected = `${name}.${code}`
|
||||
|
||||
if (expected !== row.id) {
|
||||
errors.push({
|
||||
line: i + 2,
|
||||
message: `"${row.id}" must be derived from the channel name "${row.name}" and the country code "${row.country}"`
|
||||
})
|
||||
}
|
||||
|
||||
function normalize(name) {
|
||||
let translit = transliterate(name)
|
||||
|
||||
return translit
|
||||
.replace(/^@/i, 'At')
|
||||
.replace(/^&/i, 'And')
|
||||
.replace(/\+/gi, 'Plus')
|
||||
.replace(/\s\-(\d)/gi, ' Minus$1')
|
||||
.replace(/[^a-z\d]+/gi, '')
|
||||
}
|
||||
|
||||
return errors
|
||||
}
|
||||
|
||||
function validateChannelCategories(row, i) {
|
||||
const errors = []
|
||||
row.categories.forEach(category => {
|
||||
if (!db.categories[category]) {
|
||||
errors.push({
|
||||
line: i + 2,
|
||||
message: `"${row.id}" has the wrong category "${category}"`
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
return errors
|
||||
}
|
||||
|
||||
function validateChannelCountry(row, i) {
|
||||
const errors = []
|
||||
if (!db.countries[row.country]) {
|
||||
errors.push({
|
||||
line: i + 2,
|
||||
message: `"${row.id}" has the wrong country "${row.country}"`
|
||||
})
|
||||
}
|
||||
|
||||
return errors
|
||||
}
|
||||
|
||||
function validateChannelReplacedBy(row, i) {
|
||||
const errors = []
|
||||
if (row.replaced_by && !db.channels[row.replaced_by]) {
|
||||
errors.push({
|
||||
line: i + 2,
|
||||
message: `"${row.id}" has the wrong replaced_by "${row.replaced_by}"`
|
||||
})
|
||||
}
|
||||
|
||||
return errors
|
||||
}
|
||||
|
||||
function validateChannelSubdivision(row, i) {
|
||||
const errors = []
|
||||
if (row.subdivision && !db.subdivisions[row.subdivision]) {
|
||||
errors.push({
|
||||
line: i + 2,
|
||||
message: `"${row.id}" has the wrong subdivision "${row.subdivision}"`
|
||||
})
|
||||
}
|
||||
|
||||
return errors
|
||||
}
|
||||
|
||||
function validateChannelBroadcastArea(row, i) {
|
||||
const errors = []
|
||||
row.broadcast_area.forEach(area => {
|
||||
const [type, code] = area.split('/')
|
||||
if (
|
||||
(type === 'r' && !db.regions[code]) ||
|
||||
(type === 'c' && !db.countries[code]) ||
|
||||
(type === 's' && !db.subdivisions[code])
|
||||
) {
|
||||
errors.push({
|
||||
line: i + 2,
|
||||
message: `"${row.id}" has the wrong broadcast_area "${area}"`
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
return errors
|
||||
}
|
||||
|
||||
function validateChannelLanguages(row, i) {
|
||||
const errors = []
|
||||
row.languages.forEach(language => {
|
||||
if (!db.languages[language]) {
|
||||
errors.push({
|
||||
line: i + 2,
|
||||
message: `"${row.id}" has the wrong language "${language}"`
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
return errors
|
||||
}
|
||||
|
||||
function validateChannel(row, i) {
|
||||
const errors = []
|
||||
if (!db.channels[row.channel]) {
|
||||
errors.push({
|
||||
line: i + 2,
|
||||
message: `"${row.channel}" is missing in the channels.csv`
|
||||
})
|
||||
}
|
||||
|
||||
return errors
|
||||
}
|
||||
|
||||
function validateCountryLanguages(row, i) {
|
||||
const errors = []
|
||||
for (let lang of row.languages) {
|
||||
if (!db.languages[lang]) {
|
||||
errors.push({
|
||||
line: i + 2,
|
||||
message: `"${row.code}" has the wrong language "${lang}"`
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return errors
|
||||
}
|
||||
|
||||
function validateSubdivisionCountry(row, i) {
|
||||
const errors = []
|
||||
if (!db.countries[row.country]) {
|
||||
errors.push({
|
||||
line: i + 2,
|
||||
message: `"${row.code}" has the wrong country "${row.country}"`
|
||||
})
|
||||
}
|
||||
|
||||
return errors
|
||||
}
|
||||
|
||||
function validateRegionCountries(row, i) {
|
||||
const errors = []
|
||||
row.countries.forEach(country => {
|
||||
if (!db.countries[country]) {
|
||||
errors.push({
|
||||
line: i + 2,
|
||||
message: `"${row.code}" has the wrong country "${country}"`
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
return errors
|
||||
}
|
||||
|
||||
function handleError(message) {
|
||||
logger.error(chalk.red(`\n${message}`))
|
||||
process.exit(1)
|
||||
}
|
||||
257
scripts/db/validate.ts
Normal file
257
scripts/db/validate.ts
Normal file
|
|
@ -0,0 +1,257 @@
|
|||
import { Collection, Storage, File, Dictionary, Logger } from '@freearhey/core'
|
||||
import { DATA_DIR } from '../constants'
|
||||
import { program } from 'commander'
|
||||
import Joi from 'joi'
|
||||
import { CSVParser, IDCreator } from '../core'
|
||||
import chalk from 'chalk'
|
||||
|
||||
program.argument('[filepath]', 'Path to file to validate').parse(process.argv)
|
||||
|
||||
const logger = new Logger()
|
||||
const buffer = new Dictionary()
|
||||
const files = new Dictionary()
|
||||
const schemes: { [key: string]: object } = require('../schemes')
|
||||
|
||||
async function main() {
|
||||
const dataStorage = new Storage(DATA_DIR)
|
||||
const _files = await dataStorage.list('*.csv')
|
||||
let globalErrors = new Collection()
|
||||
const parser = new CSVParser()
|
||||
|
||||
for (const filepath of _files) {
|
||||
const file = new File(filepath)
|
||||
if (file.extension() !== 'csv') continue
|
||||
|
||||
const csv = await dataStorage.load(file.basename())
|
||||
if (/\s+$/.test(csv))
|
||||
return handleError(`Error: empty lines at the end of file not allowed (${filepath})`)
|
||||
|
||||
const rows = csv.split(/\r\n/)
|
||||
const headers = rows[0].split(',')
|
||||
for (const [i, line] of rows.entries()) {
|
||||
if (line.indexOf('\n') > -1)
|
||||
return handleError(
|
||||
`Error: row ${i + 1} has the wrong line ending character, should be CRLF (${filepath})`
|
||||
)
|
||||
if (line.split(/,(?=(?:(?:[^"]*"){2})*[^"]*$)/).length !== headers.length)
|
||||
return handleError(`Error: row ${i + 1} has the wrong number of columns (${filepath})`)
|
||||
}
|
||||
|
||||
const data = await parser.parse(csv)
|
||||
const filename = file.name()
|
||||
|
||||
let grouped
|
||||
switch (filename) {
|
||||
case 'blocklist':
|
||||
grouped = data.keyBy(item => item.channel)
|
||||
break
|
||||
case 'categories':
|
||||
case 'channels':
|
||||
grouped = data.keyBy(item => item.id)
|
||||
break
|
||||
default:
|
||||
grouped = data.keyBy(item => item.code)
|
||||
break
|
||||
}
|
||||
|
||||
buffer.set(filename, grouped)
|
||||
files.set(filename, data)
|
||||
}
|
||||
|
||||
const filesToCheck = program.args.length ? program.args : _files
|
||||
for (const filepath of filesToCheck) {
|
||||
const file = new File(filepath)
|
||||
const filename = file.name()
|
||||
if (!schemes[filename]) return handleError(`Error: "${filename}" scheme is missing`)
|
||||
|
||||
const rows: Collection = files.get(filename)
|
||||
const rowsCopy = JSON.parse(JSON.stringify(rows.all()))
|
||||
|
||||
let fileErrors = new Collection()
|
||||
switch (filename) {
|
||||
case 'channels':
|
||||
fileErrors = fileErrors.concat(findDuplicatesBy(rowsCopy, 'id'))
|
||||
for (const [i, row] of rowsCopy.entries()) {
|
||||
fileErrors = fileErrors.concat(validateChannelId(row, i))
|
||||
fileErrors = fileErrors.concat(validateChannelBroadcastArea(row, i))
|
||||
fileErrors = fileErrors.concat(
|
||||
checkValue(i, row, 'id', 'subdivision', buffer.get('subdivisions'))
|
||||
)
|
||||
fileErrors = fileErrors.concat(
|
||||
checkValue(i, row, 'id', 'categories', buffer.get('categories'))
|
||||
)
|
||||
fileErrors = fileErrors.concat(
|
||||
checkValue(i, row, 'id', 'replaced_by', buffer.get('channels'))
|
||||
)
|
||||
fileErrors = fileErrors.concat(
|
||||
checkValue(i, row, 'id', 'languages', buffer.get('languages'))
|
||||
)
|
||||
fileErrors = fileErrors.concat(
|
||||
checkValue(i, row, 'id', 'country', buffer.get('countries'))
|
||||
)
|
||||
}
|
||||
break
|
||||
case 'blocklist':
|
||||
for (const [i, row] of rowsCopy.entries()) {
|
||||
fileErrors = fileErrors.concat(validateChannel(row.channel, i))
|
||||
}
|
||||
break
|
||||
case 'countries':
|
||||
fileErrors = fileErrors.concat(findDuplicatesBy(rowsCopy, 'code'))
|
||||
for (const [i, row] of rowsCopy.entries()) {
|
||||
fileErrors = fileErrors.concat(
|
||||
checkValue(i, row, 'code', 'languages', buffer.get('languages'))
|
||||
)
|
||||
}
|
||||
break
|
||||
case 'subdivisions':
|
||||
fileErrors = fileErrors.concat(findDuplicatesBy(rowsCopy, 'code'))
|
||||
for (const [i, row] of rowsCopy.entries()) {
|
||||
fileErrors = fileErrors.concat(
|
||||
checkValue(i, row, 'code', 'country', buffer.get('countries'))
|
||||
)
|
||||
}
|
||||
break
|
||||
case 'regions':
|
||||
fileErrors = fileErrors.concat(findDuplicatesBy(rowsCopy, 'code'))
|
||||
for (const [i, row] of rowsCopy.entries()) {
|
||||
fileErrors = fileErrors.concat(
|
||||
checkValue(i, row, 'code', 'countries', buffer.get('countries'))
|
||||
)
|
||||
}
|
||||
break
|
||||
case 'categories':
|
||||
fileErrors = fileErrors.concat(findDuplicatesBy(rowsCopy, 'id'))
|
||||
break
|
||||
case 'languages':
|
||||
fileErrors = fileErrors.concat(findDuplicatesBy(rowsCopy, 'code'))
|
||||
break
|
||||
}
|
||||
|
||||
const schema = Joi.object(schemes[filename])
|
||||
rows.forEach((row: string | string[] | boolean, i: number) => {
|
||||
const { error } = schema.validate(row, { abortEarly: false })
|
||||
if (error) {
|
||||
error.details.forEach(detail => {
|
||||
fileErrors.push({ line: i + 2, message: detail.message })
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
if (fileErrors.count()) {
|
||||
logger.info(`\n${chalk.underline(filepath)}`)
|
||||
fileErrors.forEach(err => {
|
||||
const position = err.line.toString().padEnd(6, ' ')
|
||||
logger.info(` ${chalk.gray(position)} ${err.message}`)
|
||||
})
|
||||
globalErrors = globalErrors.concat(fileErrors)
|
||||
}
|
||||
}
|
||||
|
||||
if (globalErrors.count()) return handleError(`${globalErrors.count()} error(s)`)
|
||||
}
|
||||
|
||||
main()
|
||||
|
||||
function checkValue(
|
||||
i: number,
|
||||
row: { [key: string]: string[] | string | boolean },
|
||||
key: string,
|
||||
field: string,
|
||||
collection: Collection
|
||||
) {
|
||||
const errors = new Collection()
|
||||
let values: string[] = []
|
||||
if (Array.isArray(row[field])) {
|
||||
values = row[field] as string[]
|
||||
} else if (typeof row[field] === 'string') {
|
||||
values = new Array(row[field]) as string[]
|
||||
}
|
||||
|
||||
values.forEach((value: string) => {
|
||||
if (collection.missing(value)) {
|
||||
errors.push({
|
||||
line: i + 2,
|
||||
message: `"${row[key]}" has an invalid ${field} "${value}"`
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
return errors
|
||||
}
|
||||
|
||||
function validateChannel(channelId: string, i: number) {
|
||||
const errors = new Collection()
|
||||
const channels = buffer.get('channels')
|
||||
|
||||
if (channels.missing(channelId)) {
|
||||
errors.push({
|
||||
line: i + 2,
|
||||
message: `"${channelId}" is missing in the channels.csv`
|
||||
})
|
||||
}
|
||||
|
||||
return errors
|
||||
}
|
||||
|
||||
function findDuplicatesBy(rows: { [key: string]: string }[], key: string) {
|
||||
const errors = new Collection()
|
||||
const buffer = new Dictionary()
|
||||
|
||||
rows.forEach((row, i) => {
|
||||
const normId = row[key].toLowerCase()
|
||||
if (buffer.has(normId)) {
|
||||
errors.push({
|
||||
line: i + 2,
|
||||
message: `entry with the ${key} "${row[key]}" already exists`
|
||||
})
|
||||
}
|
||||
|
||||
buffer.set(normId, true)
|
||||
})
|
||||
|
||||
return errors
|
||||
}
|
||||
|
||||
function validateChannelId(row: { [key: string]: string }, i: number) {
|
||||
const errors = new Collection()
|
||||
|
||||
const expectedId = new IDCreator().create(row.name, row.country)
|
||||
|
||||
if (expectedId !== row.id) {
|
||||
errors.push({
|
||||
line: i + 2,
|
||||
message: `"${row.id}" must be derived from the channel name "${row.name}" and the country code "${row.country}"`
|
||||
})
|
||||
}
|
||||
|
||||
return errors
|
||||
}
|
||||
|
||||
function validateChannelBroadcastArea(row: { [key: string]: string[] }, i: number) {
|
||||
const errors = new Collection()
|
||||
const regions = buffer.get('regions')
|
||||
const countries = buffer.get('countries')
|
||||
const subdivisions = buffer.get('subdivisions')
|
||||
|
||||
row.broadcast_area.forEach((areaCode: string) => {
|
||||
const [type, code] = areaCode.split('/')
|
||||
if (
|
||||
(type === 'r' && regions.missing(code)) ||
|
||||
(type === 'c' && countries.missing(code)) ||
|
||||
(type === 's' && subdivisions.missing(code))
|
||||
) {
|
||||
errors.push({
|
||||
line: i + 2,
|
||||
message: `"${row.id}" has the wrong broadcast_area "${areaCode}"`
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
return errors
|
||||
}
|
||||
|
||||
function handleError(message: string) {
|
||||
logger.error(chalk.red(message))
|
||||
process.exit(1)
|
||||
}
|
||||
14
scripts/models/blocked.ts
Normal file
14
scripts/models/blocked.ts
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
type BlockedProps = {
|
||||
channel: string
|
||||
ref: string
|
||||
}
|
||||
|
||||
export class Blocked {
|
||||
channel: string
|
||||
ref: string
|
||||
|
||||
constructor({ ref, channel }: BlockedProps) {
|
||||
this.channel = channel
|
||||
this.ref = ref
|
||||
}
|
||||
}
|
||||
85
scripts/models/channel.ts
Normal file
85
scripts/models/channel.ts
Normal file
|
|
@ -0,0 +1,85 @@
|
|||
type ChannelProps = {
|
||||
id: string
|
||||
name: string
|
||||
alt_names: string[]
|
||||
network: string
|
||||
owners: string[]
|
||||
country: string
|
||||
subdivision: string
|
||||
city: string
|
||||
broadcast_area: string[]
|
||||
languages: string[]
|
||||
categories: string[]
|
||||
is_nsfw: boolean
|
||||
launched: string
|
||||
closed: string
|
||||
replaced_by: string
|
||||
website: string
|
||||
logo: string
|
||||
}
|
||||
|
||||
export class Channel {
|
||||
id: string
|
||||
name: string
|
||||
alt_names: string[]
|
||||
network: string
|
||||
owners: string[]
|
||||
country: string
|
||||
subdivision: string
|
||||
city: string
|
||||
broadcast_area: string[]
|
||||
languages: string[]
|
||||
categories: string[]
|
||||
is_nsfw: boolean
|
||||
launched: string
|
||||
closed: string
|
||||
replaced_by: string
|
||||
website: string
|
||||
logo: string
|
||||
|
||||
constructor({
|
||||
id,
|
||||
name,
|
||||
alt_names,
|
||||
network,
|
||||
owners,
|
||||
country,
|
||||
subdivision,
|
||||
city,
|
||||
broadcast_area,
|
||||
languages,
|
||||
categories,
|
||||
is_nsfw,
|
||||
launched,
|
||||
closed,
|
||||
replaced_by,
|
||||
website,
|
||||
logo
|
||||
}: ChannelProps) {
|
||||
this.id = id
|
||||
this.name = name
|
||||
this.alt_names = alt_names
|
||||
this.network = network
|
||||
this.owners = owners
|
||||
this.country = country
|
||||
this.subdivision = subdivision
|
||||
this.city = city
|
||||
this.broadcast_area = broadcast_area
|
||||
this.languages = languages
|
||||
this.categories = categories
|
||||
this.is_nsfw = is_nsfw
|
||||
this.launched = launched
|
||||
this.closed = closed
|
||||
this.replaced_by = replaced_by
|
||||
this.website = website
|
||||
this.logo = logo
|
||||
}
|
||||
|
||||
update(data: { [key: string]: string }) {
|
||||
for (const key in data) {
|
||||
if (this[key] && data[key]) {
|
||||
this[key] = data[key]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
3
scripts/models/index.ts
Normal file
3
scripts/models/index.ts
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
export * from './channel'
|
||||
export * from './issue'
|
||||
export * from './blocked'
|
||||
19
scripts/models/issue.ts
Normal file
19
scripts/models/issue.ts
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
import { Dictionary } from '@freearhey/core'
|
||||
|
||||
type IssueProps = {
|
||||
number: number
|
||||
labels: string[]
|
||||
data: Dictionary
|
||||
}
|
||||
|
||||
export class Issue {
|
||||
number: number
|
||||
labels: string[]
|
||||
data: Dictionary
|
||||
|
||||
constructor({ number, labels, data }: IssueProps) {
|
||||
this.number = number
|
||||
this.labels = labels
|
||||
this.data = data
|
||||
}
|
||||
}
|
||||
1
tests/__data__/.gitignore
vendored
Normal file
1
tests/__data__/.gitignore
vendored
Normal file
|
|
@ -0,0 +1 @@
|
|||
output/
|
||||
1
tests/__data__/expected/api/blocklist.json
Normal file
1
tests/__data__/expected/api/blocklist.json
Normal file
|
|
@ -0,0 +1 @@
|
|||
[{"channel":"AnimalPlanetAfrica.za","ref":"https://github.com/iptv-org/iptv/issues/1831"}]
|
||||
1
tests/__data__/expected/api/channels.json
Normal file
1
tests/__data__/expected/api/channels.json
Normal file
|
|
@ -0,0 +1 @@
|
|||
[{"id":"002RadioTV.do","name":"002 Radio TV","alt_names":[],"network":null,"owners":[],"country":"DO","subdivision":null,"city":null,"broadcast_area":["c/DO"],"languages":["spa"],"categories":["general"],"is_nsfw":false,"launched":null,"closed":null,"replaced_by":null,"website":"https://www.002radio.com/","logo":"https://i.imgur.com/7oNe8xj.png"},{"id":"BeijingSatelliteTV.cn","name":"Beijing Satellite TV","alt_names":["北京卫视"],"network":null,"owners":[],"country":"CN","subdivision":null,"city":"Beijing","broadcast_area":["c/CN"],"languages":["zho"],"categories":["general"],"is_nsfw":false,"launched":"1979-05-16","closed":null,"replaced_by":null,"website":"https://www.brtn.cn/btv/","logo":"https://i.imgur.com/vsktAez.png"},{"id":"M5.hu","name":"M5","alt_names":[],"network":null,"owners":[],"country":"HU","subdivision":null,"city":null,"broadcast_area":["c/HU"],"languages":["hun"],"categories":[],"is_nsfw":false,"launched":null,"closed":null,"replaced_by":null,"website":"https://www.mediaklikk.hu/m5/","logo":"https://i.imgur.com/y21wFd0.png"}]
|
||||
2
tests/__data__/expected/data/blocklist.csv
Normal file
2
tests/__data__/expected/data/blocklist.csv
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
channel,ref
|
||||
HGTVHungary.hu,https://github.com/iptv-org/iptv/issues/1831
|
||||
|
6
tests/__data__/expected/data/channels.csv
Normal file
6
tests/__data__/expected/data/channels.csv
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
id,name,alt_names,network,owners,country,subdivision,city,broadcast_area,languages,categories,is_nsfw,launched,closed,replaced_by,website,logo
|
||||
beINMoviesTurk.tr,beIN Movies Turk,beIN Movies Türk,,,TR,,Beijing,c/TR,tur,movies,FALSE,1979-05-16,,,http://www.digiturk.com.tr/,https://i.imgur.com/nw8Sa2z.png
|
||||
M5.hu,M5,,,Duna Médiaszolgáltató Nonprofit Zrt.,HU,,,c/HU,hun,,FALSE,,,,https://www.mediaklikk.hu/m5/,https://i.imgur.com/y21wFd0.png
|
||||
WenzhouEconomicandEducation.cn,Wenzhou Economic and Education,,,,CN,,Wenzhou,c/CN,zho,science,FALSE,,,,,https://www.tvchinese.net/uploads/tv/wzjjkj.jpg
|
||||
YiwuBusinessChannel.cn,Yiwu Business Channel,,,,CN,,,c/CN,zho,business,FALSE,,,,,https://www.tvchinese.net/uploads/tv/yiwutv.jpg
|
||||
YiwuNewsIntegratedChannel.cn,Yiwu News Integrated Channel,,,,CN,,,c/CN,zho,news,FALSE,,,,,https://www.tvchinese.net/uploads/tv/yiwutv.jpg
|
||||
|
2
tests/__data__/input/data/blocklist.csv
Normal file
2
tests/__data__/input/data/blocklist.csv
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
channel,ref
|
||||
AnimalPlanetAfrica.za,https://github.com/iptv-org/iptv/issues/1831
|
||||
|
4
tests/__data__/input/data/channels.csv
Normal file
4
tests/__data__/input/data/channels.csv
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
id,name,alt_names,network,owners,country,subdivision,city,broadcast_area,languages,categories,is_nsfw,launched,closed,replaced_by,website,logo
|
||||
002RadioTV.do,002 Radio TV,,,,DO,,,c/DO,spa,general,FALSE,,,,https://www.002radio.com/,https://i.imgur.com/7oNe8xj.png
|
||||
BeijingSatelliteTV.cn,Beijing Satellite TV,北京卫视,,,CN,,Beijing,c/CN,zho,general,FALSE,1979-05-16,,,https://www.brtn.cn/btv/,https://i.imgur.com/vsktAez.png
|
||||
M5.hu,M5,,,,HU,,,c/HU,hun,,FALSE,,,,https://www.mediaklikk.hu/m5/,https://i.imgur.com/y21wFd0.png
|
||||
|
81
tests/__data__/input/issues/blocklist_add_approved.js
Normal file
81
tests/__data__/input/issues/blocklist_add_approved.js
Normal file
|
|
@ -0,0 +1,81 @@
|
|||
module.exports = [
|
||||
{
|
||||
url: 'https://api.github.com/repos/iptv-org/database/issues/5897',
|
||||
repository_url: 'https://api.github.com/repos/iptv-org/database',
|
||||
labels_url: 'https://api.github.com/repos/iptv-org/database/issues/5897/labels{/name}',
|
||||
comments_url: 'https://api.github.com/repos/iptv-org/database/issues/5897/comments',
|
||||
events_url: 'https://api.github.com/repos/iptv-org/database/issues/5897/events',
|
||||
html_url: 'https://github.com/iptv-org/database/issues/5897',
|
||||
id: 1929261634,
|
||||
node_id: 'I_kwDOG1Kwp85y_jJC',
|
||||
number: 5897,
|
||||
title: 'Block: HGTV Hungary',
|
||||
user: {
|
||||
login: 'freearhey',
|
||||
id: 7253922,
|
||||
node_id: 'MDQ6VXNlcjcyNTM5MjI=',
|
||||
avatar_url: 'https://avatars.githubusercontent.com/u/7253922?v=4',
|
||||
gravatar_id: '',
|
||||
url: 'https://api.github.com/users/freearhey',
|
||||
html_url: 'https://github.com/freearhey',
|
||||
followers_url: 'https://api.github.com/users/freearhey/followers',
|
||||
following_url: 'https://api.github.com/users/freearhey/following{/other_user}',
|
||||
gists_url: 'https://api.github.com/users/freearhey/gists{/gist_id}',
|
||||
starred_url: 'https://api.github.com/users/freearhey/starred{/owner}{/repo}',
|
||||
subscriptions_url: 'https://api.github.com/users/freearhey/subscriptions',
|
||||
organizations_url: 'https://api.github.com/users/freearhey/orgs',
|
||||
repos_url: 'https://api.github.com/users/freearhey/repos',
|
||||
events_url: 'https://api.github.com/users/freearhey/events{/privacy}',
|
||||
received_events_url: 'https://api.github.com/users/freearhey/received_events',
|
||||
type: 'User',
|
||||
site_admin: false
|
||||
},
|
||||
labels: [
|
||||
{
|
||||
id: 5366738347,
|
||||
node_id: 'LA_kwDOG1Kwp88AAAABP-Htqw',
|
||||
url: 'https://api.github.com/repos/iptv-org/database/labels/approved',
|
||||
name: 'approved',
|
||||
color: '85DDDE',
|
||||
default: false,
|
||||
description: ''
|
||||
},
|
||||
{
|
||||
id: 6049155772,
|
||||
node_id: 'LA_kwDOG1Kwp88AAAABaI7KvA',
|
||||
url: 'https://api.github.com/repos/iptv-org/database/labels/blocklist:add',
|
||||
name: 'blocklist:add',
|
||||
color: 'e99695',
|
||||
default: false,
|
||||
description: 'Request to add a channel to the blocklist'
|
||||
}
|
||||
],
|
||||
state: 'open',
|
||||
locked: false,
|
||||
assignee: null,
|
||||
assignees: [],
|
||||
milestone: null,
|
||||
comments: 0,
|
||||
created_at: '2023-10-06T00:35:32Z',
|
||||
updated_at: '2023-10-06T00:35:32Z',
|
||||
closed_at: null,
|
||||
author_association: 'CONTRIBUTOR',
|
||||
active_lock_reason: null,
|
||||
body: '### Channel ID\n\nHGTVHungary.hu\n\n### Reference\n\nhttps://github.com/iptv-org/iptv/issues/1831\n\n### Notes (optional)\n\n_No response_',
|
||||
reactions: {
|
||||
url: 'https://api.github.com/repos/iptv-org/database/issues/5897/reactions',
|
||||
total_count: 0,
|
||||
'+1': 0,
|
||||
'-1': 0,
|
||||
laugh: 0,
|
||||
hooray: 0,
|
||||
confused: 0,
|
||||
heart: 0,
|
||||
rocket: 0,
|
||||
eyes: 0
|
||||
},
|
||||
timeline_url: 'https://api.github.com/repos/iptv-org/database/issues/5897/timeline',
|
||||
performed_via_github_app: null,
|
||||
state_reason: null
|
||||
}
|
||||
]
|
||||
81
tests/__data__/input/issues/blocklist_remove_approved.js
Normal file
81
tests/__data__/input/issues/blocklist_remove_approved.js
Normal file
|
|
@ -0,0 +1,81 @@
|
|||
module.exports = [
|
||||
{
|
||||
url: 'https://api.github.com/repos/iptv-org/database/issues/5891',
|
||||
repository_url: 'https://api.github.com/repos/iptv-org/database',
|
||||
labels_url: 'https://api.github.com/repos/iptv-org/database/issues/5891/labels{/name}',
|
||||
comments_url: 'https://api.github.com/repos/iptv-org/database/issues/5891/comments',
|
||||
events_url: 'https://api.github.com/repos/iptv-org/database/issues/5891/events',
|
||||
html_url: 'https://github.com/iptv-org/database/issues/5891',
|
||||
id: 1929261634,
|
||||
node_id: 'I_kwDOG1Kwp85y_jJC',
|
||||
number: 5891,
|
||||
title: 'Unblock: Animal Planet Africa',
|
||||
user: {
|
||||
login: 'freearhey',
|
||||
id: 7253922,
|
||||
node_id: 'MDQ6VXNlcjcyNTM5MjI=',
|
||||
avatar_url: 'https://avatars.githubusercontent.com/u/7253922?v=4',
|
||||
gravatar_id: '',
|
||||
url: 'https://api.github.com/users/freearhey',
|
||||
html_url: 'https://github.com/freearhey',
|
||||
followers_url: 'https://api.github.com/users/freearhey/followers',
|
||||
following_url: 'https://api.github.com/users/freearhey/following{/other_user}',
|
||||
gists_url: 'https://api.github.com/users/freearhey/gists{/gist_id}',
|
||||
starred_url: 'https://api.github.com/users/freearhey/starred{/owner}{/repo}',
|
||||
subscriptions_url: 'https://api.github.com/users/freearhey/subscriptions',
|
||||
organizations_url: 'https://api.github.com/users/freearhey/orgs',
|
||||
repos_url: 'https://api.github.com/users/freearhey/repos',
|
||||
events_url: 'https://api.github.com/users/freearhey/events{/privacy}',
|
||||
received_events_url: 'https://api.github.com/users/freearhey/received_events',
|
||||
type: 'User',
|
||||
site_admin: false
|
||||
},
|
||||
labels: [
|
||||
{
|
||||
id: 5366738347,
|
||||
node_id: 'LA_kwDOG1Kwp88AAAABP-Htqw',
|
||||
url: 'https://api.github.com/repos/iptv-org/database/labels/approved',
|
||||
name: 'approved',
|
||||
color: '85DDDE',
|
||||
default: false,
|
||||
description: ''
|
||||
},
|
||||
{
|
||||
id: 6049155772,
|
||||
node_id: 'LA_kwDOG1Kwp88AAAABaI7KvA',
|
||||
url: 'https://api.github.com/repos/iptv-org/database/labels/blocklist:add',
|
||||
name: 'blocklist:remove',
|
||||
color: 'e99695',
|
||||
default: false,
|
||||
description: 'Request to remove a channel from the blocklist'
|
||||
}
|
||||
],
|
||||
state: 'open',
|
||||
locked: false,
|
||||
assignee: null,
|
||||
assignees: [],
|
||||
milestone: null,
|
||||
comments: 0,
|
||||
created_at: '2023-10-06T00:35:32Z',
|
||||
updated_at: '2023-10-06T00:35:32Z',
|
||||
closed_at: null,
|
||||
author_association: 'CONTRIBUTOR',
|
||||
active_lock_reason: null,
|
||||
body: '### Channel ID\n\nAnimalPlanetAfrica.za\n\n### Reason\n\nOther\n\n### Notes (optional)\n\n_No response_',
|
||||
reactions: {
|
||||
url: 'https://api.github.com/repos/iptv-org/database/issues/5891/reactions',
|
||||
total_count: 0,
|
||||
'+1': 0,
|
||||
'-1': 0,
|
||||
laugh: 0,
|
||||
hooray: 0,
|
||||
confused: 0,
|
||||
heart: 0,
|
||||
rocket: 0,
|
||||
eyes: 0
|
||||
},
|
||||
timeline_url: 'https://api.github.com/repos/iptv-org/database/issues/5891/timeline',
|
||||
performed_via_github_app: null,
|
||||
state_reason: null
|
||||
}
|
||||
]
|
||||
239
tests/__data__/input/issues/channels_add_approved.js
Normal file
239
tests/__data__/input/issues/channels_add_approved.js
Normal file
|
|
@ -0,0 +1,239 @@
|
|||
module.exports = [
|
||||
{
|
||||
url: 'https://api.github.com/repos/iptv-org/database/issues/5900',
|
||||
repository_url: 'https://api.github.com/repos/iptv-org/database',
|
||||
labels_url: 'https://api.github.com/repos/iptv-org/database/issues/5900/labels{/name}',
|
||||
comments_url: 'https://api.github.com/repos/iptv-org/database/issues/5900/comments',
|
||||
events_url: 'https://api.github.com/repos/iptv-org/database/issues/5900/events',
|
||||
html_url: 'https://github.com/iptv-org/database/issues/5900',
|
||||
id: 1929321995,
|
||||
node_id: 'I_kwDOG1Kwp85y_x4L',
|
||||
number: 5900,
|
||||
title: 'Add: Yiwu News Integrated Channel',
|
||||
user: {
|
||||
login: 'AntiPontifex',
|
||||
id: 81566772,
|
||||
node_id: 'MDQ6VXNlcjgxNTY2Nzcy',
|
||||
avatar_url: 'https://avatars.githubusercontent.com/u/81566772?v=4',
|
||||
gravatar_id: '',
|
||||
url: 'https://api.github.com/users/AntiPontifex',
|
||||
html_url: 'https://github.com/AntiPontifex',
|
||||
followers_url: 'https://api.github.com/users/AntiPontifex/followers',
|
||||
following_url: 'https://api.github.com/users/AntiPontifex/following{/other_user}',
|
||||
gists_url: 'https://api.github.com/users/AntiPontifex/gists{/gist_id}',
|
||||
starred_url: 'https://api.github.com/users/AntiPontifex/starred{/owner}{/repo}',
|
||||
subscriptions_url: 'https://api.github.com/users/AntiPontifex/subscriptions',
|
||||
organizations_url: 'https://api.github.com/users/AntiPontifex/orgs',
|
||||
repos_url: 'https://api.github.com/users/AntiPontifex/repos',
|
||||
events_url: 'https://api.github.com/users/AntiPontifex/events{/privacy}',
|
||||
received_events_url: 'https://api.github.com/users/AntiPontifex/received_events',
|
||||
type: 'User',
|
||||
site_admin: false
|
||||
},
|
||||
labels: [
|
||||
{
|
||||
id: 5303575699,
|
||||
node_id: 'LA_kwDOG1Kwp88AAAABPB4kkw',
|
||||
url: 'https://api.github.com/repos/iptv-org/database/labels/channels:add',
|
||||
name: 'channels:add',
|
||||
color: '017ff8',
|
||||
default: false,
|
||||
description: 'Request to add a channel into the database'
|
||||
},
|
||||
{
|
||||
id: 5366738347,
|
||||
node_id: 'LA_kwDOG1Kwp88AAAABP-Htqw',
|
||||
url: 'https://api.github.com/repos/iptv-org/database/labels/approved',
|
||||
name: 'approved',
|
||||
color: '85DDDE',
|
||||
default: false,
|
||||
description: ''
|
||||
}
|
||||
],
|
||||
state: 'open',
|
||||
locked: false,
|
||||
assignee: null,
|
||||
assignees: [],
|
||||
milestone: null,
|
||||
comments: 0,
|
||||
created_at: '2023-10-06T02:10:41Z',
|
||||
updated_at: '2023-10-06T02:52:02Z',
|
||||
closed_at: null,
|
||||
author_association: 'CONTRIBUTOR',
|
||||
active_lock_reason: null,
|
||||
body: '### Channel Name\n\nYiwu News Integrated Channel\n\n### Alternative Names (optional)\n\n_No response_\n\n### Network (optional)\n\n_No response_\n\n### Owners (optional)\n\n_No response_\n\n### Country\n\nCN\n\n### Subdivision (optional)\n\n_No response_\n\n### City (optional)\n\n_No response_\n\n### Broadcast Area\n\nc/CN\n\n### Languages\n\nzho\n\n### Categories (optional)\n\nnews\n\n### NSFW\n\nFALSE\n\n### Launched (optional)\n\n_No response_\n\n### Closed (optional)\n\n_No response_\n\n### Replaced By (optional)\n\n_No response_\n\n### Website (optional)\n\n_No response_\n\n### Logo\n\nhttps://www.tvchinese.net/uploads/tv/yiwutv.jpg\n\n### Notes\n\n_No response_',
|
||||
reactions: {
|
||||
url: 'https://api.github.com/repos/iptv-org/database/issues/5900/reactions',
|
||||
total_count: 0,
|
||||
'+1': 0,
|
||||
'-1': 0,
|
||||
laugh: 0,
|
||||
hooray: 0,
|
||||
confused: 0,
|
||||
heart: 0,
|
||||
rocket: 0,
|
||||
eyes: 0
|
||||
},
|
||||
timeline_url: 'https://api.github.com/repos/iptv-org/database/issues/5900/timeline',
|
||||
performed_via_github_app: null,
|
||||
state_reason: null
|
||||
},
|
||||
{
|
||||
url: 'https://api.github.com/repos/iptv-org/database/issues/5899',
|
||||
repository_url: 'https://api.github.com/repos/iptv-org/database',
|
||||
labels_url: 'https://api.github.com/repos/iptv-org/database/issues/5899/labels{/name}',
|
||||
comments_url: 'https://api.github.com/repos/iptv-org/database/issues/5899/comments',
|
||||
events_url: 'https://api.github.com/repos/iptv-org/database/issues/5899/events',
|
||||
html_url: 'https://github.com/iptv-org/database/issues/5899',
|
||||
id: 1929318573,
|
||||
node_id: 'I_kwDOG1Kwp85y_xCt',
|
||||
number: 5899,
|
||||
title: 'Add: Yiwu Business Channel',
|
||||
user: {
|
||||
login: 'AntiPontifex',
|
||||
id: 81566772,
|
||||
node_id: 'MDQ6VXNlcjgxNTY2Nzcy',
|
||||
avatar_url: 'https://avatars.githubusercontent.com/u/81566772?v=4',
|
||||
gravatar_id: '',
|
||||
url: 'https://api.github.com/users/AntiPontifex',
|
||||
html_url: 'https://github.com/AntiPontifex',
|
||||
followers_url: 'https://api.github.com/users/AntiPontifex/followers',
|
||||
following_url: 'https://api.github.com/users/AntiPontifex/following{/other_user}',
|
||||
gists_url: 'https://api.github.com/users/AntiPontifex/gists{/gist_id}',
|
||||
starred_url: 'https://api.github.com/users/AntiPontifex/starred{/owner}{/repo}',
|
||||
subscriptions_url: 'https://api.github.com/users/AntiPontifex/subscriptions',
|
||||
organizations_url: 'https://api.github.com/users/AntiPontifex/orgs',
|
||||
repos_url: 'https://api.github.com/users/AntiPontifex/repos',
|
||||
events_url: 'https://api.github.com/users/AntiPontifex/events{/privacy}',
|
||||
received_events_url: 'https://api.github.com/users/AntiPontifex/received_events',
|
||||
type: 'User',
|
||||
site_admin: false
|
||||
},
|
||||
labels: [
|
||||
{
|
||||
id: 5303575699,
|
||||
node_id: 'LA_kwDOG1Kwp88AAAABPB4kkw',
|
||||
url: 'https://api.github.com/repos/iptv-org/database/labels/channels:add',
|
||||
name: 'channels:add',
|
||||
color: '017ff8',
|
||||
default: false,
|
||||
description: 'Request to add a channel into the database'
|
||||
},
|
||||
{
|
||||
id: 5366738347,
|
||||
node_id: 'LA_kwDOG1Kwp88AAAABP-Htqw',
|
||||
url: 'https://api.github.com/repos/iptv-org/database/labels/approved',
|
||||
name: 'approved',
|
||||
color: '85DDDE',
|
||||
default: false,
|
||||
description: ''
|
||||
}
|
||||
],
|
||||
state: 'open',
|
||||
locked: false,
|
||||
assignee: null,
|
||||
assignees: [],
|
||||
milestone: null,
|
||||
comments: 0,
|
||||
created_at: '2023-10-06T02:05:11Z',
|
||||
updated_at: '2023-10-06T02:51:46Z',
|
||||
closed_at: null,
|
||||
author_association: 'CONTRIBUTOR',
|
||||
active_lock_reason: null,
|
||||
body: '### Channel Name\n\nYiwu Business Channel\n\n### Alternative Names (optional)\n\n_No response_\n\n### Network (optional)\n\n_No response_\n\n### Owners (optional)\n\n_No response_\n\n### Country\n\nCN\n\n### Subdivision (optional)\n\n_No response_\n\n### City (optional)\n\n_No response_\n\n### Broadcast Area\n\nc/CN\n\n### Languages\n\nzho\n\n### Categories (optional)\n\nbusiness\n\n### NSFW\n\nFALSE\n\n### Launched (optional)\n\n_No response_\n\n### Closed (optional)\n\n_No response_\n\n### Replaced By (optional)\n\n_No response_\n\n### Website (optional)\n\n_No response_\n\n### Logo\n\nhttps://www.tvchinese.net/uploads/tv/yiwutv.jpg\n\n### Notes\n\n_No response_',
|
||||
reactions: {
|
||||
url: 'https://api.github.com/repos/iptv-org/database/issues/5899/reactions',
|
||||
total_count: 0,
|
||||
'+1': 0,
|
||||
'-1': 0,
|
||||
laugh: 0,
|
||||
hooray: 0,
|
||||
confused: 0,
|
||||
heart: 0,
|
||||
rocket: 0,
|
||||
eyes: 0
|
||||
},
|
||||
timeline_url: 'https://api.github.com/repos/iptv-org/database/issues/5899/timeline',
|
||||
performed_via_github_app: null,
|
||||
state_reason: null
|
||||
},
|
||||
{
|
||||
url: 'https://api.github.com/repos/iptv-org/database/issues/5898',
|
||||
repository_url: 'https://api.github.com/repos/iptv-org/database',
|
||||
labels_url: 'https://api.github.com/repos/iptv-org/database/issues/5898/labels{/name}',
|
||||
comments_url: 'https://api.github.com/repos/iptv-org/database/issues/5898/comments',
|
||||
events_url: 'https://api.github.com/repos/iptv-org/database/issues/5898/events',
|
||||
html_url: 'https://github.com/iptv-org/database/issues/5898',
|
||||
id: 1929313117,
|
||||
node_id: 'I_kwDOG1Kwp85y_vtd',
|
||||
number: 5898,
|
||||
title: 'Add: Wenzhou Economic and Education',
|
||||
user: {
|
||||
login: 'AntiPontifex',
|
||||
id: 81566772,
|
||||
node_id: 'MDQ6VXNlcjgxNTY2Nzcy',
|
||||
avatar_url: 'https://avatars.githubusercontent.com/u/81566772?v=4',
|
||||
gravatar_id: '',
|
||||
url: 'https://api.github.com/users/AntiPontifex',
|
||||
html_url: 'https://github.com/AntiPontifex',
|
||||
followers_url: 'https://api.github.com/users/AntiPontifex/followers',
|
||||
following_url: 'https://api.github.com/users/AntiPontifex/following{/other_user}',
|
||||
gists_url: 'https://api.github.com/users/AntiPontifex/gists{/gist_id}',
|
||||
starred_url: 'https://api.github.com/users/AntiPontifex/starred{/owner}{/repo}',
|
||||
subscriptions_url: 'https://api.github.com/users/AntiPontifex/subscriptions',
|
||||
organizations_url: 'https://api.github.com/users/AntiPontifex/orgs',
|
||||
repos_url: 'https://api.github.com/users/AntiPontifex/repos',
|
||||
events_url: 'https://api.github.com/users/AntiPontifex/events{/privacy}',
|
||||
received_events_url: 'https://api.github.com/users/AntiPontifex/received_events',
|
||||
type: 'User',
|
||||
site_admin: false
|
||||
},
|
||||
labels: [
|
||||
{
|
||||
id: 5303575699,
|
||||
node_id: 'LA_kwDOG1Kwp88AAAABPB4kkw',
|
||||
url: 'https://api.github.com/repos/iptv-org/database/labels/channels:add',
|
||||
name: 'channels:add',
|
||||
color: '017ff8',
|
||||
default: false,
|
||||
description: 'Request to add a channel into the database'
|
||||
},
|
||||
{
|
||||
id: 5366738347,
|
||||
node_id: 'LA_kwDOG1Kwp88AAAABP-Htqw',
|
||||
url: 'https://api.github.com/repos/iptv-org/database/labels/approved',
|
||||
name: 'approved',
|
||||
color: '85DDDE',
|
||||
default: false,
|
||||
description: ''
|
||||
}
|
||||
],
|
||||
state: 'open',
|
||||
locked: false,
|
||||
assignee: null,
|
||||
assignees: [],
|
||||
milestone: null,
|
||||
comments: 0,
|
||||
created_at: '2023-10-06T01:56:32Z',
|
||||
updated_at: '2023-10-06T02:51:22Z',
|
||||
closed_at: null,
|
||||
author_association: 'CONTRIBUTOR',
|
||||
active_lock_reason: null,
|
||||
body: '### Channel Name\n\nWenzhou Economic and Education\n\n### Alternative Names (optional)\n\n_No response_\n\n### Network (optional)\n\n_No response_\n\n### Owners (optional)\n\n_No response_\n\n### Country\n\nCN\n\n### Subdivision (optional)\n\n_No response_\n\n### City (optional)\n\nWenzhou\n\n### Broadcast Area\n\nc/CN\n\n### Languages\n\nzho\n\n### Categories (optional)\n\nscience\n\n### NSFW\n\nFALSE\n\n### Launched (optional)\n\n_No response_\n\n### Closed (optional)\n\n_No response_\n\n### Replaced By (optional)\n\n_No response_\n\n### Website (optional)\n\n_No response_\n\n### Logo\n\nhttps://www.tvchinese.net/uploads/tv/wzjjkj.jpg\n\n### Notes\n\n_No response_',
|
||||
reactions: {
|
||||
url: 'https://api.github.com/repos/iptv-org/database/issues/5898/reactions',
|
||||
total_count: 0,
|
||||
'+1': 0,
|
||||
'-1': 0,
|
||||
laugh: 0,
|
||||
hooray: 0,
|
||||
confused: 0,
|
||||
heart: 0,
|
||||
rocket: 0,
|
||||
eyes: 0
|
||||
},
|
||||
timeline_url: 'https://api.github.com/repos/iptv-org/database/issues/5898/timeline',
|
||||
performed_via_github_app: null,
|
||||
state_reason: null
|
||||
}
|
||||
]
|
||||
160
tests/__data__/input/issues/channels_edit_approved.js
Normal file
160
tests/__data__/input/issues/channels_edit_approved.js
Normal file
|
|
@ -0,0 +1,160 @@
|
|||
module.exports = [
|
||||
{
|
||||
url: 'https://api.github.com/repos/iptv-org/database/issues/5901',
|
||||
repository_url: 'https://api.github.com/repos/iptv-org/database',
|
||||
labels_url: 'https://api.github.com/repos/iptv-org/database/issues/5901/labels{/name}',
|
||||
comments_url: 'https://api.github.com/repos/iptv-org/database/issues/5901/comments',
|
||||
events_url: 'https://api.github.com/repos/iptv-org/database/issues/5901/events',
|
||||
html_url: 'https://github.com/iptv-org/database/issues/5901',
|
||||
id: 1929459171,
|
||||
node_id: 'I_kwDOG1Kwp85zATXj',
|
||||
number: 5901,
|
||||
title: 'Edit: M5',
|
||||
user: {
|
||||
login: 'freearhey',
|
||||
id: 7253922,
|
||||
node_id: 'MDQ6VXNlcjcyNTM5MjI=',
|
||||
avatar_url: 'https://avatars.githubusercontent.com/u/7253922?v=4',
|
||||
gravatar_id: '',
|
||||
url: 'https://api.github.com/users/freearhey',
|
||||
html_url: 'https://github.com/freearhey',
|
||||
followers_url: 'https://api.github.com/users/freearhey/followers',
|
||||
following_url: 'https://api.github.com/users/freearhey/following{/other_user}',
|
||||
gists_url: 'https://api.github.com/users/freearhey/gists{/gist_id}',
|
||||
starred_url: 'https://api.github.com/users/freearhey/starred{/owner}{/repo}',
|
||||
subscriptions_url: 'https://api.github.com/users/freearhey/subscriptions',
|
||||
organizations_url: 'https://api.github.com/users/freearhey/orgs',
|
||||
repos_url: 'https://api.github.com/users/freearhey/repos',
|
||||
events_url: 'https://api.github.com/users/freearhey/events{/privacy}',
|
||||
received_events_url: 'https://api.github.com/users/freearhey/received_events',
|
||||
type: 'User',
|
||||
site_admin: false
|
||||
},
|
||||
labels: [
|
||||
{
|
||||
id: 5303574335,
|
||||
node_id: 'LA_kwDOG1Kwp88AAAABPB4fPw',
|
||||
url: 'https://api.github.com/repos/iptv-org/database/labels/channels:edit',
|
||||
name: 'channels:edit',
|
||||
color: 'E12977',
|
||||
default: false,
|
||||
description: 'Request to edit channel description'
|
||||
},
|
||||
{
|
||||
id: 5366738347,
|
||||
node_id: 'LA_kwDOG1Kwp88AAAABP-Htqw',
|
||||
url: 'https://api.github.com/repos/iptv-org/database/labels/approved',
|
||||
name: 'approved',
|
||||
color: '85DDDE',
|
||||
default: false,
|
||||
description: ''
|
||||
}
|
||||
],
|
||||
state: 'open',
|
||||
locked: false,
|
||||
assignee: null,
|
||||
assignees: [],
|
||||
milestone: null,
|
||||
comments: 0,
|
||||
created_at: '2023-10-06T05:25:44Z',
|
||||
updated_at: '2023-10-06T05:25:44Z',
|
||||
closed_at: null,
|
||||
author_association: 'CONTRIBUTOR',
|
||||
active_lock_reason: null,
|
||||
body: '### Channel ID (required)\n\nM5.hu\n\n### Channel Name\n\n_No response_\n\n### Alternative Names\n\n_No response_\n\n### Network\n\n_No response_\n\n### Owners\n\nDuna Médiaszolgáltató Nonprofit Zrt.\n\n### Country\n\n_No response_\n\n### Subdivision\n\n_No response_\n\n### City\n\n_No response_\n\n### Broadcast Area\n\n_No response_\n\n### Languages\n\n_No response_\n\n### Categories\n\n_No response_\n\n### NSFW\n\nFALSE\n\n### Launched\n\n_No response_\n\n### Closed\n\n_No response_\n\n### Replaced By\n\n_No response_\n\n### Website\n\n_No response_\n\n### Logo\n\n_No response_\n\n### Notes\n\n_No response_',
|
||||
reactions: {
|
||||
url: 'https://api.github.com/repos/iptv-org/database/issues/5901/reactions',
|
||||
total_count: 0,
|
||||
'+1': 0,
|
||||
'-1': 0,
|
||||
laugh: 0,
|
||||
hooray: 0,
|
||||
confused: 0,
|
||||
heart: 0,
|
||||
rocket: 0,
|
||||
eyes: 0
|
||||
},
|
||||
timeline_url: 'https://api.github.com/repos/iptv-org/database/issues/5901/timeline',
|
||||
performed_via_github_app: null,
|
||||
state_reason: null
|
||||
},
|
||||
{
|
||||
url: 'https://api.github.com/repos/iptv-org/database/issues/5701',
|
||||
repository_url: 'https://api.github.com/repos/iptv-org/database',
|
||||
labels_url: 'https://api.github.com/repos/iptv-org/database/issues/5701/labels{/name}',
|
||||
comments_url: 'https://api.github.com/repos/iptv-org/database/issues/5701/comments',
|
||||
events_url: 'https://api.github.com/repos/iptv-org/database/issues/5701/events',
|
||||
html_url: 'https://github.com/iptv-org/database/issues/5701',
|
||||
id: 1929459171,
|
||||
node_id: 'I_kwDOG1Kwp85zATXj',
|
||||
number: 5701,
|
||||
title: 'Edit: M5',
|
||||
user: {
|
||||
login: 'freearhey',
|
||||
id: 7253922,
|
||||
node_id: 'MDQ6VXNlcjcyNTM5MjI=',
|
||||
avatar_url: 'https://avatars.githubusercontent.com/u/7253922?v=4',
|
||||
gravatar_id: '',
|
||||
url: 'https://api.github.com/users/freearhey',
|
||||
html_url: 'https://github.com/freearhey',
|
||||
followers_url: 'https://api.github.com/users/freearhey/followers',
|
||||
following_url: 'https://api.github.com/users/freearhey/following{/other_user}',
|
||||
gists_url: 'https://api.github.com/users/freearhey/gists{/gist_id}',
|
||||
starred_url: 'https://api.github.com/users/freearhey/starred{/owner}{/repo}',
|
||||
subscriptions_url: 'https://api.github.com/users/freearhey/subscriptions',
|
||||
organizations_url: 'https://api.github.com/users/freearhey/orgs',
|
||||
repos_url: 'https://api.github.com/users/freearhey/repos',
|
||||
events_url: 'https://api.github.com/users/freearhey/events{/privacy}',
|
||||
received_events_url: 'https://api.github.com/users/freearhey/received_events',
|
||||
type: 'User',
|
||||
site_admin: false
|
||||
},
|
||||
labels: [
|
||||
{
|
||||
id: 5303574335,
|
||||
node_id: 'LA_kwDOG1Kwp88AAAABPB4fPw',
|
||||
url: 'https://api.github.com/repos/iptv-org/database/labels/channels:edit',
|
||||
name: 'channels:edit',
|
||||
color: 'E12977',
|
||||
default: false,
|
||||
description: 'Request to edit channel description'
|
||||
},
|
||||
{
|
||||
id: 5366738347,
|
||||
node_id: 'LA_kwDOG1Kwp88AAAABP-Htqw',
|
||||
url: 'https://api.github.com/repos/iptv-org/database/labels/approved',
|
||||
name: 'approved',
|
||||
color: '85DDDE',
|
||||
default: false,
|
||||
description: ''
|
||||
}
|
||||
],
|
||||
state: 'open',
|
||||
locked: false,
|
||||
assignee: null,
|
||||
assignees: [],
|
||||
milestone: null,
|
||||
comments: 0,
|
||||
created_at: '2023-10-06T05:25:44Z',
|
||||
updated_at: '2023-10-06T05:25:44Z',
|
||||
closed_at: null,
|
||||
author_association: 'CONTRIBUTOR',
|
||||
active_lock_reason: null,
|
||||
body: '### Channel ID (required)\n\nBeijingSatelliteTV.cn\n\n### Channel Name\n\nbeIN Movies Turk\n\n### Alternative Names\n\nbeIN Movies Türk\n\n### Network\n\n_No response_\n\n### Owners\n\n_No response_\n\n### Country\n\nTR\n\n### Subdivision\n\n_No response_\n\n### City\n\n_No response_\n\n### Broadcast Area\n\nc/TR\n\n### Languages\n\ntur\n\n### Categories\n\nmovies\n\n### NSFW\n\nFALSE\n\n### Launched\n\n1979-05-16\n\n### Closed\n\n_No response_\n\n### Replaced By\n\n_No response_\n\n### Website\n\nhttp://www.digiturk.com.tr/\n\n### Logo\n\nhttps://i.imgur.com/nw8Sa2z.png\n\n### Notes\n\n_No response_',
|
||||
reactions: {
|
||||
url: 'https://api.github.com/repos/iptv-org/database/issues/5701/reactions',
|
||||
total_count: 0,
|
||||
'+1': 0,
|
||||
'-1': 0,
|
||||
laugh: 0,
|
||||
hooray: 0,
|
||||
confused: 0,
|
||||
heart: 0,
|
||||
rocket: 0,
|
||||
eyes: 0
|
||||
},
|
||||
timeline_url: 'https://api.github.com/repos/iptv-org/database/issues/5701/timeline',
|
||||
performed_via_github_app: null,
|
||||
state_reason: null
|
||||
}
|
||||
]
|
||||
81
tests/__data__/input/issues/channels_remove_approved.js
Normal file
81
tests/__data__/input/issues/channels_remove_approved.js
Normal file
|
|
@ -0,0 +1,81 @@
|
|||
module.exports = [
|
||||
{
|
||||
url: 'https://api.github.com/repos/iptv-org/database/issues/5871',
|
||||
repository_url: 'https://api.github.com/repos/iptv-org/database',
|
||||
labels_url: 'https://api.github.com/repos/iptv-org/database/issues/5871/labels{/name}',
|
||||
comments_url: 'https://api.github.com/repos/iptv-org/database/issues/5871/comments',
|
||||
events_url: 'https://api.github.com/repos/iptv-org/database/issues/5871/events',
|
||||
html_url: 'https://github.com/iptv-org/database/issues/5871',
|
||||
id: 1929261634,
|
||||
node_id: 'I_kwDOG1Kwp85y_jJC',
|
||||
number: 5871,
|
||||
title: 'Remove: 002 Radio TV',
|
||||
user: {
|
||||
login: 'freearhey',
|
||||
id: 7253922,
|
||||
node_id: 'MDQ6VXNlcjcyNTM5MjI=',
|
||||
avatar_url: 'https://avatars.githubusercontent.com/u/7253922?v=4',
|
||||
gravatar_id: '',
|
||||
url: 'https://api.github.com/users/freearhey',
|
||||
html_url: 'https://github.com/freearhey',
|
||||
followers_url: 'https://api.github.com/users/freearhey/followers',
|
||||
following_url: 'https://api.github.com/users/freearhey/following{/other_user}',
|
||||
gists_url: 'https://api.github.com/users/freearhey/gists{/gist_id}',
|
||||
starred_url: 'https://api.github.com/users/freearhey/starred{/owner}{/repo}',
|
||||
subscriptions_url: 'https://api.github.com/users/freearhey/subscriptions',
|
||||
organizations_url: 'https://api.github.com/users/freearhey/orgs',
|
||||
repos_url: 'https://api.github.com/users/freearhey/repos',
|
||||
events_url: 'https://api.github.com/users/freearhey/events{/privacy}',
|
||||
received_events_url: 'https://api.github.com/users/freearhey/received_events',
|
||||
type: 'User',
|
||||
site_admin: false
|
||||
},
|
||||
labels: [
|
||||
{
|
||||
id: 5366738347,
|
||||
node_id: 'LA_kwDOG1Kwp88AAAABP-Htqw',
|
||||
url: 'https://api.github.com/repos/iptv-org/database/labels/approved',
|
||||
name: 'approved',
|
||||
color: '85DDDE',
|
||||
default: false,
|
||||
description: ''
|
||||
},
|
||||
{
|
||||
id: 6049155772,
|
||||
node_id: 'LA_kwDOG1Kwp88AAAABaI7KvA',
|
||||
url: 'https://api.github.com/repos/iptv-org/database/labels/blocklist:add',
|
||||
name: 'channels:remove',
|
||||
color: 'e99695',
|
||||
default: false,
|
||||
description: 'Request to remove a channel'
|
||||
}
|
||||
],
|
||||
state: 'open',
|
||||
locked: false,
|
||||
assignee: null,
|
||||
assignees: [],
|
||||
milestone: null,
|
||||
comments: 0,
|
||||
created_at: '2023-10-06T00:35:32Z',
|
||||
updated_at: '2023-10-06T00:35:32Z',
|
||||
closed_at: null,
|
||||
author_association: 'CONTRIBUTOR',
|
||||
active_lock_reason: null,
|
||||
body: '### Channel ID\n\n002RadioTV.do\n\n### Reason\n\nOther\n\n### Notes (optional)\n\n_No response_',
|
||||
reactions: {
|
||||
url: 'https://api.github.com/repos/iptv-org/database/issues/5871/reactions',
|
||||
total_count: 0,
|
||||
'+1': 0,
|
||||
'-1': 0,
|
||||
laugh: 0,
|
||||
hooray: 0,
|
||||
confused: 0,
|
||||
heart: 0,
|
||||
rocket: 0,
|
||||
eyes: 0
|
||||
},
|
||||
timeline_url: 'https://api.github.com/repos/iptv-org/database/issues/5871/timeline',
|
||||
performed_via_github_app: null,
|
||||
state_reason: null
|
||||
}
|
||||
]
|
||||
3
tests/__data__/input/validate/duplicate/categories.csv
Normal file
3
tests/__data__/input/validate/duplicate/categories.csv
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
id,name
|
||||
aaa,AAA
|
||||
aaa,BBB
|
||||
|
1
tests/__data__/input/validate/empty_line/channels.csv
Normal file
1
tests/__data__/input/validate/empty_line/channels.csv
Normal file
|
|
@ -0,0 +1 @@
|
|||
id,name,alt_names,network,owners,country,subdivision,city,broadcast_area,languages,categories,is_nsfw,launched,closed,replaced_by,website,logo
|
||||
|
|
|
@ -0,0 +1,2 @@
|
|||
id,name
|
||||
aaa,AAA
|
||||
|
|
|
@ -0,0 +1,2 @@
|
|||
channel,ref
|
||||
aaa.us,https://github.com/iptv-org/iptv/issues/1831
|
||||
|
2
tests/__data__/input/validate/invalid_value/channels.csv
Normal file
2
tests/__data__/input/validate/invalid_value/channels.csv
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
id,name,alt_names,network,owners,country,subdivision,city,broadcast_area,languages,categories,is_nsfw,launched,closed,replaced_by,website,logo
|
||||
002RadioTV.do,002 Radio TV,,,,DO,,,c/DO,spa,,FALSE,,,,https://www.002radio.com/,https://i.imgur.com/7oNe8xj.png
|
||||
|
|
|
@ -0,0 +1,3 @@
|
|||
name,code,languages,flag
|
||||
Andorra,AD,cat,🇦🇩
|
||||
Dominican Republic,DO,spa,🇩🇴
|
||||
|
|
|
@ -0,0 +1,3 @@
|
|||
code,name
|
||||
cat,Catalan
|
||||
spa,Spanish
|
||||
|
|
|
@ -0,0 +1,2 @@
|
|||
country,name,code
|
||||
AD,Andorra la Vella,AD-07
|
||||
|
2
tests/__data__/input/validate/valid_data/channels.csv
Normal file
2
tests/__data__/input/validate/valid_data/channels.csv
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
id,name,alt_names,network,owners,country,subdivision,city,broadcast_area,languages,categories,is_nsfw,launched,closed,replaced_by,website,logo
|
||||
PeoplesWeather.do,People°s Weather,,,,DO,,,c/DO,spa,,FALSE,,,,,https://i.imgur.com/7oNe8xj.png
|
||||
|
3
tests/__data__/input/validate/valid_data/countries.csv
Normal file
3
tests/__data__/input/validate/valid_data/countries.csv
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
name,code,languages,flag
|
||||
Andorra,AD,cat,🇦🇩
|
||||
Dominican Republic,DO,spa,🇩🇴
|
||||
|
3
tests/__data__/input/validate/valid_data/languages.csv
Normal file
3
tests/__data__/input/validate/valid_data/languages.csv
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
code,name
|
||||
cat,Catalan
|
||||
spa,Spanish
|
||||
|
|
|
@ -0,0 +1,2 @@
|
|||
id,name
|
||||
auto
|
||||
|
24
tests/db/export.test.ts
Normal file
24
tests/db/export.test.ts
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
import { execSync } from 'child_process'
|
||||
import * as fs from 'fs-extra'
|
||||
|
||||
beforeEach(() => {
|
||||
fs.emptyDirSync('tests/__data__/output')
|
||||
})
|
||||
|
||||
it('can export data as json', () => {
|
||||
execSync(
|
||||
'DATA_DIR=tests/__data__/input/data API_DIR=tests/__data__/output/api npm run db:export',
|
||||
{
|
||||
encoding: 'utf8'
|
||||
}
|
||||
)
|
||||
|
||||
expect(content('output/api/blocklist.json')).toEqual(content('expected/api/blocklist.json'))
|
||||
expect(content('output/api/channels.json')).toEqual(content('expected/api/channels.json'))
|
||||
})
|
||||
|
||||
function content(filepath: string) {
|
||||
return fs.readFileSync(`tests/__data__/${filepath}`, {
|
||||
encoding: 'utf8'
|
||||
})
|
||||
}
|
||||
25
tests/db/update.test.ts
Normal file
25
tests/db/update.test.ts
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
import { execSync } from 'child_process'
|
||||
import * as fs from 'fs-extra'
|
||||
|
||||
beforeEach(() => {
|
||||
fs.emptyDirSync('tests/__data__/output')
|
||||
fs.copySync('tests/__data__/input/data', 'tests/__data__/output/data')
|
||||
})
|
||||
|
||||
it('can update db with data from issues', () => {
|
||||
const stdout = execSync('DATA_DIR=tests/__data__/output/data npm run db:update --silent', {
|
||||
encoding: 'utf8'
|
||||
})
|
||||
|
||||
expect(content('output/data/blocklist.csv')).toEqual(content('expected/data/blocklist.csv'))
|
||||
expect(content('output/data/channels.csv')).toEqual(content('expected/data/channels.csv'))
|
||||
expect(stdout).toEqual(
|
||||
'OUTPUT=closes #5871, closes #5901, closes #5701, closes #5900, closes #5899, closes #5898, closes #5897, closes #5891'
|
||||
)
|
||||
})
|
||||
|
||||
function content(filepath: string) {
|
||||
return fs.readFileSync(`tests/__data__/${filepath}`, {
|
||||
encoding: 'utf8'
|
||||
})
|
||||
}
|
||||
81
tests/db/validate.test.ts
Normal file
81
tests/db/validate.test.ts
Normal file
|
|
@ -0,0 +1,81 @@
|
|||
import { execSync } from 'child_process'
|
||||
|
||||
type ExecError = {
|
||||
status: number
|
||||
stdout: string
|
||||
}
|
||||
|
||||
describe('db:validate', () => {
|
||||
it('shows an error if there is an empty line at the end of the file', () => {
|
||||
try {
|
||||
execSync('DATA_DIR=tests/__data__/input/validate/empty_line npm run db:validate', {
|
||||
encoding: 'utf8'
|
||||
})
|
||||
process.exit(1)
|
||||
} catch (error) {
|
||||
expect((error as ExecError).status).toBe(1)
|
||||
expect((error as ExecError).stdout).toContain(
|
||||
'Error: empty lines at the end of file not allowed (channels.csv)'
|
||||
)
|
||||
}
|
||||
})
|
||||
|
||||
it('shows an error if the number of columns in a row is incorrect', () => {
|
||||
try {
|
||||
execSync('DATA_DIR=tests/__data__/input/validate/wrong_num_cols npm run db:validate', {
|
||||
encoding: 'utf8'
|
||||
})
|
||||
process.exit(1)
|
||||
} catch (error) {
|
||||
expect((error as ExecError).status).toBe(1)
|
||||
expect((error as ExecError).stdout).toContain(
|
||||
'Error: row 2 has the wrong number of columns (categories.csv)'
|
||||
)
|
||||
}
|
||||
})
|
||||
|
||||
it('shows an error if one of the lines ends with an invalid character', () => {
|
||||
try {
|
||||
execSync('DATA_DIR=tests/__data__/input/validate/invalid_line_ending npm run db:validate', {
|
||||
encoding: 'utf8'
|
||||
})
|
||||
process.exit(1)
|
||||
} catch (error) {
|
||||
expect((error as ExecError).status).toBe(1)
|
||||
expect((error as ExecError).stdout).toContain(
|
||||
'Error: row 1 has the wrong line ending character, should be CRLF (categories.csv)'
|
||||
)
|
||||
}
|
||||
})
|
||||
|
||||
it('shows an error if there are duplicates in the file', () => {
|
||||
try {
|
||||
execSync('DATA_DIR=tests/__data__/input/validate/duplicate npm run db:validate', {
|
||||
encoding: 'utf8'
|
||||
})
|
||||
process.exit(1)
|
||||
} catch (error) {
|
||||
expect((error as ExecError).status).toBe(1)
|
||||
expect((error as ExecError).stdout).toContain('entry with the id "aaa" already exists')
|
||||
}
|
||||
})
|
||||
|
||||
it('shows an error if an invalid value is specified', () => {
|
||||
try {
|
||||
execSync('DATA_DIR=tests/__data__/input/validate/invalid_value npm run db:validate', {
|
||||
encoding: 'utf8'
|
||||
})
|
||||
process.exit(1)
|
||||
} catch (error) {
|
||||
expect((error as ExecError).status).toBe(1)
|
||||
expect((error as ExecError).stdout).toContain('"aaa.us" is missing in the channels.csv')
|
||||
expect((error as ExecError).stdout).toContain('1 error(s)')
|
||||
}
|
||||
})
|
||||
|
||||
it('does not show an error if all data are correct', () => {
|
||||
execSync('DATA_DIR=tests/__data__/input/validate/valid_data npm run db:validate', {
|
||||
encoding: 'utf8'
|
||||
})
|
||||
})
|
||||
})
|
||||
19
tsconfig.json
Normal file
19
tsconfig.json
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
{
|
||||
"compilerOptions": {
|
||||
"strict": true,
|
||||
"module": "commonjs",
|
||||
"moduleResolution": "node",
|
||||
"target": "es2020",
|
||||
"esModuleInterop": true,
|
||||
"declaration": true,
|
||||
"typeRoots": [
|
||||
"./node_modules/@types",
|
||||
"./src/types"
|
||||
],
|
||||
"allowJs": true
|
||||
},
|
||||
"ts-node": {
|
||||
"esm": true,
|
||||
"transpileOnly": true
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue