From 113c5a1f974990c37e7c74de3301b24ac31996a5 Mon Sep 17 00:00:00 2001 From: nyagami Date: Tue, 26 Mar 2024 17:41:22 +0700 Subject: [PATCH] Update sandbox (#858) * add htmlparser2 * update Hako * find chapter text with regex * flat json * use native language * fix sorting * v2.1.0 --- README.md | 2 +- package-lock.json | 87 ++++++-- package.json | 3 +- plugins/vietnamese/LNHako.ts | 399 +++++++++++++++++++++-------------- scripts/json_plugins.ts | 36 ++-- 5 files changed, 339 insertions(+), 188 deletions(-) diff --git a/README.md b/README.md index c0e495a..a4ab13f 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # LNReader Plugins

-LNReader plugins counting +LNReader plugins counting GitHub issues by-label GitHub issues by-label

diff --git a/package-lock.json b/package-lock.json index 8c04ee8..481d1c1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "lnreader-plugins", - "version": "2.0.0", + "version": "2.1.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "lnreader-plugins", - "version": "2.0.0", + "version": "2.1.0", "license": "MIT", "dependencies": { "body-parser": "^1.20.2", @@ -15,6 +15,7 @@ "cli-color": "^2.0.3", "dayjs": "^1.11.10", "express": "^4.18.2", + "htmlparser2": "^9.1.0", "less": "^4.2.0", "module-alias": "^2.2.3", "protobufjs": "^7.2.6", @@ -870,6 +871,24 @@ "url": "https://github.com/sponsors/fb55" } }, + "node_modules/cheerio/node_modules/htmlparser2": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-8.0.2.tgz", + "integrity": "sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA==", + "funding": [ + "https://github.com/fb55/htmlparser2?sponsor=1", + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.0.1", + "entities": "^4.4.0" + } + }, "node_modules/chokidar": { "version": "3.5.3", "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.3.tgz", @@ -2110,9 +2129,9 @@ } }, "node_modules/htmlparser2": { - "version": "8.0.2", - "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-8.0.2.tgz", - "integrity": "sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA==", + "version": "9.1.0", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-9.1.0.tgz", + "integrity": "sha512-5zfg6mHUoaer/97TxnGpxmbR7zJtPwIYFMZ/H5ucTlPZhKvtum05yiPK3Mgai3a0DyVxv7qYqoweaEd2nrYQzQ==", "funding": [ "https://github.com/fb55/htmlparser2?sponsor=1", { @@ -2123,8 +2142,8 @@ "dependencies": { "domelementtype": "^2.3.0", "domhandler": "^5.0.3", - "domutils": "^3.0.1", - "entities": "^4.4.0" + "domutils": "^3.1.0", + "entities": "^4.5.0" } }, "node_modules/http-errors": { @@ -3661,6 +3680,24 @@ "postcss": "^8.3.11" } }, + "node_modules/sanitize-html/node_modules/htmlparser2": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-8.0.2.tgz", + "integrity": "sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA==", + "funding": [ + "https://github.com/fb55/htmlparser2?sponsor=1", + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.0.1", + "entities": "^4.4.0" + } + }, "node_modules/sax": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/sax/-/sax-1.3.0.tgz", @@ -5185,6 +5222,19 @@ "htmlparser2": "^8.0.1", "parse5": "^7.0.0", "parse5-htmlparser2-tree-adapter": "^7.0.0" + }, + "dependencies": { + "htmlparser2": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-8.0.2.tgz", + "integrity": "sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA==", + "requires": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.0.1", + "entities": "^4.4.0" + } + } } }, "cheerio-select": { @@ -6182,14 +6232,14 @@ "integrity": "sha512-eVcrzgbR4tim7c7soKQKtxa/kQM4TzjnlU83rcZ9bHU6t31ehfV7SktN6McWgwPWg+JYMA/O3qpGxBvFq1z2Jg==" }, "htmlparser2": { - "version": "8.0.2", - "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-8.0.2.tgz", - "integrity": "sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA==", + "version": "9.1.0", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-9.1.0.tgz", + "integrity": "sha512-5zfg6mHUoaer/97TxnGpxmbR7zJtPwIYFMZ/H5ucTlPZhKvtum05yiPK3Mgai3a0DyVxv7qYqoweaEd2nrYQzQ==", "requires": { "domelementtype": "^2.3.0", "domhandler": "^5.0.3", - "domutils": "^3.0.1", - "entities": "^4.4.0" + "domutils": "^3.1.0", + "entities": "^4.5.0" } }, "http-errors": { @@ -7290,6 +7340,19 @@ "is-plain-object": "^5.0.0", "parse-srcset": "^1.0.2", "postcss": "^8.3.11" + }, + "dependencies": { + "htmlparser2": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-8.0.2.tgz", + "integrity": "sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA==", + "requires": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.0.1", + "entities": "^4.4.0" + } + } } }, "sax": { diff --git a/package.json b/package.json index 02d823c..a6e2b9b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "lnreader-plugins", - "version": "2.0.0", + "version": "2.1.0", "description": "Plugins repo for LNReader", "main": "index.js", "scripts": { @@ -24,6 +24,7 @@ "cli-color": "^2.0.3", "dayjs": "^1.11.10", "express": "^4.18.2", + "htmlparser2": "^9.1.0", "less": "^4.2.0", "module-alias": "^2.2.3", "protobufjs": "^7.2.6", diff --git a/plugins/vietnamese/LNHako.ts b/plugins/vietnamese/LNHako.ts index af92eb2..6621aff 100644 --- a/plugins/vietnamese/LNHako.ts +++ b/plugins/vietnamese/LNHako.ts @@ -1,7 +1,6 @@ -import { CheerioAPI, load as parseHTML } from 'cheerio'; +import { Parser } from 'htmlparser2'; import { fetchFile } from '@libs/fetch'; import { Plugin } from '@typings/plugin'; -import { isUrlAbsolute } from '@libs/isAbsoluteUrl'; import { NovelStatus } from '@libs/novelStatus'; import { FilterTypes, Filters } from '@libs/filterInputs'; @@ -11,29 +10,43 @@ class HakoPlugin implements Plugin.PluginBase { icon = 'src/vi/hakolightnovel/icon.png'; site = 'https://ln.hako.vn'; version = '1.0.1'; - parseNovels(loadedCheerio: CheerioAPI) { - const novels: Plugin.NovelItem[] = []; - loadedCheerio('#mainpart .row .thumb-item-flow').each((index, ele) => { - let url = loadedCheerio(ele) - .find('div.thumb_attr.series-title > a') - .attr('href'); - - if (url) { - const name = loadedCheerio(ele).find('.series-title').text().trim(); - let cover = loadedCheerio(ele).find('.img-in-ratio').attr('data-bg'); - - if (cover && !isUrlAbsolute(cover)) { - cover = this.site + cover; - } - - const novel = { name, path: url.replace(this.site, ''), cover }; - - novels.push(novel); - } - }); - return novels; + parseNovels(url: string) { + return fetch(url) + .then(res => res.text()) + .then(html => { + const novels: Plugin.NovelItem[] = []; + let tempNovel = {} as Plugin.NovelItem; + let isGettingUrl = false; + let isParsingNovel = false; + const parser = new Parser({ + onopentag(name, attribs) { + if (attribs['class']?.includes('thumb-item-flow')) { + isParsingNovel = true; + } + if (isParsingNovel) { + if (attribs['class']?.includes('series-title')) { + isGettingUrl = true; + } + if (attribs['class']?.includes('img-in-ratio')) { + tempNovel.cover = attribs['data-bg']; + } + if (isGettingUrl && name === 'a') { + tempNovel.name = attribs['title']; + tempNovel.path = attribs['href']; + novels.push(tempNovel); + tempNovel = {} as Plugin.NovelItem; // re-assign new reference + isGettingUrl = false; + isParsingNovel = false; + } + } + }, + }); + parser.write(html); + parser.end(); + return novels; + }); } - async popularNovels( + popularNovels( pageNo: number, { showLatestNovels, @@ -41,156 +54,228 @@ class HakoPlugin implements Plugin.PluginBase { }: Plugin.PopularNovelsOptions, ): Promise { let link = this.site + '/danh-sach'; - if (filters.alphabet.value) { - link += '/' + filters.alphabet.value; + if (filters) { + if (filters.alphabet.value) { + link += '/' + filters.alphabet.value; + } + const params = new URLSearchParams(); + for (const novelType of filters.type.value) { + params.append(novelType, '1'); + } + for (const status of filters.status.value) { + params.append(status, '1'); + } + params.append('sapxep', filters.sort.value); + link += '?' + params.toString() + '&page=' + pageNo; + } else { + link += '?page=' + pageNo; } - const params = new URLSearchParams(); - for (const novelType of filters.type.value) { - params.append(novelType, '1'); - } - for (const status of filters.status.value) { - params.append(status, '1'); - } - params.append('sapxep', filters.sort.value); - link += '?' + params.toString() + '&page=' + pageNo; - const result = await fetch(link); - const body = await result.text(); - const loadedCheerio = parseHTML(body); - return this.parseNovels(loadedCheerio); + return this.parseNovels(link); } - async parseNovel(novelPath: string): Promise { - const novel: Plugin.SourceNovel = { - path: novelPath, - name: 'Không có tiêu đề', - }; - const result = await fetch(this.site + novelPath); - const body = await result.text(); - - let loadedCheerio = parseHTML(body); - - novel.name = loadedCheerio('.series-name').text().trim(); - - const background = - loadedCheerio('.series-cover > .a6-ratio > div').attr('style') || ''; - const novelCover = background.substring( - background.indexOf('http'), - background.length - 2, - ); - - novel.cover = novelCover - ? isUrlAbsolute(novelCover) - ? novelCover - : this.site + novelCover - : ''; - - novel.summary = loadedCheerio('.summary-content').text().trim(); - - novel.genres = loadedCheerio('.series-information > div:nth-child(1)') - .text() - .trim() - .split(/\n[\s\n]*/) - .join(','); - - novel.author = loadedCheerio( - '.series-information > div:nth-child(2) > .info-value', - ) - .text() - .trim(); - novel.artist = loadedCheerio( - '.series-information > div:nth-child(3) > .info-value', - ) - .text() - .trim(); - - novel.status = loadedCheerio( - '.series-information > div:nth-child(4) > .info-value', - ) - .text() - .trim(); - - switch (novel.status) { - case 'Đang tiến hành': - novel.status = NovelStatus.Ongoing; - break; - case 'Tạm ngưng': - novel.status = NovelStatus.OnHiatus; - break; - case 'Completed': - novel.status = NovelStatus.Completed; - break; - default: - novel.status = NovelStatus.Unknown; - } - - novel.chapters = []; - loadedCheerio('.volume-list').each((idx, ele) => { - const customPage = loadedCheerio(ele).find('.sect-title').text().trim(); - let num = 0, - part = 1; - loadedCheerio(ele) - .find('.list-chapters > li') - .each((idx, chapterEle) => { - const path = loadedCheerio(chapterEle).find('a').attr('href'); - const chapterName = loadedCheerio(chapterEle) - .find('.chapter-name') - .text() - .trim(); - let chapterNumber = Number(chapterName.match(/Chương\s*(\d+)/i)?.[1]); - if (chapterNumber) { - num = chapterNumber; - part = 1; - } else { - chapterNumber = num + part / 10; - part++; - } - const chapterTime = loadedCheerio(chapterEle) - .find('.chapter-time') - .text() - .split('/') - .map(x => Number(x)); - if (path) { - novel.chapters?.push({ - page: customPage, - path, - name: chapterName, - releaseTime: new Date( + parseNovel(novelPath: string): Promise { + return fetch(this.site + novelPath) + .then(res => res.text()) + .then(html => { + const novel: Plugin.SourceNovel = { + path: novelPath, + name: '', + author: '', + artist: '', + summary: '', + genres: '', + status: '', + }; + let isReadingName = 0; + let isReadingSummary = 0; + let isReadingGenre = 0; + let isReadingInfo = 0; + let isReadingInfoValue = 0; + let isParsingVolume = false; + let isReadingVolume = false; + let currentVolume = ''; + let isParsingChapterList = false; + let num = 0, + part = 0; + let isReadingChapterTime = false; + const chapters: Plugin.ChapterItem[] = []; + let tempChapter = {} as Plugin.ChapterItem; + const parser = new Parser({ + onopentag(name, attribs) { + if (attribs['class'] === 'series-name') { + isReadingName = 1; + } else if ( + !novel.cover && + attribs['class']?.includes('img-in-ratio') + ) { + const background = attribs['style']; + if (background) { + novel.cover = background.substring( + background.indexOf('http'), + background.length - 2, + ); + } + } else if (attribs['class'] === 'summary-content') { + isReadingSummary = 1; + } else if (attribs['class'] === 'series-gerne-item') { + isReadingGenre = 1; + } else if (attribs['class'] === 'info-item') { + isReadingInfo += 1; + } else if (attribs['class']?.includes('volume-list')) { + isParsingVolume = true; + num = 0; + part = 1; + } else if (isParsingVolume && attribs['class'] === 'sect-title') { + isReadingVolume = true; + currentVolume = ''; + } else if (isParsingVolume && name === 'ul') { + isParsingChapterList = true; + } else if ( + isParsingChapterList && + name === 'a' && + attribs['title'] !== null + ) { + const chapterName = attribs['title']; + let chapterNumber = Number( + chapterName.match(/Chương\s*(\d+)/i)?.[1], + ); + if (chapterNumber) { + if (num === chapterNumber) { + chapterNumber = num + part / 10; + part += 1; + } else { + num = chapterNumber; + part = 1; + } + } else { + chapterNumber = num + part / 10; + part++; + } + tempChapter = { + path: attribs['href'], + name: chapterName, + page: currentVolume, + chapterNumber: chapterNumber, + }; + } else if ( + isParsingChapterList && + attribs['class'] === 'chapter-time' + ) { + isReadingChapterTime = true; + } + if (isReadingName === 1 && name === 'a') { + isReadingName = 2; + } + if (isReadingInfo !== 0 && name === 'a') { + isReadingInfoValue = 1; + } + }, + ontext(data) { + if (isReadingName === 2) { + novel.name += data; + } else if (isReadingSummary !== 0) { + if (isReadingSummary === 2) { + novel.summary += '\n' + data; + isReadingSummary = 1; + } else { + novel.summary += data; + } + } else if (isReadingGenre === 1) { + novel.genres += data; + } else if (isReadingInfoValue === 1 && isReadingInfo <= 3) { + if (isReadingInfo === 1) { + novel.author += data; + } else if (isReadingInfo === 2) { + novel.artist += data; + } else if (isReadingInfo === 3) { + novel.status += data; + } + } else if (isReadingVolume) { + currentVolume += data.trim(); + } else if (isReadingChapterTime) { + const chapterTime = data.split('/').map(x => Number(x)); + tempChapter.releaseTime = new Date( chapterTime[2], chapterTime[1], chapterTime[0], - ).toISOString(), - chapterNumber: chapterNumber, - }); - } + ).toISOString(); + chapters.push(tempChapter); + isReadingChapterTime = false; + tempChapter = {} as Plugin.ChapterItem; + } + }, + onclosetag(name) { + if (isReadingName === 2) { + isReadingName = 0; + } else if (isReadingSummary !== 0) { + if (name !== 'div') { + isReadingSummary = 2; + } else { + isReadingSummary = 0; + } + } else if (isReadingGenre === 1) { + isReadingGenre = 0; + novel.genres += ','; + } else if (isReadingVolume) { + isReadingVolume = false; + } else if (isReadingChapterTime) isReadingChapterTime = false; + if (name === 'ul') { + isParsingChapterList = false; + } + if (isReadingInfoValue === 1) { + isReadingInfoValue = 0; + if (isReadingInfo === 3) { + isReadingInfo = 0; + } + } + if (name === 'section') { + isParsingVolume = false; + } + }, }); - }); - return novel; + + parser.write(html); + parser.end(); + novel.chapters = chapters; + switch (novel.status?.trim()) { + case 'Đang tiến hành': + novel.status = NovelStatus.Ongoing; + break; + case 'Tạm ngưng': + novel.status = NovelStatus.OnHiatus; + break; + case 'Completed': + novel.status = NovelStatus.Completed; + break; + default: + novel.status = NovelStatus.Unknown; + } + novel.genres = novel.genres?.replace(/,*\s*$/, ''); + return novel; + }); } - async parseChapter(chapterPath: string): Promise { - const result = await fetch(this.site + chapterPath); - const body = await result.text(); - - const loadedCheerio = parseHTML(body); - - const chapterText = loadedCheerio('#chapter-content').html() || ''; - - return chapterText; + parseChapter(chapterPath: string): Promise { + return fetch(this.site + chapterPath) + .then(res => res.text()) + .then( + html => + html.match( + /(
[^]+)
a.id.localeCompare(b.id)); +json.sort((a, b) => { + if (a.lang === b.lang) return a.id.localeCompare(b.id); + return 0; +}); fs.writeFileSync(jsonMinPath, JSON.stringify(json)); fs.writeFileSync(jsonPath, JSON.stringify(json, null, '\t'));