parser2 sources (royalroad, ranobes, lightnovelpub, lnmtl) (#865)

* lnmtl parser2

* add royalroad, change version

* ranobes parser

* lightnovelpub

* cleanup
This commit is contained in:
K1ngfish3r
2024-04-24 13:56:31 +08:00
committed by GitHub
parent a93cd70415
commit 2c4490e5bd
4 changed files with 1098 additions and 333 deletions
+292 -99
View File
@@ -1,4 +1,4 @@
import { load as parseHTML } from 'cheerio'; import { Parser } from 'htmlparser2';
import { fetchApi, fetchFile } from '@libs/fetch'; import { fetchApi, fetchFile } from '@libs/fetch';
import { Plugin } from '@typings/plugin'; import { Plugin } from '@typings/plugin';
import { Filters, FilterTypes } from '@libs/filterInputs'; import { Filters, FilterTypes } from '@libs/filterInputs';
@@ -7,7 +7,7 @@ import dayjs from 'dayjs';
class LightNovelPub implements Plugin.PagePlugin { class LightNovelPub implements Plugin.PagePlugin {
id = 'lightnovelpub'; id = 'lightnovelpub';
name = 'LightNovelPub'; name = 'LightNovelPub';
version = '1.0.1'; version = '2.0.0';
icon = 'src/en/lightnovelpub/icon.png'; icon = 'src/en/lightnovelpub/icon.png';
site = 'https://www.lightnovelpub.com/'; site = 'https://www.lightnovelpub.com/';
headers = { headers = {
@@ -15,6 +15,42 @@ class LightNovelPub implements Plugin.PagePlugin {
'Content-Type': 'application/json', 'Content-Type': 'application/json',
}; };
parseNovels(html: string) {
const novels: Plugin.NovelItem[] = [];
let tempNovel = {} as Plugin.NovelItem;
let isParsingNovel = false;
const parser = new Parser ({
onopentag(name, attribs){
if (isParsingNovel){
if (name === 'a'){
tempNovel.path = attribs['href'].slice(1);
tempNovel.name = attribs['title'];
}
if (name === 'img'){
tempNovel.cover = attribs['data-src'] || attribs['src'];
}
if (tempNovel.path && tempNovel.cover){
novels.push(tempNovel);
tempNovel = {} as Plugin.NovelItem;
}
}
},
onattribute(name, value){
if (name === 'class' && value === 'novel-item'){
isParsingNovel = true;
}
},
onclosetag(name){
if(name === 'li'){
isParsingNovel = false;
}
}
})
parser.write(html);
parser.end();
return novels;
}
async popularNovels( async popularNovels(
page: number, page: number,
{ filters }: Plugin.PopularNovelsOptions<typeof this.filters>, { filters }: Plugin.PopularNovelsOptions<typeof this.filters>,
@@ -27,91 +63,171 @@ class LightNovelPub implements Plugin.PagePlugin {
const body = await fetchApi(link).then(r => r.text()); const body = await fetchApi(link).then(r => r.text());
const loadedCheerio = parseHTML(body); return this.parseNovels(body);
const novels: Plugin.NovelItem[] = [];
loadedCheerio('.novel-item.ads').remove();
loadedCheerio('.novel-item').each((idx, ele) => {
const novelName = loadedCheerio(ele).find('.novel-title').text().trim();
const novelCover = loadedCheerio(ele).find('img').attr('data-src');
const novelUrl = loadedCheerio(ele)
.find('.novel-title > a')
.attr('href')
?.substring(1);
if (!novelUrl) return;
const novel = {
name: novelName,
cover: novelCover,
path: novelUrl,
};
novels.push(novel);
});
return novels;
} }
async parseNovel( async parseNovel(
novelPath: string, novelPath: string,
): Promise<Plugin.SourceNovel & { totalPages: number }> { ): Promise<Plugin.SourceNovel & { totalPages: number }> {
const body = await fetchApi(this.site + novelPath).then(r => r.text()); const body = await fetchApi(this.site + novelPath).then(r => r.text());
const loadedCheerio = parseHTML(body);
const totalChapters = parseInt(
loadedCheerio('.header-stats span:first strong').text(),
10,
);
const novel: Plugin.SourceNovel & { totalPages: number } = { const novel: Plugin.SourceNovel & { totalPages: number } = {
path: novelPath, path: novelPath,
name: loadedCheerio('h1.novel-title').text().trim() || 'Untitled', name: '',
cover: loadedCheerio('figure.cover > img').attr('data-src'), summary: '',
author: loadedCheerio('.author > a > span').text(), totalPages: 1,
summary: loadedCheerio('.summary > .content').text().trim(),
status: loadedCheerio('.header-stats span:last strong').text(),
totalPages: Math.ceil(totalChapters / 100),
chapters: [], chapters: [],
}; }
let isHeaderStats = false;
novel.genres = loadedCheerio('.categories ul li') let isStatus = false;
.map((a, ex) => loadedCheerio(ex).text().trim()) let isTotalChapters = false;
.toArray() let isNovelName = false;
.join(','); let isAuthorName = false;
let isSummary = false;
let isGenres = false;
let isTags = false;
let genreArray: string[] = []
let isCover = false;
const parser = new Parser({
onopentag(name, attribs){
if(name === 'h1' && attribs['class']?.includes('novel-title')){
isNovelName = true;
}
if(name === 'div' && attribs['class']?.includes('content')){
isSummary = true;
}
if(name === 'figure' && attribs['class'] === 'cover'){
isCover = true;
}
if(isCover){
if(name === 'img'){
novel.cover = attribs['data-src'] || attribs['src'];
}
}
},
onopentagname(name){
if (isHeaderStats && name === 'strong'){
isStatus = true;
}
if (isSummary && name === 'br'){
novel.summary += '\n';
}
if (isGenres && name === 'a'){
isTags = true;
}
},
onattribute(name,value){
if (name === 'class' && value === 'header-stats'){
isTotalChapters = true;
isHeaderStats = true;
}
if (name === 'itemprop' && value === 'author'){
isAuthorName = true;
}
if (name === 'class' && value === 'categories'){
isGenres = true;
}
},
ontext(data){
if(isTotalChapters){
novel.totalPages = Math.ceil(parseInt(data, 10) / 100);
}
if(isStatus){
novel.status = data.trim();
}
if(isNovelName){
novel.name += data.trim();
}
if(isAuthorName){
novel.author = data;
}
if(isSummary){
novel.summary += data;
}
if(isTags){
genreArray.push(data);
}
},
onclosetag(name){
if(name === 'strong'){
isTotalChapters = false;
isStatus = false;
}
if(name === 'i'){
isStatus = false;
}
if(name === 'h1'){
isNovelName = false;
}
if(name === 'span'){
isAuthorName = false;
}
if(name === 'div'){
isHeaderStats = false;
isSummary = false;
isGenres = false;
}
if(name === 'a'){
isTags = false;
}
if(name === 'figure'){
isCover = false;
}
}
})
parser.write(body);
parser.end();
novel.genres = genreArray.join(', ');
return novel; return novel;
} }
async parsePage(novelPath: string, page: string): Promise<Plugin.SourcePage> { async parsePage(novelPath: string, page: string): Promise<Plugin.SourcePage> {
const url = this.site + novelPath + '/chapters/page-' + page; const url = this.site + novelPath + '/chapters/page-' + page;
const body = await fetchApi(url).then(res => res.text()); const body = await fetchApi(url).then(res => res.text());
const loadedCheerio = parseHTML(body);
const chapter: Plugin.ChapterItem[] = []; const chapter: Plugin.ChapterItem[] = [];
loadedCheerio('.chapter-list li').each(function () { let tempChapter = {} as Plugin.ChapterItem;
const chapterName = let isChapterItem = false;
'Chapter ' + let isChapterList = false;
loadedCheerio(this).find('.chapter-no').text().trim() + const parser = new Parser ({
' - ' + onopentag(name, attribs){
loadedCheerio(this).find('.chapter-title').text().trim(); if(isChapterList){
if (name === 'li'){
const releaseDate = loadedCheerio(this) isChapterItem = true;
.find('.chapter-update') tempChapter.chapterNumber = Number(attribs['data-orderno']);
.attr('datetime'); }
if (isChapterItem){
const chapterUrl = loadedCheerio(this) if (name === 'a'){
.find('a') tempChapter.name = attribs['title'];
.attr('href') tempChapter.path = attribs['href'].slice(1);
?.substring(1); }
if (!chapterUrl) return; if (name === 'time'){
tempChapter.releaseTime = dayjs(attribs['datetime']).toISOString();
chapter.push({ }
name: chapterName, if (
path: chapterUrl, tempChapter.chapterNumber &&
releaseTime: dayjs(releaseDate).toISOString(), tempChapter.path &&
}); tempChapter.releaseTime){
}); chapter.push(tempChapter);
tempChapter = {} as Plugin.ChapterItem;
}
}
}
},
onattribute(name, value){
if (name === 'class' && value === 'chapter-list'){
isChapterList = true;
}
},
onclosetag(name){
if (name === 'a'){
isChapterItem = false;
}
if (name === 'ul'){
isChapterList = false;
}
}
})
parser.write(body);
parser.end();
const chapters = chapter; const chapters = chapter;
return { chapters }; return { chapters };
} }
@@ -119,9 +235,98 @@ class LightNovelPub implements Plugin.PagePlugin {
async parseChapter(chapterPath: string): Promise<string> { async parseChapter(chapterPath: string): Promise<string> {
const body = await fetchApi(this.site + chapterPath).then(r => r.text()); const body = await fetchApi(this.site + chapterPath).then(r => r.text());
const loadedCheerio = parseHTML(body); let chapterText = '';
let isChapter = false;
let chapterText = loadedCheerio('#chapter-container').html() || ''; let isPtag = false;
let isStyleText = false;
const parser = new Parser({
onopentag(name,attribs){
if (isChapter && name === 'div'){
let stylediv = attribs['style']
if(stylediv){
chapterText += `<div style="${stylediv}">`
isStyleText = true;
} else {
chapterText += `<div>`
}
}
if (isChapter && name === 'table'){
let w = attribs['width'];
if(w) {
chapterText += `<table width="${w}">`;
} else {
chapterText += `<table>`;
}
}
if (isChapter && name === 'tbody'){
chapterText += `<tbody>`;
}
if (isChapter && name === 'tr'){
chapterText += `<tr>`;
}
if (isChapter && name === 'td'){
let w1 = attribs['width'];
if(w1) {
chapterText += `<td width="${w1}">`;
} else {
chapterText += `<td>`;
}
}
},
onattribute(name, value){
if (name === 'id' && value === 'chapter-container'){
isChapter = true;
}
if (name === 'class' && value?.includes('chapternav')){
isChapter = false;
isPtag = false;
}
},
onopentagname(name){
if (isChapter && name === 'p'){
chapterText += '<p>';
isPtag = true;
if(isStyleText){
isStyleText = false;
}
}
if (isChapter && name === 'br'){
chapterText += `<br>`;
}
},
ontext(data){
if(isPtag){
chapterText += data;
}
if(isStyleText){
chapterText += data;
}
},
onclosetag(name){
if (name === 'p'){
isPtag = false;
chapterText += '</p>'
}
if (isChapter && name === 'td'){
chapterText += `</td>`;
}
if (isChapter && name === 'tr'){
chapterText += `</tr>`;
}
if (isChapter && name === 'tbody'){
chapterText += `</tbody>`;
}
if (isChapter && name === 'table'){
chapterText += `</table>`;
}
if (isChapter && name === 'div'){
isStyleText = false;
chapterText += `</div>`
}
}
})
parser.write(body)
parser.end();
return chapterText; return chapterText;
} }
@@ -133,39 +338,27 @@ class LightNovelPub implements Plugin.PagePlugin {
const url = `${this.site}lnsearchlive`; const url = `${this.site}lnsearchlive`;
const link = `${this.site}search`; const link = `${this.site}search`;
const response = await fetchApi(link).then(r => r.text()); const response = await fetchApi(link).then(r => r.text());
const token = parseHTML(response); let verifytoken = '';
let verifytoken = token('#novelSearchForm > input').attr('value'); const parser = new Parser ({
onopentag(name, attribs) {
if (name === 'input' && attribs['name']?.includes('LNRequestVerifyToken')){
verifytoken = attribs['value'];
}
},
})
parser.write(response);
parser.end();
let formData = new FormData(); const formData = new FormData();
formData.append('inputContent', searchTerm); formData.append('inputContent', searchTerm);
const body = await fetchApi(url, { const body = await fetchApi(url, {
method: 'POST', method: 'POST',
headers: { LNRequestVerifyToken: verifytoken! }, headers: { LNRequestVerifyToken: verifytoken! },
body: formData, body: formData,
}).then(r => r.text()); }).then(r => r.json());
let loadedCheerio = parseHTML(body); return this.parseNovels(body.resultview)
let novels: Plugin.NovelItem[] = [];
let results = JSON.parse(loadedCheerio('body').text());
loadedCheerio = parseHTML(results.resultview);
loadedCheerio('.novel-item').each((idx, ele) => {
const novelName = loadedCheerio(ele).find('h4.novel-title').text().trim();
const novelCover = loadedCheerio(ele).find('img').attr('src');
const novelUrl = loadedCheerio(ele).find('a').attr('href')?.substring(1);
if (!novelUrl) return;
novels.push({
name: novelName,
path: novelUrl,
cover: novelCover,
});
});
return novels;
} }
async fetchImage(url: string): Promise<string | undefined> { async fetchImage(url: string): Promise<string | undefined> {
+231 -92
View File
@@ -1,4 +1,4 @@
import { load as parseHTML } from 'cheerio'; import { Parser } from 'htmlparser2';
import { fetchApi, fetchFile } from '@libs/fetch'; import { fetchApi, fetchFile } from '@libs/fetch';
import { FilterTypes, Filters } from '@libs/filterInputs'; import { FilterTypes, Filters } from '@libs/filterInputs';
import { Plugin } from '@typings/plugin'; import { Plugin } from '@typings/plugin';
@@ -8,7 +8,7 @@ class LnMTLPlugin implements Plugin.PagePlugin {
name = 'LnMTL'; name = 'LnMTL';
icon = 'src/en/lnmtl/icon.png'; icon = 'src/en/lnmtl/icon.png';
site = 'https://lnmtl.com/'; site = 'https://lnmtl.com/';
version = '1.0.1'; version = '2.0.0';
async sleep(ms: number) { async sleep(ms: number) {
return new Promise(resolve => setTimeout(resolve, ms)); return new Promise(resolve => setTimeout(resolve, ms));
@@ -24,98 +24,191 @@ class LnMTLPlugin implements Plugin.PagePlugin {
link += `&filter=${filters.storyStatus.value}`; link += `&filter=${filters.storyStatus.value}`;
link += `&page=${page}`; link += `&page=${page}`;
const body = await fetchApi(link).then(result => result.text()); const body = await fetchApi(link);
const html = await body.text();
const loadedCheerio = parseHTML(body); const baseUrl = this.site;
let isParsingNovel = false;
let tempNovel = {} as Plugin.NovelItem;
const novels: Plugin.NovelItem[] = []; const novels: Plugin.NovelItem[] = [];
const parser = new Parser ({
loadedCheerio('.media').each((i, el) => { onopentag(name, attribs) {
const novelName = loadedCheerio(el).find('h4').text(); if (attribs['class']?.includes('media-left')) {
const novelCover = loadedCheerio(el).find('img').attr('src'); isParsingNovel = true;
const novelUrl = loadedCheerio(el).find('h4 > a').attr('href'); }
if (isParsingNovel) {
if (!novelUrl) return; switch (name){
case 'a':
const novel = { tempNovel.path = attribs['href'].slice(baseUrl.length);
name: novelName, break;
cover: novelCover, case 'img':
path: novelUrl.replace(this.site, ''), tempNovel.name = attribs['alt'];
}; tempNovel.cover = attribs['src'];
break;
novels.push(novel); }
if (tempNovel.path && tempNovel.name){
novels.push(tempNovel);
tempNovel = {} as Plugin.NovelItem;
}
}
},
onclosetag(name){
if (name === 'div'){
isParsingNovel = false;
}
},
}); });
parser.write(html);
parser.end();
return novels; return novels;
} }
async parseNovel( async parseNovel(
novelPath: string, novelPath: string,
): Promise<Plugin.SourceNovel & { totalPages: number }> { ): Promise<Plugin.SourceNovel & { totalPages: number }> {
const result = await fetchApi(this.site + novelPath); const body = await fetchApi(this.site + novelPath);
const body = await result.text(); const html = await body.text().then(r => r.replace(/>\s+</g, '><'));
const loadedCheerio = parseHTML(body);
const volumes = JSON.parse(
loadedCheerio('main')
.next()
.html()
?.match(/lnmtl.volumes = \[(.*?)\]/)![0]
?.replace('lnmtl.volumes = ', '') || '',
);
const novel: Plugin.SourceNovel & { totalPages: number } = { const novel: Plugin.SourceNovel & { totalPages: number } = {
path: novelPath, path: novelPath,
name: loadedCheerio('.novel-name').text() || 'Untitled', name: '',
cover: loadedCheerio('div.novel').find('img').attr('src'), totalPages: 1,
summary: loadedCheerio('div.description').text().trim(), summary: '',
totalPages: volumes.length, author: '',
status: '',
chapters: [], chapters: [],
}; };
loadedCheerio('.panel-body > dl').each(function () { let isScript = false;
const detailName = loadedCheerio(this).find('dt').text().trim(); let isDescription = false;
const detail = loadedCheerio(this).find('dd').text().trim(); let isSource = false;
let isPanelKey = false;
switch (detailName) { let isPanelValue = 0;
case 'Authors': let isAuthorKey = false
novel.author = detail; let isStatusKey = false
break; let isList = 0;
case 'Current status': let isGenres = false;
novel.status = detail; let genreArray: string[] = []
break; const parser = new Parser({
} onopentag(name,attribs) {
}); if (name === 'img' && attribs['class']?.includes('img-rounded')){
novel.name = attribs['title'];
novel.genres = loadedCheerio('.panel-heading:contains(" Genres ")') novel.cover = attribs['src'];
.next() }
.find('a') if (name === 'ul' && attribs['class']?.includes('list-inline')){
.map((i, el) => loadedCheerio(el).text()) isList++;
.toArray() }
.join(','); },
onopentagname(name){
if (name === 'dt'){
isPanelKey = true;
}
if (name === 'dd'){
isPanelValue++;
}
if (isList === 1 && name === 'li'){
isGenres = true;
}
},
onattribute(name, value){
if (name === 'class' && value === 'description'){
isDescription = true;
}
if (name === 'class' && value === 'source'){
isDescription = false;
isSource = true;
}
if (name === 'class' && value === 'progress'){
isSource = false;
}
},
ontext(data){
if(isScript){
const volume = JSON.parse(
data.match(/lnmtl.volumes = (.+])(?=;)/)![1] || ''
);
novel.totalPages = volume.length;
}
if(isDescription){
novel.summary += data.trim() + '\n\n';
}
if(isSource){
novel.summary += data;
}
if(isPanelKey){
switch(data){
case 'Authors':
isAuthorKey = true;
break;
case 'Current status':
isStatusKey = true;
break;
}
}
if(isAuthorKey && isPanelValue === 1){
novel.author += data.trim();
isAuthorKey = false;
}
if(isStatusKey && isPanelValue === 2){
novel.status += data.trim();
isStatusKey = false;
}
if(isGenres){
genreArray.push(data.trim());
novel.genres = genreArray.join(', ');
}
},
onclosetag(name){
if (name === 'ul'){
isGenres = false;
}
if(name === 'main'){
isPanelKey = false;
isScript = true;
}
if (name === 'script'){
isScript = false;
}
},
})
parser.write(html);
parser.end();
return novel; return novel;
} }
async parsePage(novelPath: string, page: string): Promise<Plugin.SourcePage> { async parsePage(novelPath: string, page: string): Promise<Plugin.SourcePage> {
const result = await fetchApi(this.site + novelPath); const result = await fetchApi(this.site + novelPath);
const body = await result.text(); const html = await result.text().then(r => r.replace(/>\s+</g, '><'));
let isScript = false;
const loadedCheerio = parseHTML(body); let volume: VolumeEntry = {
id: '',
const volumes = JSON.parse( title: '',
loadedCheerio('main') }
.next() const parser = new Parser({
.html() ontext(data){
?.match(/lnmtl.volumes = \[(.*?)\]/)![0] if(isScript){
?.replace('lnmtl.volumes = ', '') || '', const volumes = JSON.parse(
)[+page - 1]; data.match(/lnmtl.volumes = (.+])(?=;)/)![1] || ''
);
volume = volumes[+page-1]
}
},
onclosetag(name){
if(name === 'main'){
isScript = true;
}
if (name === 'script'){
isScript = false;
}
},
})
parser.write(html);
parser.end();
const chapter: Plugin.ChapterItem[] = []; const chapter: Plugin.ChapterItem[] = [];
await this.sleep(1000); await this.sleep(1000);
const volumeData = await fetchApi( const volumeData = await fetchApi(
`${this.site}chapter?volumeId=${volumes.id}`, `${this.site}chapter?volumeId=${volume.id}`,
); );
const volumePage = await volumeData.json(); const volumePage = await volumeData.json();
const firstPage = volumePage.data.map((chapter: ChapterEntry) => ({ const firstPage = volumePage.data.map((chapter: ChapterEntry) => ({
@@ -128,7 +221,7 @@ class LnMTLPlugin implements Plugin.PagePlugin {
for (let i = 2; i <= volumePage.last_page; i++) { for (let i = 2; i <= volumePage.last_page; i++) {
await this.sleep(1000); await this.sleep(1000);
const chapterData = await fetchApi( const chapterData = await fetchApi(
`${this.site}chapter?page=${i}&volumeId=${volumes.id}`, `${this.site}chapter?page=${i}&volumeId=${volume.id}`,
); );
const chapterInfo = await chapterData.json(); const chapterInfo = await chapterData.json();
@@ -146,34 +239,75 @@ class LnMTLPlugin implements Plugin.PagePlugin {
async parseChapter(chapterPath: string): Promise<string> { async parseChapter(chapterPath: string): Promise<string> {
const result = await fetchApi(this.site + chapterPath); const result = await fetchApi(this.site + chapterPath);
const body = await result.text(); const html = await result.text();
let isText = false;
let chapterText = '';
const parser = new Parser({
onopentag(name,attribs){
if(name === 'sentence' && attribs['class']?.includes('translated')){
isText = true;
chapterText += '<p>'
}
},
onopentagname(name){
if(name === 'nav'){
isText = false;
}
},
ontext(data){
if(isText){
chapterText += data;
}
},
onclosetag(name){
if (name === 'sentence'){
chapterText += '</p>'
isText = false;
}
}
})
parser.write(html);
parser.end();
const loadedCheerio = parseHTML(body); return chapterText.replace(/„/g, '“');
loadedCheerio('.original, script').remove();
loadedCheerio('sentence.translated').wrap('<p></p>');
let chapterText = loadedCheerio('.chapter-body').html()?.replace(/„/g, '“');
if (!chapterText) {
chapterText = loadedCheerio('.alert.alert-warning').text();
}
return chapterText;
} }
async searchNovels( async searchNovels(
searchTerm: string, searchTerm: string,
pageNo: number, pageNo: number,
): Promise<Plugin.NovelItem[]> { ): Promise<Plugin.NovelItem[]> {
const body = await fetchApi(this.site).then(r => r.text()); const html = await fetchApi(this.site)
const loadedCheerio = parseHTML(body); .then(b => b.text())
.then(r => r.replace(/>\s+</g, '><'));
const list = loadedCheerio('footer') let isScript = false;
.next() let isFooter = false;
.next() let list = ''
.html() const parser = new Parser({
?.match(/prefetch: '\/(.*json)/)![1]; onopentag(name, attribs) {
if(isFooter && name === 'script' &&
attribs['type']?.includes('application/javascript')){
isScript = true;
}
},
ontext(data){
if (isScript){
list = data.match(/prefetch: '\/(.*json)/)![1];
}
},
onclosetag(name){
if (name === 'footer'){
isFooter = true;
}
if (name === 'script'){
isScript = false;
}
if (name === 'body'){
isFooter = false;
}
}
})
parser.write(html);
parser.end();
const search = await fetch(`${this.site}${list}`); const search = await fetch(`${this.site}${list}`);
const data = await search.json(); const data = await search.json();
@@ -243,3 +377,8 @@ interface ChapterEntry {
slug: string; slug: string;
created_at: string; created_at: string;
} }
interface VolumeEntry {
id: string;
title: string;
}
+299 -80
View File
@@ -1,47 +1,69 @@
import { CheerioAPI, load as parseHTML } from 'cheerio'; import { Parser } from 'htmlparser2';
import { fetchApi, fetchFile } from '@libs/fetch'; import { fetchApi, fetchFile } from '@libs/fetch';
import { FilterTypes, Filters } from '@libs/filterInputs'; import { FilterTypes, Filters } from '@libs/filterInputs';
import { Plugin } from '@typings/plugin'; import { Plugin } from '@typings/plugin';
interface ChapterEntry {
id: number;
title: string;
date: string;
link: string;
}
class RanobesPlugin implements Plugin.PagePlugin { class RanobesPlugin implements Plugin.PagePlugin {
id = 'ranobes'; id = 'ranobes';
name = 'Ranobes'; name = 'Ranobes';
icon = 'src/en/ranobes/icon.png'; icon = 'src/en/ranobes/icon.png';
site = 'https://ranobes.top'; site = 'https://ranobes.top';
filters?: Filters | undefined; filters?: Filters | undefined;
version = '1.1.0'; version = '2.0.0';
async sleep(ms: number) { async sleep(ms: number) {
return new Promise(resolve => setTimeout(resolve, ms)); return new Promise(resolve => setTimeout(resolve, ms));
} }
parseNovels(loadedCheerio: CheerioAPI) { parseNovels(html: string) {
const novels: Plugin.NovelItem[] = []; const novels: Plugin.NovelItem[] = [];
let tempNovel = {} as Plugin.NovelItem;
loadedCheerio('.short-cont').each((i, el) => { tempNovel.name = '';
const novelName = loadedCheerio(el).find('h2.title').text(); const baseUrl = this.site
const novelUrl = loadedCheerio(el).find('h2.title a').attr('href'); let isParsingNovel = false;
const novelCover = loadedCheerio(el) let isTitleTag = false;
.find('figure') let isNovelName = false;
.attr('style') const parser = new Parser({
?.replace(/.*url\((.*?)\)./g, '$1'); onopentag(name, attribs) {
if (attribs['class']?.includes('short-cont')){
if (!novelUrl) return; isParsingNovel = true;
}
const novel = { if (isParsingNovel){
name: novelName, if (name === 'h2' && attribs['class']?.includes('title')){
cover: novelCover, isTitleTag = true;
path: novelUrl?.replace(this.site, ''), }
}; if (isTitleTag && name === 'a'){
novels.push(novel); tempNovel.path = attribs['href'].slice(baseUrl.length);
}); isNovelName = true;
}
if (name === 'figure'){
tempNovel.cover = attribs['style']
.replace(/.*url\((.*?)\)./g, '$1');
}
if (tempNovel.path && tempNovel.cover){
novels.push(tempNovel);
tempNovel = {} as Plugin.NovelItem;
tempNovel.name = '';
}
}
},
ontext(data){
if(isNovelName){
tempNovel.name += data;
}
},
onclosetag(name){
if(name === 'h2'){
isNovelName = false;
isTitleTag = false;
}
if(name === 'figure'){
isParsingNovel = false;
}
}
})
parser.write(html);
parser.end();
return novels; return novels;
} }
@@ -51,7 +73,7 @@ class RanobesPlugin implements Plugin.PagePlugin {
chapter.push({ chapter.push({
name: item.title, name: item.title,
releaseTime: new Date(item.date).toISOString(), releaseTime: new Date(item.date).toISOString(),
path: item.link.replace(this.site, ''), path: item.link.slice(this.site.length),
}); });
}); });
return chapter.reverse(); return chapter.reverse();
@@ -65,10 +87,9 @@ class RanobesPlugin implements Plugin.PagePlugin {
}: Plugin.PopularNovelsOptions<typeof this.filters>, }: Plugin.PopularNovelsOptions<typeof this.filters>,
): Promise<Plugin.NovelItem[]> { ): Promise<Plugin.NovelItem[]> {
let link = `${this.site}/novels/page/${page}/`; let link = `${this.site}/novels/page/${page}/`;
const body = await fetchApi(link).then(result => result.text()); const body = await fetchApi(link).then(r => r.text());
const loadedCheerio = parseHTML(body); return this.parseNovels(body);
return this.parseNovels(loadedCheerio);
} }
async parseNovel( async parseNovel(
@@ -76,52 +97,133 @@ class RanobesPlugin implements Plugin.PagePlugin {
): Promise<Plugin.SourceNovel & { totalPages: number }> { ): Promise<Plugin.SourceNovel & { totalPages: number }> {
const url = this.site + novelPath; const url = this.site + novelPath;
const result = await fetchApi(url); const result = await fetchApi(url);
const body = await result.text(); const html = await result.text();
let loadedCheerio = parseHTML(body);
const novel: Plugin.SourceNovel & { totalPages: number } = { const novel: Plugin.SourceNovel & { totalPages: number } = {
path: novelPath, path: novelPath,
name: loadedCheerio('.poster img').attr('alt') || 'Untitled', name: '',
cover: this.site + loadedCheerio('.poster img').attr('src'), summary: '',
author: loadedCheerio('[itemprop="creator"]').text(),
chapters: [], chapters: [],
totalPages: 1, totalPages: 1,
};
novel.summary = loadedCheerio('.moreless')
.find('br')
.replaceWith('\n')
.end()
.text()
.trim();
novel.status =
loadedCheerio('li:contains("Status") span').text() === 'Ongoing'
? 'Ongoing'
: 'Completed';
novel.genres = loadedCheerio('div#mc-fs-genre').text().trim();
let chapterListUrl = loadedCheerio('.read-continue:last').attr('href');
if (!chapterListUrl?.startsWith('http')) {
chapterListUrl = this.site + chapterListUrl;
} }
let baseUrl = this.site;
let isCover = false;
let isAuthor = false;
let isSummary = false;
let isStatus = false;
let isStatusText = false;
let isGenres = false;
let isGenresText = false;
let genreArray: string[] = []
let novelId = '';
const parser = new Parser ({
onopentag(name,attribs){
if (attribs['class'] === 'poster'){
isCover = true;
}
if (isCover && name === 'img'){
novel.name = attribs['alt'];
novel.cover = baseUrl + attribs['src'];
}
if (name === 'div' && attribs['class'] === 'moreless'){
isSummary = true;
}
if (name === 'li' && attribs['title']?.includes('Original status')){
isStatus = true;
}
if (name === 'input' && attribs['name'] === 'newsid'){
novelId = attribs['value']
}
},
onopentagname(name){
if (isSummary && name === 'br'){
novel.summary += `/n`;
}
if (isStatus && name === 'a'){
isStatusText = true;
}
if (isGenres && name === 'a'){
isGenresText = true;
}
},
onattribute(name,value){
if (name === 'itemprop' && value === 'creator'){
isAuthor = true;
}
if (name === 'id' && value === 'mc-fs-genre'){
isGenres = true;
}
},
ontext(data){
if(isAuthor){
novel.author = data;
}
if(isSummary){
novel.summary += data;
}
if(isStatusText){
novel.status = data === 'Ongoing'
? 'Ongoing'
: 'Completed';
}
if(isGenresText){
genreArray.push(data);
}
},
onclosetag(name){
if(name === 'a'){
isCover = false;
isAuthor = false;
isStatusText = false;
isGenresText = false;
isStatus = false;
}
if(name === 'div'){
isSummary = false;
isGenres = false;
}
}
})
parser.write(html);
parser.end();
novel.genres = genreArray.join(', ');
const chaptersHtmlToString = await fetchApi(chapterListUrl).then(r => const chapterListUrl = this.site + '/chapters/' + novelId + '/';
const chaptersHtml = await fetchApi(chapterListUrl).then(r =>
r.text(), r.text(),
); );
const pageCheerio = parseHTML(chaptersHtmlToString); let dataJson: {
pages_count: string,
chapters: ChapterEntry[]
} = { pages_count: '', chapters : [] };
let isScript = false;
const parser2 = new Parser ({
ontext(data){
if(isScript){
if (data.includes('window.__DATA__ =')){
dataJson = JSON.parse(
data.replace('window.__DATA__ =', '')
);
}
}
},
onclosetag(name){
if(name === 'main'){
isScript = true;
}
if (name === 'script'){
isScript = false;
}
},
})
parser2.write(chaptersHtml);
parser2.end();
const json = pageCheerio('#dle-content main').next().html()!; novel.totalPages = Number(dataJson.pages_count);
const data = JSON.parse(json.replace('window.__DATA__ =', '')); novel.chapters = this.parseChapters(dataJson);
novel.totalPages = Number(data.pages_count);
novel.chapters = this.parseChapters(data);
const latestChapterUrl = data.chapters[0].link; const latestChapterUrl = dataJson.chapters[0].link;
const latestChapterName = data.chapters[0].title; const latestChapterName = dataJson.chapters[0].title;
const latestChapterDate = data.chapters[0].date; const latestChapterDate = dataJson.chapters[0].date;
novel.latestChapter = latestChapterUrl novel.latestChapter = latestChapterUrl
? { ? {
@@ -138,10 +240,33 @@ class RanobesPlugin implements Plugin.PagePlugin {
const firstUrl = this.site + '/chapters' + pagePath.replace('novels/', ''); const firstUrl = this.site + '/chapters' + pagePath.replace('novels/', '');
const pageUrl = firstUrl + '/page/' + page; const pageUrl = firstUrl + '/page/' + page;
const pageBody = await fetchApi(pageUrl).then(r => r.text()); const pageBody = await fetchApi(pageUrl).then(r => r.text());
const loadedCheerio = parseHTML(pageBody); let isScript = false;
const json = loadedCheerio('#dle-content main').next().html()!; let dataJson: {
const data = JSON.parse(json.replace('window.__DATA__ =', '')); pages_count: string,
const chapters = this.parseChapters(data); chapters: ChapterEntry[]
} = { pages_count: '', chapters : [] };
const parser = new Parser({
ontext(data){
if(isScript){
if (data.includes('window.__DATA__ =')){
dataJson = JSON.parse(
data.replace('window.__DATA__ =', '')
);
}
}
},
onclosetag(name){
if(name === 'main'){
isScript = true;
}
if (name === 'script'){
isScript = false;
}
}
})
parser.write(pageBody);
parser.end();
const chapters = this.parseChapters(dataJson);
return { return {
chapters, chapters,
}; };
@@ -149,11 +274,99 @@ class RanobesPlugin implements Plugin.PagePlugin {
async parseChapter(chapterPath: string): Promise<string> { async parseChapter(chapterPath: string): Promise<string> {
const result = await fetchApi(this.site + chapterPath); const result = await fetchApi(this.site + chapterPath);
const body = await result.text(); const html = await result.text();
let chapterText = '';
const loadedCheerio = parseHTML(body); let isChapter = false;
let isPtag = false;
const chapterText = loadedCheerio('#arrticle').html() || ''; let isStyleText = false;
const parser = new Parser({
onopentag(name,attribs){
if (isChapter && name === 'div'){
let stylediv = attribs['style']
if(stylediv){
chapterText += `<div style="${stylediv}">`
isStyleText = true;
} else {
chapterText += `<div>`
}
}
if (isChapter && name === 'table'){
let w = attribs['width'];
if(w) {
chapterText += `<table width="${w}">`;
} else {
chapterText += `<table>`;
}
}
if (isChapter && name === 'tbody'){
chapterText += `<tbody>`;
}
if (isChapter && name === 'tr'){
chapterText += `<tr>`;
}
if (isChapter && name === 'td'){
let w1 = attribs['width'];
if(w1) {
chapterText += `<td width="${w1}">`;
} else {
chapterText += `<td>`;
}
}
},
onattribute(name, value){
if (name === 'id' && value === 'arrticle'){
isChapter = true;
}
if (name === 'class' && value === 'category grey ellipses'){
isChapter = false;
isPtag = false;
}
},
onopentagname(name){
if (isChapter && name === 'p'){
chapterText += '<p>';
isPtag = true;
if(isStyleText){
isStyleText = false;
}
}
if (isChapter && name === 'br'){
chapterText += `<br>`;
}
},
ontext(data){
if(isPtag){
chapterText += data;
}
if(isStyleText){
chapterText += data;
}
},
onclosetag(name){
if (name === 'p'){
isPtag = false;
chapterText += '</p>'
}
if (isChapter && name === 'td'){
chapterText += `</td>`;
}
if (isChapter && name === 'tr'){
chapterText += `</tr>`;
}
if (isChapter && name === 'tbody'){
chapterText += `</tbody>`;
}
if (isChapter && name === 'table'){
chapterText += `</table>`;
}
if (isChapter && name === 'div'){
isStyleText = false;
chapterText += `</div>`
}
}
})
parser.write(html)
parser.end();
return chapterText; return chapterText;
} }
@@ -163,15 +376,21 @@ class RanobesPlugin implements Plugin.PagePlugin {
page: number, page: number,
): Promise<Plugin.NovelItem[]> { ): Promise<Plugin.NovelItem[]> {
let link = `${this.site}/search/${searchTerm}/page/${page}`; let link = `${this.site}/search/${searchTerm}/page/${page}`;
const body = await fetchApi(link).then(r => r.text()); const body = await fetchApi(link).then(r => r.text());
const loadedCheerio = parseHTML(body); return this.parseNovels(body);
return this.parseNovels(loadedCheerio);
} }
async fetchImage(url: string): Promise<string | undefined> { async fetchImage(url: string): Promise<string | undefined> {
return await fetchFile(url); return await fetchFile(url);
} }
} }
export default new RanobesPlugin(); export default new RanobesPlugin();
interface ChapterEntry {
id: number;
title: string;
date: string;
link: string;
}
+274 -60
View File
@@ -1,34 +1,56 @@
import { CheerioAPI, load as parseHTML } from 'cheerio'; import { Parser } from 'htmlparser2';
import { fetchApi, fetchFile } from '@libs/fetch'; import { fetchApi, fetchFile } from '@libs/fetch';
import { Plugin } from '@typings/plugin'; import { Plugin } from '@typings/plugin';
import { Filters, FilterTypes } from '@libs/filterInputs'; import { Filters, FilterTypes } from '@libs/filterInputs';
import { NovelStatus } from '@libs/novelStatus';
class RoyalRoad implements Plugin.PluginBase { class RoyalRoad implements Plugin.PluginBase {
id = 'royalroad'; id = 'royalroad';
name = 'Royal Road'; name = 'Royal Road';
version = '1.0.1'; version = '2.0.0';
icon = 'src/en/royalroad/icon.png'; icon = 'src/en/royalroad/icon.png';
site = 'https://www.royalroad.com/'; site = 'https://www.royalroad.com/';
parseNovels(loadedCheerio: CheerioAPI) { parseNovels(html: string) {
const novels: Plugin.NovelItem[] = []; const novels: Plugin.NovelItem[] = [];
let tempNovel = {} as Plugin.NovelItem;
loadedCheerio('.fiction-list-item').each((idx, ele) => { tempNovel.name = '';
const novelName = loadedCheerio(ele).find('a.bold').text(); let isParsingNovel = false;
const novelCover = loadedCheerio(ele).find('img').attr('src'); let isNovelName = false;
const novelUrl = loadedCheerio(ele).find('a.bold').attr('href')?.slice(1); const parser = new Parser({
onopentag(name, attribs){
if (!novelUrl) return; if (attribs['class']?.includes('fiction-list-item')){
isParsingNovel = true;
const novel = { }
name: novelName, if (isParsingNovel){
cover: novelCover, if (name === 'a' && attribs['class']?.includes('bold')){
path: novelUrl, tempNovel.path = attribs['href'].slice(1);
}; isNovelName = true;
}
novels.push(novel); if (name === 'img'){
}); tempNovel.cover = attribs['src'];
}
if (tempNovel.path && tempNovel.name){
novels.push(tempNovel);
tempNovel = {} as Plugin.NovelItem;
tempNovel.name = '';
}
}
},
ontext(data){
if(isNovelName){
tempNovel.name += data;
}
},
onclosetag(name){
if(name === 'h2'){
isNovelName = false;
isParsingNovel = false;
}
}
})
parser.write(html);
parser.end();
return novels; return novels;
} }
@@ -45,42 +67,133 @@ class RoyalRoad implements Plugin.PluginBase {
const body = await fetchApi(link).then(r => r.text()); const body = await fetchApi(link).then(r => r.text());
const loadedCheerio = parseHTML(body); return this.parseNovels(body);
return this.parseNovels(loadedCheerio);
} }
async parseNovel(novelPath: string): Promise<Plugin.SourceNovel> { async parseNovel(novelPath: string): Promise<Plugin.SourceNovel> {
const result = await fetchApi(this.site + novelPath); const result = await fetchApi(this.site + novelPath);
const body = await result.text(); const html = await result.text();
const loadedCheerio = parseHTML(body);
const novel: Plugin.SourceNovel = { const novel: Plugin.SourceNovel = {
path: novelPath, path: novelPath,
name: loadedCheerio('h1').text() || 'Untitled', name: '',
cover: loadedCheerio('.cover-art-container img').attr('src'), summary: '',
author: loadedCheerio('h4 a').text(),
summary: loadedCheerio('.description').text().trim(),
status: loadedCheerio('span.label-sm:last').text().trim(),
chapters: [], chapters: [],
}; }
let isNovelName = false;
novel.genres = loadedCheerio('span.tags a') let isAuthorName = false;
.map((i, el) => loadedCheerio(el).text()) let isDescription = false;
.toArray() let isH4 = false;
.join(','); let isSpan = 0
let isTags = false;
const chapterJson = JSON.parse( let isGenres = false;
loadedCheerio('script:contains("window.chapters")') let genreArray: string[] = []
.html() let isFooter = false;
?.match(/window.chapters = (.+])(?=;)/)![1] || '', let isScript = false;
); let chapterJson: ChapterEntry[] = [];
let volumeJson: VolumeEntry[] = [];
const volumeJson = JSON.parse( const parser = new Parser ({
loadedCheerio('script:contains("window.chapters")') onopentag(name,attribs){
.html() if (name === 'img' && attribs['class']?.includes('thumbnail')){
?.match(/window.volumes = (.+])(?=;)/)![1] || '', novel.cover = attribs['src'];
); }
if (name === 'span' && attribs['class']?.includes('label-sm')){
isSpan++;
}
if (name === 'span' && attribs['class']?.includes('tags')){
isTags = true;
}
},
onopentagname(name){
if (name === 'h1'){
isNovelName = true;
}
if (isH4 && name === 'a'){
isAuthorName = true;
}
if(isTags && name === 'a'){
isGenres = true;
}
if (name === 'label'){
isDescription = false;
isTags = false;
}
if (isFooter && name === 'script'){
isScript = true;
}
},
onattribute(name,value){
if (name === 'class' && value === 'description'){
isDescription = true;
}
if (name === 'class' && value === 'page-footer footer'){
isFooter = true;
}
},
ontext(data){
if (isNovelName){
novel.name = data;
}
if (isAuthorName){
novel.author = data;
isAuthorName = false;
}
if (isDescription){
novel.summary += data;
}
if (isSpan === 2){
novel.status = data.trim();
isSpan++;
}
if (isGenres){
genreArray.push(data);
}
if (isScript){
if (data.includes('window.chapters =')){
chapterJson = JSON.parse(
data.match(/window.chapters = (.+])(?=;)/)![1]
)
volumeJson = JSON.parse(
data.match(/window.volumes = (.+])(?=;)/)![1]
)
}
}
},
onclosetag(name){
if (name === 'h1'){
isNovelName = false;
isH4 = true;
}
if (name === 'h4'){
isH4 = false;
}
if (name === 'a'){
isGenres = false;
}
if (name === 'script'){
isScript = false;
}
if (name === 'body'){
isFooter = false;
}
}
})
parser.write(html);
parser.end();
novel.summary = novel.summary?.trim();
novel.genres = genreArray.join(', ');
switch(novel.status){
case 'ONGOING':
novel.status = NovelStatus.Ongoing;
break;
case 'HIATUS':
novel.status = NovelStatus.OnHiatus;
break;
case 'COMPLETED':
novel.status = NovelStatus.Completed;
break;
default:
novel.status = NovelStatus.Unknown;
}
const chapter: Plugin.ChapterItem[] = chapterJson.map( const chapter: Plugin.ChapterItem[] = chapterJson.map(
(chapter: ChapterEntry) => { (chapter: ChapterEntry) => {
@@ -92,7 +205,7 @@ class RoyalRoad implements Plugin.PluginBase {
path: chapter.url.slice(1), path: chapter.url.slice(1),
releaseTime: chapter.date, releaseTime: chapter.date,
chapterNumber: chapter?.order, chapterNumber: chapter?.order,
page: matchingVolume ? matchingVolume.title : null, page: matchingVolume?.title,
}; };
}, },
); );
@@ -103,11 +216,115 @@ class RoyalRoad implements Plugin.PluginBase {
async parseChapter(chapterPath: string): Promise<string> { async parseChapter(chapterPath: string): Promise<string> {
const result = await fetchApi(this.site + chapterPath); const result = await fetchApi(this.site + chapterPath);
const body = await result.text(); const html = await result.text();
let chapterText = '';
const loadedCheerio = parseHTML(body); let isChapter = false;
let isPtag = false;
const chapterText = loadedCheerio('.chapter-content').html() || ''; let isStyle = false;
let isStyleText = false;
let styles = '';
const parser = new Parser({
onopentag(name,attribs){
if (isChapter && name === 'div'){
let stylediv = attribs['style']
if(stylediv){
chapterText += `<div style="${stylediv}">`
isStyleText = true;
} else {
chapterText += `<div>`
}
}
if (isChapter && name === 'table'){
let w = attribs['width'];
if(w) {
chapterText += `<table width="${w}">`;
} else {
chapterText += `<table>`;
}
}
if (isChapter && name === 'tbody'){
chapterText += `<tbody>`;
}
if (isChapter && name === 'tr'){
chapterText += `<tr>`;
}
if (isChapter && name === 'td'){
let w1 = attribs['width'];
if(w1) {
chapterText += `<td width="${w1}">`;
} else {
chapterText += `<td>`;
}
}
},
onattribute(name, value){
if (name === 'class' && value === 'chapter-inner chapter-content'){
isChapter = true;
}
if (name === 'class' && value === 'portlet light t-center-3'){
isChapter = false;
isPtag = false;
}
if (name === 'class' && value === styles){
isPtag = false;
}
},
onopentagname(name){
if (isChapter && name === 'p'){
chapterText += '<p>';
isPtag = true;
if(isStyleText){
isStyleText = false;
}
}
if (name === 'style'){
isStyle = true;
}
if (isChapter && name === 'br'){
chapterText += `<br>`;
}
},
ontext(data){
if(isPtag){
chapterText += data;
}
if(isStyleText){
chapterText += data;
}
if (isStyle){
if (data.includes('display: none;')){
styles = data.match(/\.(.*)\{/)![1];
}
}
},
onclosetag(name){
if (name === 'p'){
isPtag = false;
chapterText += '</p>'
}
if (name === 'style'){
isStyle = false;
}
if (isChapter && name === 'td'){
chapterText += `</td>`;
}
if (isChapter && name === 'tr'){
chapterText += `</tr>`;
}
if (isChapter && name === 'tbody'){
chapterText += `</tbody>`;
}
if (isChapter && name === 'table'){
chapterText += `</table>`;
}
if (isChapter && name === 'div'){
isStyleText = false;
chapterText += `</div>`
}
}
})
parser.write(html)
parser.end();
return chapterText; return chapterText;
} }
@@ -118,11 +335,8 @@ class RoyalRoad implements Plugin.PluginBase {
): Promise<Plugin.NovelItem[]> { ): Promise<Plugin.NovelItem[]> {
const searchUrl = `${this.site}fictions/search?page=${page}&title=${searchTerm}`; const searchUrl = `${this.site}fictions/search?page=${page}&title=${searchTerm}`;
const result = await fetchApi(searchUrl); const body = await fetchApi(searchUrl).then(r => r.text());
const body = await result.text(); return this.parseNovels(body);
const loadedCheerio = parseHTML(body);
return this.parseNovels(loadedCheerio);
} }
async fetchImage(url: string): Promise<string | undefined> { async fetchImage(url: string): Promise<string | undefined> {