parser2 sources (royalroad, ranobes, lightnovelpub, lnmtl) (#865)
* lnmtl parser2 * add royalroad, change version * ranobes parser * lightnovelpub * cleanup
This commit is contained in:
K1ngfish3r
+276
-62
@@ -1,34 +1,56 @@
|
||||
import { CheerioAPI, load as parseHTML } from 'cheerio';
|
||||
import { Parser } from 'htmlparser2';
|
||||
import { fetchApi, fetchFile } from '@libs/fetch';
|
||||
import { Plugin } from '@typings/plugin';
|
||||
import { Filters, FilterTypes } from '@libs/filterInputs';
|
||||
import { NovelStatus } from '@libs/novelStatus';
|
||||
|
||||
class RoyalRoad implements Plugin.PluginBase {
|
||||
id = 'royalroad';
|
||||
name = 'Royal Road';
|
||||
version = '1.0.1';
|
||||
version = '2.0.0';
|
||||
icon = 'src/en/royalroad/icon.png';
|
||||
site = 'https://www.royalroad.com/';
|
||||
|
||||
parseNovels(loadedCheerio: CheerioAPI) {
|
||||
parseNovels(html: string) {
|
||||
const novels: Plugin.NovelItem[] = [];
|
||||
|
||||
loadedCheerio('.fiction-list-item').each((idx, ele) => {
|
||||
const novelName = loadedCheerio(ele).find('a.bold').text();
|
||||
const novelCover = loadedCheerio(ele).find('img').attr('src');
|
||||
const novelUrl = loadedCheerio(ele).find('a.bold').attr('href')?.slice(1);
|
||||
|
||||
if (!novelUrl) return;
|
||||
|
||||
const novel = {
|
||||
name: novelName,
|
||||
cover: novelCover,
|
||||
path: novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
let tempNovel = {} as Plugin.NovelItem;
|
||||
tempNovel.name = '';
|
||||
let isParsingNovel = false;
|
||||
let isNovelName = false;
|
||||
const parser = new Parser({
|
||||
onopentag(name, attribs){
|
||||
if (attribs['class']?.includes('fiction-list-item')){
|
||||
isParsingNovel = true;
|
||||
}
|
||||
if (isParsingNovel){
|
||||
if (name === 'a' && attribs['class']?.includes('bold')){
|
||||
tempNovel.path = attribs['href'].slice(1);
|
||||
isNovelName = true;
|
||||
}
|
||||
if (name === 'img'){
|
||||
tempNovel.cover = attribs['src'];
|
||||
}
|
||||
if (tempNovel.path && tempNovel.name){
|
||||
novels.push(tempNovel);
|
||||
tempNovel = {} as Plugin.NovelItem;
|
||||
tempNovel.name = '';
|
||||
}
|
||||
}
|
||||
},
|
||||
ontext(data){
|
||||
if(isNovelName){
|
||||
tempNovel.name += data;
|
||||
}
|
||||
},
|
||||
onclosetag(name){
|
||||
if(name === 'h2'){
|
||||
isNovelName = false;
|
||||
isParsingNovel = false;
|
||||
}
|
||||
}
|
||||
})
|
||||
parser.write(html);
|
||||
parser.end();
|
||||
return novels;
|
||||
}
|
||||
|
||||
@@ -42,45 +64,136 @@ class RoyalRoad implements Plugin.PluginBase {
|
||||
link += `?page=${page}`;
|
||||
|
||||
if (filters.genre.value !== '') link += `&genre=${filters.genre.value}`;
|
||||
|
||||
|
||||
const body = await fetchApi(link).then(r => r.text());
|
||||
|
||||
const loadedCheerio = parseHTML(body);
|
||||
return this.parseNovels(loadedCheerio);
|
||||
|
||||
return this.parseNovels(body);
|
||||
}
|
||||
|
||||
async parseNovel(novelPath: string): Promise<Plugin.SourceNovel> {
|
||||
const result = await fetchApi(this.site + novelPath);
|
||||
const body = await result.text();
|
||||
|
||||
const loadedCheerio = parseHTML(body);
|
||||
|
||||
const html = await result.text();
|
||||
const novel: Plugin.SourceNovel = {
|
||||
path: novelPath,
|
||||
name: loadedCheerio('h1').text() || 'Untitled',
|
||||
cover: loadedCheerio('.cover-art-container img').attr('src'),
|
||||
author: loadedCheerio('h4 a').text(),
|
||||
summary: loadedCheerio('.description').text().trim(),
|
||||
status: loadedCheerio('span.label-sm:last').text().trim(),
|
||||
name: '',
|
||||
summary: '',
|
||||
chapters: [],
|
||||
};
|
||||
|
||||
novel.genres = loadedCheerio('span.tags a')
|
||||
.map((i, el) => loadedCheerio(el).text())
|
||||
.toArray()
|
||||
.join(',');
|
||||
|
||||
const chapterJson = JSON.parse(
|
||||
loadedCheerio('script:contains("window.chapters")')
|
||||
.html()
|
||||
?.match(/window.chapters = (.+])(?=;)/)![1] || '',
|
||||
);
|
||||
|
||||
const volumeJson = JSON.parse(
|
||||
loadedCheerio('script:contains("window.chapters")')
|
||||
.html()
|
||||
?.match(/window.volumes = (.+])(?=;)/)![1] || '',
|
||||
);
|
||||
}
|
||||
let isNovelName = false;
|
||||
let isAuthorName = false;
|
||||
let isDescription = false;
|
||||
let isH4 = false;
|
||||
let isSpan = 0
|
||||
let isTags = false;
|
||||
let isGenres = false;
|
||||
let genreArray: string[] = []
|
||||
let isFooter = false;
|
||||
let isScript = false;
|
||||
let chapterJson: ChapterEntry[] = [];
|
||||
let volumeJson: VolumeEntry[] = [];
|
||||
const parser = new Parser ({
|
||||
onopentag(name,attribs){
|
||||
if (name === 'img' && attribs['class']?.includes('thumbnail')){
|
||||
novel.cover = attribs['src'];
|
||||
}
|
||||
if (name === 'span' && attribs['class']?.includes('label-sm')){
|
||||
isSpan++;
|
||||
}
|
||||
if (name === 'span' && attribs['class']?.includes('tags')){
|
||||
isTags = true;
|
||||
}
|
||||
},
|
||||
onopentagname(name){
|
||||
if (name === 'h1'){
|
||||
isNovelName = true;
|
||||
}
|
||||
if (isH4 && name === 'a'){
|
||||
isAuthorName = true;
|
||||
}
|
||||
if(isTags && name === 'a'){
|
||||
isGenres = true;
|
||||
}
|
||||
if (name === 'label'){
|
||||
isDescription = false;
|
||||
isTags = false;
|
||||
}
|
||||
if (isFooter && name === 'script'){
|
||||
isScript = true;
|
||||
}
|
||||
},
|
||||
onattribute(name,value){
|
||||
if (name === 'class' && value === 'description'){
|
||||
isDescription = true;
|
||||
}
|
||||
if (name === 'class' && value === 'page-footer footer'){
|
||||
isFooter = true;
|
||||
}
|
||||
},
|
||||
ontext(data){
|
||||
if (isNovelName){
|
||||
novel.name = data;
|
||||
}
|
||||
if (isAuthorName){
|
||||
novel.author = data;
|
||||
isAuthorName = false;
|
||||
}
|
||||
if (isDescription){
|
||||
novel.summary += data;
|
||||
}
|
||||
if (isSpan === 2){
|
||||
novel.status = data.trim();
|
||||
isSpan++;
|
||||
}
|
||||
if (isGenres){
|
||||
genreArray.push(data);
|
||||
}
|
||||
if (isScript){
|
||||
if (data.includes('window.chapters =')){
|
||||
chapterJson = JSON.parse(
|
||||
data.match(/window.chapters = (.+])(?=;)/)![1]
|
||||
)
|
||||
volumeJson = JSON.parse(
|
||||
data.match(/window.volumes = (.+])(?=;)/)![1]
|
||||
)
|
||||
}
|
||||
}
|
||||
},
|
||||
onclosetag(name){
|
||||
if (name === 'h1'){
|
||||
isNovelName = false;
|
||||
isH4 = true;
|
||||
}
|
||||
if (name === 'h4'){
|
||||
isH4 = false;
|
||||
}
|
||||
if (name === 'a'){
|
||||
isGenres = false;
|
||||
}
|
||||
if (name === 'script'){
|
||||
isScript = false;
|
||||
}
|
||||
if (name === 'body'){
|
||||
isFooter = false;
|
||||
}
|
||||
}
|
||||
})
|
||||
parser.write(html);
|
||||
parser.end();
|
||||
novel.summary = novel.summary?.trim();
|
||||
novel.genres = genreArray.join(', ');
|
||||
switch(novel.status){
|
||||
case 'ONGOING':
|
||||
novel.status = NovelStatus.Ongoing;
|
||||
break;
|
||||
case 'HIATUS':
|
||||
novel.status = NovelStatus.OnHiatus;
|
||||
break;
|
||||
case 'COMPLETED':
|
||||
novel.status = NovelStatus.Completed;
|
||||
break;
|
||||
default:
|
||||
novel.status = NovelStatus.Unknown;
|
||||
}
|
||||
|
||||
const chapter: Plugin.ChapterItem[] = chapterJson.map(
|
||||
(chapter: ChapterEntry) => {
|
||||
@@ -92,7 +205,7 @@ class RoyalRoad implements Plugin.PluginBase {
|
||||
path: chapter.url.slice(1),
|
||||
releaseTime: chapter.date,
|
||||
chapterNumber: chapter?.order,
|
||||
page: matchingVolume ? matchingVolume.title : null,
|
||||
page: matchingVolume?.title,
|
||||
};
|
||||
},
|
||||
);
|
||||
@@ -103,11 +216,115 @@ class RoyalRoad implements Plugin.PluginBase {
|
||||
|
||||
async parseChapter(chapterPath: string): Promise<string> {
|
||||
const result = await fetchApi(this.site + chapterPath);
|
||||
const body = await result.text();
|
||||
|
||||
const loadedCheerio = parseHTML(body);
|
||||
|
||||
const chapterText = loadedCheerio('.chapter-content').html() || '';
|
||||
const html = await result.text();
|
||||
let chapterText = '';
|
||||
let isChapter = false;
|
||||
let isPtag = false;
|
||||
let isStyle = false;
|
||||
let isStyleText = false;
|
||||
let styles = '';
|
||||
const parser = new Parser({
|
||||
onopentag(name,attribs){
|
||||
if (isChapter && name === 'div'){
|
||||
let stylediv = attribs['style']
|
||||
if(stylediv){
|
||||
chapterText += `<div style="${stylediv}">`
|
||||
isStyleText = true;
|
||||
} else {
|
||||
chapterText += `<div>`
|
||||
}
|
||||
}
|
||||
if (isChapter && name === 'table'){
|
||||
let w = attribs['width'];
|
||||
if(w) {
|
||||
chapterText += `<table width="${w}">`;
|
||||
} else {
|
||||
chapterText += `<table>`;
|
||||
}
|
||||
}
|
||||
if (isChapter && name === 'tbody'){
|
||||
chapterText += `<tbody>`;
|
||||
}
|
||||
if (isChapter && name === 'tr'){
|
||||
chapterText += `<tr>`;
|
||||
}
|
||||
if (isChapter && name === 'td'){
|
||||
let w1 = attribs['width'];
|
||||
if(w1) {
|
||||
chapterText += `<td width="${w1}">`;
|
||||
} else {
|
||||
chapterText += `<td>`;
|
||||
}
|
||||
}
|
||||
},
|
||||
onattribute(name, value){
|
||||
if (name === 'class' && value === 'chapter-inner chapter-content'){
|
||||
isChapter = true;
|
||||
}
|
||||
if (name === 'class' && value === 'portlet light t-center-3'){
|
||||
isChapter = false;
|
||||
isPtag = false;
|
||||
}
|
||||
if (name === 'class' && value === styles){
|
||||
isPtag = false;
|
||||
}
|
||||
},
|
||||
onopentagname(name){
|
||||
if (isChapter && name === 'p'){
|
||||
chapterText += '<p>';
|
||||
isPtag = true;
|
||||
if(isStyleText){
|
||||
isStyleText = false;
|
||||
}
|
||||
}
|
||||
if (name === 'style'){
|
||||
isStyle = true;
|
||||
}
|
||||
if (isChapter && name === 'br'){
|
||||
chapterText += `<br>`;
|
||||
}
|
||||
},
|
||||
ontext(data){
|
||||
if(isPtag){
|
||||
chapterText += data;
|
||||
}
|
||||
if(isStyleText){
|
||||
chapterText += data;
|
||||
}
|
||||
if (isStyle){
|
||||
if (data.includes('display: none;')){
|
||||
styles = data.match(/\.(.*)\{/)![1];
|
||||
}
|
||||
}
|
||||
},
|
||||
onclosetag(name){
|
||||
if (name === 'p'){
|
||||
isPtag = false;
|
||||
chapterText += '</p>'
|
||||
}
|
||||
if (name === 'style'){
|
||||
isStyle = false;
|
||||
}
|
||||
if (isChapter && name === 'td'){
|
||||
chapterText += `</td>`;
|
||||
}
|
||||
if (isChapter && name === 'tr'){
|
||||
chapterText += `</tr>`;
|
||||
}
|
||||
if (isChapter && name === 'tbody'){
|
||||
chapterText += `</tbody>`;
|
||||
}
|
||||
if (isChapter && name === 'table'){
|
||||
chapterText += `</table>`;
|
||||
}
|
||||
if (isChapter && name === 'div'){
|
||||
isStyleText = false;
|
||||
chapterText += `</div>`
|
||||
}
|
||||
}
|
||||
})
|
||||
parser.write(html)
|
||||
parser.end();
|
||||
|
||||
return chapterText;
|
||||
}
|
||||
@@ -118,11 +335,8 @@ class RoyalRoad implements Plugin.PluginBase {
|
||||
): Promise<Plugin.NovelItem[]> {
|
||||
const searchUrl = `${this.site}fictions/search?page=${page}&title=${searchTerm}`;
|
||||
|
||||
const result = await fetchApi(searchUrl);
|
||||
const body = await result.text();
|
||||
|
||||
const loadedCheerio = parseHTML(body);
|
||||
return this.parseNovels(loadedCheerio);
|
||||
const body = await fetchApi(searchUrl).then(r => r.text());
|
||||
return this.parseNovels(body);
|
||||
}
|
||||
|
||||
async fetchImage(url: string): Promise<string | undefined> {
|
||||
|
||||
Reference in New Issue
Block a user