Multisrc ReadNovelfull: Add anf.net, libread and fwn.com & convert to htmlparser2 (#1672)
* postsearch * small * i let gemini do a thing * stateStack and anf.net * parseNovel * fix parseNovels searchNovels and rewrite parseNovel noAjax * only parseChapter remains * confirm ulmetainfoli * Update template.ts * fwn * cut * paste * remove extra * parseChapter
This commit is contained in:
K1ngfish3r
@@ -1,4 +1,4 @@
|
||||
import { CheerioAPI, load } from 'cheerio';
|
||||
import { Parser } from 'htmlparser2';
|
||||
import { fetchApi } from '@libs/fetch';
|
||||
import { Plugin } from '@typings/plugin';
|
||||
import { NovelStatus } from '@libs/novelStatus';
|
||||
@@ -18,6 +18,10 @@ type ReadNovelFullOptions = {
|
||||
genreKey?: string;
|
||||
langParam?: string;
|
||||
urlLangCode?: string;
|
||||
searchKey?: string;
|
||||
postSearch?: boolean;
|
||||
noAjax?: boolean;
|
||||
noPages?: string[];
|
||||
};
|
||||
|
||||
export type ReadNovelFullMetadata = {
|
||||
@@ -43,56 +47,93 @@ class ReadNovelFullPlugin implements Plugin.PluginBase {
|
||||
this.icon = `multisrc/readnovelfull/${metadata.id.toLowerCase()}/icon.png`;
|
||||
this.site = metadata.sourceSite;
|
||||
const versionIncrements = metadata.options?.versionIncrements || 0;
|
||||
this.version = `2.0.${1 + versionIncrements}`;
|
||||
this.version = `2.1.${0 + versionIncrements}`;
|
||||
this.options = metadata.options;
|
||||
this.filters = metadata.filters;
|
||||
}
|
||||
|
||||
async getCheerio(url: string, search: boolean): Promise<CheerioAPI> {
|
||||
const r = await fetchApi(url);
|
||||
if (!r.ok && search != true)
|
||||
throw new Error(
|
||||
'Could not reach site (' + r.status + ') try to open in webview.',
|
||||
);
|
||||
const $ = load(await r.text());
|
||||
lastSearch: number | null = null;
|
||||
searchInterval = 3400;
|
||||
|
||||
return $;
|
||||
async sleep(ms: number) {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
parseNovels($: CheerioAPI): Plugin.NovelItem[] {
|
||||
const baseUrl = this.site;
|
||||
parseNovels(html: string) {
|
||||
const novels: Plugin.NovelItem[] = [];
|
||||
let tempNovel: Partial<Plugin.NovelItem> = {};
|
||||
let depth: number;
|
||||
|
||||
const novels = $('.archive .list-novel .row, .archive .list-truyen .row')
|
||||
.toArray()
|
||||
.map(el => {
|
||||
const $el = $(el);
|
||||
const stateStack: ParsingState[] = [ParsingState.Idle];
|
||||
const currentState = () => stateStack[stateStack.length - 1];
|
||||
const pushState = (state: ParsingState) => stateStack.push(state);
|
||||
const popState = () =>
|
||||
stateStack.length > 1 ? stateStack.pop() : currentState();
|
||||
|
||||
const novelName = $el.find('h3').text()?.trim();
|
||||
const rawHref = $el.find('a').attr('href');
|
||||
const rawCover =
|
||||
$el.find('img').attr('src') ??
|
||||
$el.find('img').attr('data-cfsrc') ??
|
||||
$el.find('img').attr('data-src');
|
||||
|
||||
if (!novelName || !rawHref) {
|
||||
return null;
|
||||
const parser = new Parser({
|
||||
onopentag: (name, attribs) => {
|
||||
const state = currentState();
|
||||
if (
|
||||
attribs.class?.includes('archive') ||
|
||||
attribs.class === 'col-content'
|
||||
) {
|
||||
pushState(ParsingState.NovelList);
|
||||
depth = 0;
|
||||
}
|
||||
|
||||
const novelUrlObject = new URL(rawHref, baseUrl);
|
||||
const novelPath = novelUrlObject.pathname.slice(1);
|
||||
if (
|
||||
state !== ParsingState.NovelList &&
|
||||
state !== ParsingState.NovelName
|
||||
)
|
||||
return;
|
||||
|
||||
let novelCover: string | undefined;
|
||||
if (rawCover) {
|
||||
novelCover = new URL(rawCover, baseUrl).href;
|
||||
switch (name) {
|
||||
case 'img':
|
||||
const cover = attribs['data-src'] || attribs.src;
|
||||
if (cover) {
|
||||
tempNovel.cover = new URL(cover, this.site).href;
|
||||
}
|
||||
break;
|
||||
case 'h3':
|
||||
if (state === ParsingState.NovelList) {
|
||||
pushState(ParsingState.NovelName);
|
||||
}
|
||||
break;
|
||||
case 'a':
|
||||
if (state === ParsingState.NovelName) {
|
||||
const href = attribs.href;
|
||||
if (href) {
|
||||
tempNovel.path = new URL(href, this.site).pathname.substring(1);
|
||||
tempNovel.name = attribs.title;
|
||||
}
|
||||
}
|
||||
break;
|
||||
case 'div':
|
||||
depth++;
|
||||
break;
|
||||
default:
|
||||
return;
|
||||
}
|
||||
},
|
||||
|
||||
return {
|
||||
name: novelName,
|
||||
path: novelPath,
|
||||
cover: novelCover,
|
||||
};
|
||||
})
|
||||
.filter(Boolean);
|
||||
onclosetag: name => {
|
||||
const state = currentState();
|
||||
if (name === 'a' && state === ParsingState.NovelName) {
|
||||
if (tempNovel.name && tempNovel.path) {
|
||||
novels.push({ ...tempNovel } as Plugin.NovelItem);
|
||||
}
|
||||
tempNovel = {};
|
||||
popState();
|
||||
}
|
||||
if (name === 'div' && state === ParsingState.NovelList) {
|
||||
depth--;
|
||||
if (depth < 0) popState();
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
parser.write(html);
|
||||
parser.end();
|
||||
|
||||
return novels;
|
||||
}
|
||||
@@ -101,174 +142,606 @@ class ReadNovelFullPlugin implements Plugin.PluginBase {
|
||||
pageNo: number,
|
||||
{ filters, showLatestNovels }: Plugin.PopularNovelsOptions,
|
||||
): Promise<Plugin.NovelItem[]> {
|
||||
let url = '';
|
||||
const pageParam = this.options.pageParam || 'page';
|
||||
const {
|
||||
pageParam = 'page',
|
||||
novelListing,
|
||||
typeParam = 'type',
|
||||
latestPage,
|
||||
genreParam = 'category_novel',
|
||||
genreKey = 'id',
|
||||
langParam,
|
||||
urlLangCode,
|
||||
noPages = [],
|
||||
} = this.options;
|
||||
|
||||
if (this.options.novelListing) {
|
||||
// Skip Pagination for FWN & LR
|
||||
if (
|
||||
pageNo !== 1 &&
|
||||
!showLatestNovels &&
|
||||
!filters.genres.value.length &&
|
||||
noPages.length > 0 &&
|
||||
noPages.includes(filters.type.value)
|
||||
) {
|
||||
return [];
|
||||
}
|
||||
|
||||
let url = '';
|
||||
|
||||
if (novelListing) {
|
||||
// URL structure with parameters
|
||||
const params = new URLSearchParams();
|
||||
const typeParam = this.options.typeParam || 'type'; // Default to 'type' if not specified
|
||||
|
||||
if (showLatestNovels) {
|
||||
params.append(typeParam, this.options.latestPage);
|
||||
params.append(typeParam, latestPage);
|
||||
} else if (filters.genres.value.length) {
|
||||
params.append(typeParam, this.options.genreParam || 'category_novel'); // same below
|
||||
const genreKey = this.options.genreKey || 'id'; // Default to 'id' if not specified
|
||||
params.append(typeParam, genreParam);
|
||||
params.append(genreKey, filters.genres.value);
|
||||
} else {
|
||||
params.append(typeParam, filters.type.value);
|
||||
}
|
||||
|
||||
// Add language parameter if specified
|
||||
if (this.options.langParam && this.options.urlLangCode) {
|
||||
params.append(this.options.langParam, this.options.urlLangCode);
|
||||
if (langParam && urlLangCode) {
|
||||
params.append(langParam, urlLangCode);
|
||||
}
|
||||
|
||||
params.append(pageParam, pageNo.toString());
|
||||
|
||||
url = `${this.site}${this.options.novelListing}?${params.toString()}`;
|
||||
url = `${this.site}${novelListing}?${params.toString()}`;
|
||||
} else {
|
||||
// URL structure with path segments
|
||||
let basePage = '';
|
||||
const params = new URLSearchParams({
|
||||
[pageParam]: pageNo.toString(),
|
||||
});
|
||||
const basePage = showLatestNovels
|
||||
? latestPage
|
||||
: filters.genres.value.length
|
||||
? filters.genres.value
|
||||
: filters.type.value;
|
||||
|
||||
if (showLatestNovels) {
|
||||
basePage = this.options.latestPage;
|
||||
} else if (filters.genres.value.length) {
|
||||
basePage = filters.genres.value;
|
||||
} else {
|
||||
basePage = filters.type.value;
|
||||
}
|
||||
|
||||
url = `${this.site}${basePage}?${params.toString()}`;
|
||||
url = `${this.site}${basePage}?${pageParam}=${pageNo}`;
|
||||
}
|
||||
|
||||
const $ = await this.getCheerio(url, false);
|
||||
return this.parseNovels($);
|
||||
const result = await fetchApi(url);
|
||||
if (!result.ok) {
|
||||
throw new Error(
|
||||
`Could not reach site (${result.status}: ${result.statusText}) try to open in webview.`,
|
||||
);
|
||||
}
|
||||
const html = await result.text();
|
||||
return this.parseNovels(html);
|
||||
}
|
||||
|
||||
async parseNovel(novelPath: string): Promise<Plugin.SourceNovel> {
|
||||
let $ = await this.getCheerio(this.site + novelPath, false);
|
||||
const baseUrl = this.site;
|
||||
const url = this.site + novelPath;
|
||||
const result = await fetchApi(url);
|
||||
const body = await result.text();
|
||||
|
||||
const novel: Partial<Plugin.SourceNovel> = {
|
||||
path: novelPath,
|
||||
chapters: [],
|
||||
};
|
||||
const summaryParts: string[] = [];
|
||||
const statusParts: string[] = [];
|
||||
const authorParts: string[] = [];
|
||||
const genreArray: string[] = [];
|
||||
const infoParts: string[] = [];
|
||||
const chapters: Plugin.ChapterItem[] = [];
|
||||
let novelId: string | null = null;
|
||||
let tempChapter: Partial<Plugin.ChapterItem> = {};
|
||||
let i = 0;
|
||||
let depth: number;
|
||||
|
||||
novel.name = $('h3.title').first().text().trim();
|
||||
const coverUrl =
|
||||
$('.book img').attr('src') ??
|
||||
$('.book img').attr('data-cfsrc') ??
|
||||
$('.book img').attr('data-src') ??
|
||||
$('.book noscript img').attr('src') ??
|
||||
$('meta[name="image"]').attr('content');
|
||||
const stateStack: ParsingState[] = [ParsingState.Idle];
|
||||
const currentState = () => stateStack[stateStack.length - 1];
|
||||
const pushState = (state: ParsingState) => stateStack.push(state);
|
||||
const popState = () =>
|
||||
stateStack.length > 1 ? stateStack.pop() : currentState();
|
||||
|
||||
if (coverUrl) {
|
||||
novel.cover = new URL(coverUrl, baseUrl).href;
|
||||
}
|
||||
const parser = new Parser({
|
||||
onopentag: (name, attribs) => {
|
||||
const state = currentState();
|
||||
switch (name) {
|
||||
case 'div':
|
||||
switch (attribs.class) {
|
||||
case 'books':
|
||||
case 'm-imgtxt':
|
||||
pushState(ParsingState.Cover);
|
||||
return;
|
||||
case 'inner':
|
||||
case 'desc-text':
|
||||
if (state === ParsingState.Cover) popState();
|
||||
pushState(ParsingState.Summary);
|
||||
break;
|
||||
case 'info':
|
||||
pushState(ParsingState.Info);
|
||||
depth = 0;
|
||||
break;
|
||||
}
|
||||
if (!this.options.noAjax && attribs.id === 'rating') {
|
||||
novelId = attribs['data-novel-id'];
|
||||
}
|
||||
if (state === ParsingState.Info) depth++;
|
||||
break;
|
||||
case 'img':
|
||||
if (state === ParsingState.Cover) {
|
||||
const cover =
|
||||
attribs.src ?? attribs['data-cfsrc'] ?? attribs['data-src'];
|
||||
const name = attribs.title;
|
||||
if (cover) {
|
||||
novel.cover = new URL(cover, this.site).href;
|
||||
}
|
||||
if (name) {
|
||||
novel.name = name;
|
||||
} else {
|
||||
popState();
|
||||
}
|
||||
}
|
||||
break;
|
||||
case 'h3':
|
||||
if (state === ParsingState.Cover) {
|
||||
pushState(ParsingState.NovelName);
|
||||
}
|
||||
break;
|
||||
case 'span':
|
||||
if (state === ParsingState.Cover && attribs.title) {
|
||||
const newState = {
|
||||
'Genre': ParsingState.Genres,
|
||||
'Author': ParsingState.Author,
|
||||
'Status': ParsingState.Status,
|
||||
}[attribs.title];
|
||||
|
||||
$('ul.info.info-meta li, .info div').each(function () {
|
||||
const detailText = $(this).text().split(':');
|
||||
const detailName = detailText[0].trim().toLowerCase();
|
||||
const detail = detailText[1]
|
||||
?.split(',')
|
||||
.map(g => g.trim())
|
||||
.join(', ');
|
||||
if (newState) pushState(newState);
|
||||
}
|
||||
break;
|
||||
case 'br':
|
||||
if (state === ParsingState.Summary) {
|
||||
summaryParts.push('\n');
|
||||
}
|
||||
break;
|
||||
case 'ul':
|
||||
if (attribs.class?.includes('info-meta')) {
|
||||
pushState(ParsingState.Info);
|
||||
}
|
||||
if (this.options.noAjax && attribs.id === 'idData') {
|
||||
pushState(ParsingState.ChapterList);
|
||||
}
|
||||
break;
|
||||
case 'a':
|
||||
if (state === ParsingState.ChapterList) {
|
||||
i++;
|
||||
const href = attribs.href;
|
||||
pushState(ParsingState.Chapter);
|
||||
|
||||
switch (detailName) {
|
||||
case 'author':
|
||||
novel.author = detail;
|
||||
break;
|
||||
case 'genre':
|
||||
novel.genres = detail;
|
||||
break;
|
||||
case 'status':
|
||||
const map: Record<string, string> = {
|
||||
completed: NovelStatus.Completed,
|
||||
ongoing: NovelStatus.Ongoing,
|
||||
hiatus: NovelStatus.OnHiatus,
|
||||
};
|
||||
novel.status = map[detail.toLowerCase()] ?? NovelStatus.Unknown;
|
||||
}
|
||||
tempChapter.name = attribs.title || `Chapter ${i}`;
|
||||
tempChapter.releaseTime = null;
|
||||
tempChapter.chapterNumber = i;
|
||||
tempChapter.path =
|
||||
href?.substring(1) ||
|
||||
novelPath.replace('.html', `/chapter-${i}.html`);
|
||||
}
|
||||
break;
|
||||
}
|
||||
},
|
||||
|
||||
ontext: data => {
|
||||
const text = data.trim();
|
||||
if (!text) return;
|
||||
|
||||
switch (currentState()) {
|
||||
case ParsingState.NovelName:
|
||||
novel.name = (novel.name || '') + text;
|
||||
break;
|
||||
case ParsingState.Summary:
|
||||
summaryParts.push(data);
|
||||
break;
|
||||
case ParsingState.Info:
|
||||
infoParts.push(text);
|
||||
break;
|
||||
case ParsingState.Genres:
|
||||
genreArray.push(data);
|
||||
break;
|
||||
case ParsingState.Author:
|
||||
authorParts.push(data);
|
||||
break;
|
||||
case ParsingState.Status:
|
||||
statusParts.push(text);
|
||||
break;
|
||||
}
|
||||
},
|
||||
|
||||
onclosetag: name => {
|
||||
const state = currentState();
|
||||
switch (name) {
|
||||
case 'div':
|
||||
switch (state) {
|
||||
case ParsingState.Info:
|
||||
depth--;
|
||||
infoParts.push('\n');
|
||||
if (depth < 0) {
|
||||
popState();
|
||||
}
|
||||
break;
|
||||
case ParsingState.Genres:
|
||||
case ParsingState.Author:
|
||||
case ParsingState.Status:
|
||||
case ParsingState.Summary:
|
||||
popState();
|
||||
break;
|
||||
}
|
||||
break;
|
||||
case 'h3':
|
||||
if (state === ParsingState.NovelName) {
|
||||
popState();
|
||||
}
|
||||
break;
|
||||
case 'a':
|
||||
if (state === ParsingState.Chapter) {
|
||||
if (tempChapter.name && tempChapter.path) {
|
||||
chapters.push({ ...tempChapter } as Plugin.ChapterItem);
|
||||
}
|
||||
tempChapter = {};
|
||||
popState();
|
||||
}
|
||||
break;
|
||||
case 'li':
|
||||
if (state === ParsingState.Info) {
|
||||
infoParts.push('\n');
|
||||
}
|
||||
break;
|
||||
case 'ul':
|
||||
switch (state) {
|
||||
case ParsingState.Info:
|
||||
case ParsingState.ChapterList:
|
||||
popState();
|
||||
break;
|
||||
}
|
||||
break;
|
||||
default:
|
||||
return;
|
||||
}
|
||||
},
|
||||
|
||||
onend: () => {
|
||||
if (infoParts.length) {
|
||||
infoParts
|
||||
.join('')
|
||||
.split('\n')
|
||||
.map(line => line.trim())
|
||||
.filter(line => line.includes(':'))
|
||||
.forEach(line => {
|
||||
const parts = line.split(':');
|
||||
const detailName = parts[0].trim().toLowerCase();
|
||||
const detail = parts[1]
|
||||
.split(',')
|
||||
.map(g => g.trim())
|
||||
.join(', ');
|
||||
|
||||
switch (detailName) {
|
||||
case 'author':
|
||||
novel.author = detail;
|
||||
break;
|
||||
case 'genre':
|
||||
novel.genres = detail;
|
||||
break;
|
||||
case 'status':
|
||||
const map: Record<string, string> = {
|
||||
ongoing: NovelStatus.Ongoing,
|
||||
hiatus: NovelStatus.OnHiatus,
|
||||
dropped: NovelStatus.Cancelled,
|
||||
cancelled: NovelStatus.Cancelled,
|
||||
completed: NovelStatus.Completed,
|
||||
};
|
||||
novel.status =
|
||||
map[detail.toLowerCase()] ?? NovelStatus.Unknown;
|
||||
break;
|
||||
default:
|
||||
return;
|
||||
}
|
||||
});
|
||||
|
||||
if (!novelId) {
|
||||
const idMatch = novelPath.match(/\d+/);
|
||||
novelId = idMatch ? idMatch[0] : null;
|
||||
}
|
||||
} else {
|
||||
novel.genres = genreArray.join('').trim();
|
||||
novel.author = authorParts.join('').trim();
|
||||
novel.status = statusParts
|
||||
.join('')
|
||||
.toLowerCase()
|
||||
.replace(/\b\w/g, char => char.toUpperCase());
|
||||
}
|
||||
novel.summary = summaryParts.join('\n\n').trim();
|
||||
},
|
||||
});
|
||||
|
||||
const summaryDiv = $('.desc-text');
|
||||
const pTagSummary = summaryDiv.find('p');
|
||||
novel.summary =
|
||||
pTagSummary.length > 0
|
||||
? pTagSummary
|
||||
.map((_, el) => $(el).text().trim())
|
||||
.get()
|
||||
.join('\n\n')
|
||||
: summaryDiv.text().trim(); // --- Else (no <p> tags, assume <br>) ---
|
||||
parser.write(body);
|
||||
parser.end();
|
||||
|
||||
const chapters: Plugin.ChapterItem[] = [];
|
||||
|
||||
const dataNovelId = $('#rating').attr('data-novel-id');
|
||||
const idMatch = novelPath.match(/\d+/);
|
||||
const novelId = dataNovelId || (idMatch ? idMatch[0] : null);
|
||||
|
||||
if (novelId) {
|
||||
if (this.options.noAjax && chapters.length > 0) {
|
||||
novel.chapters = chapters;
|
||||
} else if (novelId !== null) {
|
||||
const chapterListing =
|
||||
this.options.chapterListing || 'ajax/chapter-archive';
|
||||
|
||||
const ajaxParam = this.options.chapterParam || 'novelId';
|
||||
const params = new URLSearchParams({
|
||||
[ajaxParam]: novelId.toString(),
|
||||
});
|
||||
|
||||
const params = new URLSearchParams({ [ajaxParam]: novelId });
|
||||
const chaptersUrl = `${this.site}${chapterListing}?${params.toString()}`;
|
||||
|
||||
$ = await this.getCheerio(chaptersUrl, false);
|
||||
const ajaxResult = await fetchApi(chaptersUrl);
|
||||
if (!ajaxResult.ok) {
|
||||
console.error(`Failed to fetch chapters: ${ajaxResult.status}`);
|
||||
novel.chapters = [];
|
||||
} else {
|
||||
const ajaxBody = await ajaxResult.text();
|
||||
const ajaxChapters: Plugin.ChapterItem[] = [];
|
||||
let tempAjaxChapter: Partial<Plugin.ChapterItem> = {};
|
||||
|
||||
const listing = $('.panel-body li a, select option');
|
||||
const ajaxParser = new Parser({
|
||||
onopentag: (name, attribs) => {
|
||||
let chapterHref: string | undefined;
|
||||
let initialName: string | undefined;
|
||||
|
||||
listing.each((_, el) => {
|
||||
const $el = $(el);
|
||||
const chapterName = $el.attr('title') || $el.text() || '';
|
||||
const chapterHref = $el.attr('href') || $el.attr('value') || '';
|
||||
if (name === 'a' && attribs.href) {
|
||||
chapterHref = attribs.href;
|
||||
initialName = attribs.title || '';
|
||||
pushState(ParsingState.Chapter);
|
||||
} else if (name === 'option' && attribs.value) {
|
||||
chapterHref = attribs.value;
|
||||
initialName = '';
|
||||
pushState(ParsingState.Chapter);
|
||||
}
|
||||
|
||||
const chapterUrlObject = new URL(chapterHref, this.site);
|
||||
const chapterUrl = chapterUrlObject.pathname.slice(1);
|
||||
if (chapterHref !== undefined) {
|
||||
const href = new URL(chapterHref, this.site);
|
||||
tempAjaxChapter.path = href.pathname.substring(1);
|
||||
tempAjaxChapter.name = initialName;
|
||||
}
|
||||
},
|
||||
|
||||
chapters.push({
|
||||
name: chapterName,
|
||||
path: chapterUrl,
|
||||
releaseTime: null,
|
||||
ontext: data => {
|
||||
const text = data.trim();
|
||||
if (
|
||||
currentState() === ParsingState.Chapter &&
|
||||
!tempAjaxChapter.name &&
|
||||
text
|
||||
) {
|
||||
tempAjaxChapter.name += text;
|
||||
}
|
||||
},
|
||||
|
||||
onclosetag: name => {
|
||||
if (
|
||||
(name === 'a' || name === 'option') &&
|
||||
currentState() === ParsingState.Chapter
|
||||
) {
|
||||
if (tempAjaxChapter.name && tempAjaxChapter.path) {
|
||||
tempAjaxChapter.name = tempAjaxChapter.name.trim();
|
||||
tempAjaxChapter.releaseTime = null;
|
||||
ajaxChapters.push({
|
||||
...tempAjaxChapter,
|
||||
} as Plugin.ChapterItem);
|
||||
}
|
||||
tempAjaxChapter = {};
|
||||
popState();
|
||||
}
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
ajaxParser.write(ajaxBody);
|
||||
ajaxParser.end();
|
||||
novel.chapters = ajaxChapters;
|
||||
}
|
||||
}
|
||||
novel.chapters = chapters;
|
||||
|
||||
return novel as Plugin.SourceNovel;
|
||||
}
|
||||
|
||||
async parseChapter(chapterPath: string): Promise<string> {
|
||||
const $ = await this.getCheerio(this.site + chapterPath, false);
|
||||
$('div.ads, div.unlock-buttons').remove();
|
||||
return $('#chr-content, #chapter-content').html()! || '';
|
||||
const response = await fetchApi(this.site + chapterPath);
|
||||
const html = await response.text();
|
||||
|
||||
let depth: number;
|
||||
let depthHide: number;
|
||||
const chapterHtml: string[] = [];
|
||||
let skipClosingTag = false;
|
||||
let currentTagToSkip = '';
|
||||
|
||||
const stateStack: ParsingState[] = [ParsingState.Idle];
|
||||
const currentState = () => stateStack[stateStack.length - 1];
|
||||
const pushState = (state: ParsingState) => stateStack.push(state);
|
||||
const popState = () =>
|
||||
stateStack.length > 1 ? stateStack.pop() : currentState();
|
||||
|
||||
type EscapeChar = '&' | '<' | '>' | '"' | "'" | ' ';
|
||||
const escapeRegex = /[&<>"' ]/g;
|
||||
const escapeMap: Record<EscapeChar, string> = {
|
||||
'&': '&',
|
||||
'<': '<',
|
||||
'>': '>',
|
||||
'"': '"',
|
||||
"'": ''',
|
||||
' ': ' ',
|
||||
};
|
||||
const escapeHtml = (text: string): string =>
|
||||
text.replace(escapeRegex, char => escapeMap[char as EscapeChar]);
|
||||
|
||||
const parser = new Parser({
|
||||
onopentag(name, attribs) {
|
||||
const state = currentState();
|
||||
const attrib = attribs.class?.trim();
|
||||
switch (state) {
|
||||
case ParsingState.Idle:
|
||||
if (
|
||||
attrib === 'txt' ||
|
||||
attribs.id === 'chr-content' ||
|
||||
attribs.id === 'chapter-content'
|
||||
) {
|
||||
pushState(ParsingState.Chapter);
|
||||
depth = 0;
|
||||
}
|
||||
break;
|
||||
case ParsingState.Chapter:
|
||||
if (name === 'sub') {
|
||||
pushState(ParsingState.Hidden);
|
||||
} else if (name === 'div') {
|
||||
depth++;
|
||||
if (
|
||||
attrib?.includes('unlock-buttons') ||
|
||||
attrib?.includes('ads')
|
||||
) {
|
||||
pushState(ParsingState.Hidden);
|
||||
depthHide = 0;
|
||||
}
|
||||
}
|
||||
break;
|
||||
case ParsingState.Hidden:
|
||||
if (name === 'sub') {
|
||||
// Allow nesting of hidden states if a sub is inside a div
|
||||
pushState(ParsingState.Hidden);
|
||||
} else if (name === 'div') {
|
||||
depthHide++;
|
||||
}
|
||||
break;
|
||||
default:
|
||||
return;
|
||||
}
|
||||
|
||||
if (currentState() === ParsingState.Chapter) {
|
||||
const attrKeys = Object.keys(attribs);
|
||||
|
||||
if (attrKeys.length === 0) {
|
||||
chapterHtml.push(`<${name}>`);
|
||||
} else if (attrKeys.every(key => attribs[key].trim() === '')) {
|
||||
// Handle tags with empty attributes as text content
|
||||
// eg: novel/rising-up-from-a-nobleman-to-intergalactic-warlord/chapter-184
|
||||
skipClosingTag = true;
|
||||
currentTagToSkip = name;
|
||||
const uppercaseName = name.replace(/\b\w/g, char =>
|
||||
char.toUpperCase(),
|
||||
);
|
||||
chapterHtml.push(
|
||||
escapeHtml(`<${uppercaseName} ${attrKeys.join(' ')}>`),
|
||||
);
|
||||
} else {
|
||||
// Normal tag with attributes
|
||||
const attrString = attrKeys
|
||||
.map(key => ` ${key}="${attribs[key].replace(/"/g, '"')}"`)
|
||||
.join('');
|
||||
chapterHtml.push(`<${name}${attrString}>`);
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
ontext(text) {
|
||||
if (currentState() === ParsingState.Chapter) {
|
||||
chapterHtml.push(escapeHtml(text));
|
||||
}
|
||||
},
|
||||
|
||||
onclosetag(name) {
|
||||
const state = currentState();
|
||||
|
||||
if (state === ParsingState.Hidden) {
|
||||
if (name === 'sub') {
|
||||
popState();
|
||||
} else if (name === 'div') {
|
||||
depthHide--;
|
||||
if (depthHide < 0) {
|
||||
popState();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (state !== ParsingState.Chapter) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (!parser['isVoidElement'](name)) {
|
||||
if (skipClosingTag && name === currentTagToSkip) {
|
||||
skipClosingTag = false;
|
||||
currentTagToSkip = '';
|
||||
} else {
|
||||
chapterHtml.push(`</${name}>`);
|
||||
}
|
||||
}
|
||||
|
||||
if (name === 'div') {
|
||||
depth--;
|
||||
if (depth < 0) {
|
||||
pushState(ParsingState.Stopped);
|
||||
}
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
parser.write(html);
|
||||
parser.end();
|
||||
|
||||
return chapterHtml.join('');
|
||||
}
|
||||
|
||||
async searchNovels(
|
||||
searchTerm: string,
|
||||
page: number,
|
||||
): Promise<Plugin.NovelItem[]> {
|
||||
const pageParam = this.options.pageParam || 'page';
|
||||
|
||||
const params = new URLSearchParams({
|
||||
keyword: searchTerm,
|
||||
[pageParam]: page.toString(),
|
||||
});
|
||||
|
||||
if (this.options.langParam) {
|
||||
params.append(this.options.langParam, this.options.urlLangCode!);
|
||||
const now = Date.now();
|
||||
if (this.lastSearch && now - this.lastSearch <= this.searchInterval) {
|
||||
await this.sleep(this.searchInterval);
|
||||
}
|
||||
|
||||
const url = `${this.site}${this.options.searchPage}?${params.toString()}`;
|
||||
const $ = await this.getCheerio(url, true);
|
||||
const {
|
||||
pageParam = 'page',
|
||||
searchKey = 'keyword',
|
||||
postSearch,
|
||||
langParam,
|
||||
urlLangCode,
|
||||
searchPage,
|
||||
} = this.options;
|
||||
|
||||
return this.parseNovels($);
|
||||
const params = new URLSearchParams({
|
||||
[searchKey]: searchTerm,
|
||||
...(langParam && urlLangCode && { [langParam]: urlLangCode }),
|
||||
...(!postSearch && { [pageParam]: page.toString() }),
|
||||
});
|
||||
|
||||
const url = `${this.site}${searchPage}${!postSearch ? `?${params.toString()}` : ''}`;
|
||||
|
||||
const fetchOptions: RequestInit | undefined = postSearch
|
||||
? {
|
||||
method: 'POST',
|
||||
body: params.toString(),
|
||||
headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
|
||||
}
|
||||
: undefined;
|
||||
|
||||
const result = await fetchApi(url, fetchOptions);
|
||||
this.lastSearch = Date.now();
|
||||
|
||||
if (!result.ok) {
|
||||
throw new Error(
|
||||
`Could not reach site ('${result.status}') try to open in webview.`,
|
||||
);
|
||||
}
|
||||
|
||||
const html = await result.text();
|
||||
|
||||
// Check for alert error messages, ported over from cheerio TODO: confirm behaviour
|
||||
const alertText = html.match(/alert\((.*?)\)/)?.[1] || '';
|
||||
if (alertText) throw new Error(alertText);
|
||||
|
||||
return this.parseNovels(html);
|
||||
}
|
||||
}
|
||||
|
||||
enum ParsingState {
|
||||
Idle,
|
||||
Info,
|
||||
Cover,
|
||||
Author,
|
||||
Genres,
|
||||
Status,
|
||||
Hidden,
|
||||
Summary,
|
||||
Stopped,
|
||||
Chapter,
|
||||
ChapterList,
|
||||
NovelName,
|
||||
NovelList,
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user