diff --git a/plugins/english/lightnovelpub.ts b/plugins/english/lightnovelpub.ts index 58932fd..7693df3 100644 --- a/plugins/english/lightnovelpub.ts +++ b/plugins/english/lightnovelpub.ts @@ -1,4 +1,4 @@ -import { load as parseHTML } from 'cheerio'; +import { Parser } from 'htmlparser2'; import { fetchApi, fetchFile } from '@libs/fetch'; import { Plugin } from '@typings/plugin'; import { Filters, FilterTypes } from '@libs/filterInputs'; @@ -7,7 +7,7 @@ import dayjs from 'dayjs'; class LightNovelPub implements Plugin.PagePlugin { id = 'lightnovelpub'; name = 'LightNovelPub'; - version = '1.0.1'; + version = '2.0.0'; icon = 'src/en/lightnovelpub/icon.png'; site = 'https://www.lightnovelpub.com/'; headers = { @@ -15,6 +15,42 @@ class LightNovelPub implements Plugin.PagePlugin { 'Content-Type': 'application/json', }; + parseNovels(html: string) { + const novels: Plugin.NovelItem[] = []; + let tempNovel = {} as Plugin.NovelItem; + let isParsingNovel = false; + const parser = new Parser ({ + onopentag(name, attribs){ + if (isParsingNovel){ + if (name === 'a'){ + tempNovel.path = attribs['href'].slice(1); + tempNovel.name = attribs['title']; + } + if (name === 'img'){ + tempNovel.cover = attribs['data-src'] || attribs['src']; + } + if (tempNovel.path && tempNovel.cover){ + novels.push(tempNovel); + tempNovel = {} as Plugin.NovelItem; + } + } + }, + onattribute(name, value){ + if (name === 'class' && value === 'novel-item'){ + isParsingNovel = true; + } + }, + onclosetag(name){ + if(name === 'li'){ + isParsingNovel = false; + } + } + }) + parser.write(html); + parser.end(); + return novels; + } + async popularNovels( page: number, { filters }: Plugin.PopularNovelsOptions, @@ -27,91 +63,171 @@ class LightNovelPub implements Plugin.PagePlugin { const body = await fetchApi(link).then(r => r.text()); - const loadedCheerio = parseHTML(body); - - const novels: Plugin.NovelItem[] = []; - - loadedCheerio('.novel-item.ads').remove(); - - loadedCheerio('.novel-item').each((idx, ele) => { - const novelName = loadedCheerio(ele).find('.novel-title').text().trim(); - const novelCover = loadedCheerio(ele).find('img').attr('data-src'); - const novelUrl = loadedCheerio(ele) - .find('.novel-title > a') - .attr('href') - ?.substring(1); - - if (!novelUrl) return; - const novel = { - name: novelName, - cover: novelCover, - path: novelUrl, - }; - - novels.push(novel); - }); - - return novels; + return this.parseNovels(body); } async parseNovel( novelPath: string, ): Promise { const body = await fetchApi(this.site + novelPath).then(r => r.text()); - - const loadedCheerio = parseHTML(body); - const totalChapters = parseInt( - loadedCheerio('.header-stats span:first strong').text(), - 10, - ); - const novel: Plugin.SourceNovel & { totalPages: number } = { path: novelPath, - name: loadedCheerio('h1.novel-title').text().trim() || 'Untitled', - cover: loadedCheerio('figure.cover > img').attr('data-src'), - author: loadedCheerio('.author > a > span').text(), - summary: loadedCheerio('.summary > .content').text().trim(), - status: loadedCheerio('.header-stats span:last strong').text(), - totalPages: Math.ceil(totalChapters / 100), + name: '', + summary: '', + totalPages: 1, chapters: [], - }; - - novel.genres = loadedCheerio('.categories ul li') - .map((a, ex) => loadedCheerio(ex).text().trim()) - .toArray() - .join(','); - + } + let isHeaderStats = false; + let isStatus = false; + let isTotalChapters = false; + let isNovelName = false; + let isAuthorName = false; + let isSummary = false; + let isGenres = false; + let isTags = false; + let genreArray: string[] = [] + let isCover = false; + const parser = new Parser({ + onopentag(name, attribs){ + if(name === 'h1' && attribs['class']?.includes('novel-title')){ + isNovelName = true; + } + if(name === 'div' && attribs['class']?.includes('content')){ + isSummary = true; + } + if(name === 'figure' && attribs['class'] === 'cover'){ + isCover = true; + } + if(isCover){ + if(name === 'img'){ + novel.cover = attribs['data-src'] || attribs['src']; + } + } + }, + onopentagname(name){ + if (isHeaderStats && name === 'strong'){ + isStatus = true; + } + if (isSummary && name === 'br'){ + novel.summary += '\n'; + } + if (isGenres && name === 'a'){ + isTags = true; + } + }, + onattribute(name,value){ + if (name === 'class' && value === 'header-stats'){ + isTotalChapters = true; + isHeaderStats = true; + } + if (name === 'itemprop' && value === 'author'){ + isAuthorName = true; + } + if (name === 'class' && value === 'categories'){ + isGenres = true; + } + }, + ontext(data){ + if(isTotalChapters){ + novel.totalPages = Math.ceil(parseInt(data, 10) / 100); + } + if(isStatus){ + novel.status = data.trim(); + } + if(isNovelName){ + novel.name += data.trim(); + } + if(isAuthorName){ + novel.author = data; + } + if(isSummary){ + novel.summary += data; + } + if(isTags){ + genreArray.push(data); + } + }, + onclosetag(name){ + if(name === 'strong'){ + isTotalChapters = false; + isStatus = false; + } + if(name === 'i'){ + isStatus = false; + } + if(name === 'h1'){ + isNovelName = false; + } + if(name === 'span'){ + isAuthorName = false; + } + if(name === 'div'){ + isHeaderStats = false; + isSummary = false; + isGenres = false; + } + if(name === 'a'){ + isTags = false; + } + if(name === 'figure'){ + isCover = false; + } + } + }) + parser.write(body); + parser.end(); + novel.genres = genreArray.join(', '); return novel; } async parsePage(novelPath: string, page: string): Promise { const url = this.site + novelPath + '/chapters/page-' + page; const body = await fetchApi(url).then(res => res.text()); - const loadedCheerio = parseHTML(body); const chapter: Plugin.ChapterItem[] = []; - loadedCheerio('.chapter-list li').each(function () { - const chapterName = - 'Chapter ' + - loadedCheerio(this).find('.chapter-no').text().trim() + - ' - ' + - loadedCheerio(this).find('.chapter-title').text().trim(); - - const releaseDate = loadedCheerio(this) - .find('.chapter-update') - .attr('datetime'); - - const chapterUrl = loadedCheerio(this) - .find('a') - .attr('href') - ?.substring(1); - if (!chapterUrl) return; - - chapter.push({ - name: chapterName, - path: chapterUrl, - releaseTime: dayjs(releaseDate).toISOString(), - }); - }); + let tempChapter = {} as Plugin.ChapterItem; + let isChapterItem = false; + let isChapterList = false; + const parser = new Parser ({ + onopentag(name, attribs){ + if(isChapterList){ + if (name === 'li'){ + isChapterItem = true; + tempChapter.chapterNumber = Number(attribs['data-orderno']); + } + if (isChapterItem){ + if (name === 'a'){ + tempChapter.name = attribs['title']; + tempChapter.path = attribs['href'].slice(1); + } + if (name === 'time'){ + tempChapter.releaseTime = dayjs(attribs['datetime']).toISOString(); + } + if ( + tempChapter.chapterNumber && + tempChapter.path && + tempChapter.releaseTime){ + chapter.push(tempChapter); + tempChapter = {} as Plugin.ChapterItem; + } + } + } + }, + onattribute(name, value){ + if (name === 'class' && value === 'chapter-list'){ + isChapterList = true; + } + }, + onclosetag(name){ + if (name === 'a'){ + isChapterItem = false; + } + if (name === 'ul'){ + isChapterList = false; + } + } + }) + parser.write(body); + parser.end(); const chapters = chapter; return { chapters }; } @@ -119,9 +235,98 @@ class LightNovelPub implements Plugin.PagePlugin { async parseChapter(chapterPath: string): Promise { const body = await fetchApi(this.site + chapterPath).then(r => r.text()); - const loadedCheerio = parseHTML(body); - - let chapterText = loadedCheerio('#chapter-container').html() || ''; + let chapterText = ''; + let isChapter = false; + let isPtag = false; + let isStyleText = false; + const parser = new Parser({ + onopentag(name,attribs){ + if (isChapter && name === 'div'){ + let stylediv = attribs['style'] + if(stylediv){ + chapterText += `
` + isStyleText = true; + } else { + chapterText += `
` + } + } + if (isChapter && name === 'table'){ + let w = attribs['width']; + if(w) { + chapterText += ``; + } else { + chapterText += `
`; + } + } + if (isChapter && name === 'tbody'){ + chapterText += ``; + } + if (isChapter && name === 'tr'){ + chapterText += ``; + } + if (isChapter && name === 'td'){ + let w1 = attribs['width']; + if(w1) { + chapterText += ``; + } + if (isChapter && name === 'tr'){ + chapterText += ``; + } + if (isChapter && name === 'tbody'){ + chapterText += ``; + } + if (isChapter && name === 'table'){ + chapterText += `
`; + } else { + chapterText += ``; + } + } + }, + onattribute(name, value){ + if (name === 'id' && value === 'chapter-container'){ + isChapter = true; + } + if (name === 'class' && value?.includes('chapternav')){ + isChapter = false; + isPtag = false; + } + }, + onopentagname(name){ + if (isChapter && name === 'p'){ + chapterText += '

'; + isPtag = true; + if(isStyleText){ + isStyleText = false; + } + } + if (isChapter && name === 'br'){ + chapterText += `
`; + } + }, + ontext(data){ + if(isPtag){ + chapterText += data; + } + if(isStyleText){ + chapterText += data; + } + }, + onclosetag(name){ + if (name === 'p'){ + isPtag = false; + chapterText += '

' + } + if (isChapter && name === 'td'){ + chapterText += `
`; + } + if (isChapter && name === 'div'){ + isStyleText = false; + chapterText += `
` + } + } + }) + parser.write(body) + parser.end(); return chapterText; } @@ -133,39 +338,27 @@ class LightNovelPub implements Plugin.PagePlugin { const url = `${this.site}lnsearchlive`; const link = `${this.site}search`; const response = await fetchApi(link).then(r => r.text()); - const token = parseHTML(response); - let verifytoken = token('#novelSearchForm > input').attr('value'); + let verifytoken = ''; + const parser = new Parser ({ + onopentag(name, attribs) { + if (name === 'input' && attribs['name']?.includes('LNRequestVerifyToken')){ + verifytoken = attribs['value']; + } + }, + }) + parser.write(response); + parser.end(); - let formData = new FormData(); + const formData = new FormData(); formData.append('inputContent', searchTerm); const body = await fetchApi(url, { method: 'POST', headers: { LNRequestVerifyToken: verifytoken! }, body: formData, - }).then(r => r.text()); + }).then(r => r.json()); - let loadedCheerio = parseHTML(body); - - let novels: Plugin.NovelItem[] = []; - - let results = JSON.parse(loadedCheerio('body').text()); - - loadedCheerio = parseHTML(results.resultview); - - loadedCheerio('.novel-item').each((idx, ele) => { - const novelName = loadedCheerio(ele).find('h4.novel-title').text().trim(); - const novelCover = loadedCheerio(ele).find('img').attr('src'); - const novelUrl = loadedCheerio(ele).find('a').attr('href')?.substring(1); - if (!novelUrl) return; - novels.push({ - name: novelName, - path: novelUrl, - cover: novelCover, - }); - }); - - return novels; + return this.parseNovels(body.resultview) } async fetchImage(url: string): Promise { diff --git a/plugins/english/lnmtl.ts b/plugins/english/lnmtl.ts index 41a6571..226795e 100644 --- a/plugins/english/lnmtl.ts +++ b/plugins/english/lnmtl.ts @@ -1,4 +1,4 @@ -import { load as parseHTML } from 'cheerio'; +import { Parser } from 'htmlparser2'; import { fetchApi, fetchFile } from '@libs/fetch'; import { FilterTypes, Filters } from '@libs/filterInputs'; import { Plugin } from '@typings/plugin'; @@ -8,7 +8,7 @@ class LnMTLPlugin implements Plugin.PagePlugin { name = 'LnMTL'; icon = 'src/en/lnmtl/icon.png'; site = 'https://lnmtl.com/'; - version = '1.0.1'; + version = '2.0.0'; async sleep(ms: number) { return new Promise(resolve => setTimeout(resolve, ms)); @@ -24,98 +24,191 @@ class LnMTLPlugin implements Plugin.PagePlugin { link += `&filter=${filters.storyStatus.value}`; link += `&page=${page}`; - const body = await fetchApi(link).then(result => result.text()); - - const loadedCheerio = parseHTML(body); + const body = await fetchApi(link); + const html = await body.text(); + const baseUrl = this.site; + let isParsingNovel = false; + let tempNovel = {} as Plugin.NovelItem; const novels: Plugin.NovelItem[] = []; - - loadedCheerio('.media').each((i, el) => { - const novelName = loadedCheerio(el).find('h4').text(); - const novelCover = loadedCheerio(el).find('img').attr('src'); - const novelUrl = loadedCheerio(el).find('h4 > a').attr('href'); - - if (!novelUrl) return; - - const novel = { - name: novelName, - cover: novelCover, - path: novelUrl.replace(this.site, ''), - }; - - novels.push(novel); + const parser = new Parser ({ + onopentag(name, attribs) { + if (attribs['class']?.includes('media-left')) { + isParsingNovel = true; + } + if (isParsingNovel) { + switch (name){ + case 'a': + tempNovel.path = attribs['href'].slice(baseUrl.length); + break; + case 'img': + tempNovel.name = attribs['alt']; + tempNovel.cover = attribs['src']; + break; + } + if (tempNovel.path && tempNovel.name){ + novels.push(tempNovel); + tempNovel = {} as Plugin.NovelItem; + } + } + }, + onclosetag(name){ + if (name === 'div'){ + isParsingNovel = false; + } + }, }); - + parser.write(html); + parser.end(); return novels; } async parseNovel( novelPath: string, ): Promise { - const result = await fetchApi(this.site + novelPath); - const body = await result.text(); - - const loadedCheerio = parseHTML(body); - - const volumes = JSON.parse( - loadedCheerio('main') - .next() - .html() - ?.match(/lnmtl.volumes = \[(.*?)\]/)![0] - ?.replace('lnmtl.volumes = ', '') || '', - ); + const body = await fetchApi(this.site + novelPath); + const html = await body.text().then(r => r.replace(/>\s+<')); const novel: Plugin.SourceNovel & { totalPages: number } = { path: novelPath, - name: loadedCheerio('.novel-name').text() || 'Untitled', - cover: loadedCheerio('div.novel').find('img').attr('src'), - summary: loadedCheerio('div.description').text().trim(), - totalPages: volumes.length, + name: '', + totalPages: 1, + summary: '', + author: '', + status: '', chapters: [], }; - loadedCheerio('.panel-body > dl').each(function () { - const detailName = loadedCheerio(this).find('dt').text().trim(); - const detail = loadedCheerio(this).find('dd').text().trim(); - - switch (detailName) { - case 'Authors': - novel.author = detail; - break; - case 'Current status': - novel.status = detail; - break; - } - }); - - novel.genres = loadedCheerio('.panel-heading:contains(" Genres ")') - .next() - .find('a') - .map((i, el) => loadedCheerio(el).text()) - .toArray() - .join(','); - + let isScript = false; + let isDescription = false; + let isSource = false; + let isPanelKey = false; + let isPanelValue = 0; + let isAuthorKey = false + let isStatusKey = false + let isList = 0; + let isGenres = false; + let genreArray: string[] = [] + const parser = new Parser({ + onopentag(name,attribs) { + if (name === 'img' && attribs['class']?.includes('img-rounded')){ + novel.name = attribs['title']; + novel.cover = attribs['src']; + } + if (name === 'ul' && attribs['class']?.includes('list-inline')){ + isList++; + } + }, + onopentagname(name){ + if (name === 'dt'){ + isPanelKey = true; + } + if (name === 'dd'){ + isPanelValue++; + } + if (isList === 1 && name === 'li'){ + isGenres = true; + } + }, + onattribute(name, value){ + if (name === 'class' && value === 'description'){ + isDescription = true; + } + if (name === 'class' && value === 'source'){ + isDescription = false; + isSource = true; + } + if (name === 'class' && value === 'progress'){ + isSource = false; + } + }, + ontext(data){ + if(isScript){ + const volume = JSON.parse( + data.match(/lnmtl.volumes = (.+])(?=;)/)![1] || '' + ); + novel.totalPages = volume.length; + } + if(isDescription){ + novel.summary += data.trim() + '\n\n'; + } + if(isSource){ + novel.summary += data; + } + if(isPanelKey){ + switch(data){ + case 'Authors': + isAuthorKey = true; + break; + case 'Current status': + isStatusKey = true; + break; + } + } + if(isAuthorKey && isPanelValue === 1){ + novel.author += data.trim(); + isAuthorKey = false; + } + if(isStatusKey && isPanelValue === 2){ + novel.status += data.trim(); + isStatusKey = false; + } + if(isGenres){ + genreArray.push(data.trim()); + novel.genres = genreArray.join(', '); + } + }, + onclosetag(name){ + if (name === 'ul'){ + isGenres = false; + } + if(name === 'main'){ + isPanelKey = false; + isScript = true; + } + if (name === 'script'){ + isScript = false; + } + }, + }) + parser.write(html); + parser.end(); return novel; } async parsePage(novelPath: string, page: string): Promise { const result = await fetchApi(this.site + novelPath); - const body = await result.text(); - - const loadedCheerio = parseHTML(body); - - const volumes = JSON.parse( - loadedCheerio('main') - .next() - .html() - ?.match(/lnmtl.volumes = \[(.*?)\]/)![0] - ?.replace('lnmtl.volumes = ', '') || '', - )[+page - 1]; + const html = await result.text().then(r => r.replace(/>\s+<')); + let isScript = false; + let volume: VolumeEntry = { + id: '', + title: '', + } + const parser = new Parser({ + ontext(data){ + if(isScript){ + const volumes = JSON.parse( + data.match(/lnmtl.volumes = (.+])(?=;)/)![1] || '' + ); + volume = volumes[+page-1] + } + }, + onclosetag(name){ + if(name === 'main'){ + isScript = true; + } + if (name === 'script'){ + isScript = false; + } + }, + }) + parser.write(html); + parser.end(); const chapter: Plugin.ChapterItem[] = []; await this.sleep(1000); const volumeData = await fetchApi( - `${this.site}chapter?volumeId=${volumes.id}`, + `${this.site}chapter?volumeId=${volume.id}`, ); const volumePage = await volumeData.json(); const firstPage = volumePage.data.map((chapter: ChapterEntry) => ({ @@ -128,7 +221,7 @@ class LnMTLPlugin implements Plugin.PagePlugin { for (let i = 2; i <= volumePage.last_page; i++) { await this.sleep(1000); const chapterData = await fetchApi( - `${this.site}chapter?page=${i}&volumeId=${volumes.id}`, + `${this.site}chapter?page=${i}&volumeId=${volume.id}`, ); const chapterInfo = await chapterData.json(); @@ -146,34 +239,75 @@ class LnMTLPlugin implements Plugin.PagePlugin { async parseChapter(chapterPath: string): Promise { const result = await fetchApi(this.site + chapterPath); - const body = await result.text(); + const html = await result.text(); + let isText = false; + let chapterText = ''; + const parser = new Parser({ + onopentag(name,attribs){ + if(name === 'sentence' && attribs['class']?.includes('translated')){ + isText = true; + chapterText += '

' + } + }, + onopentagname(name){ + if(name === 'nav'){ + isText = false; + } + }, + ontext(data){ + if(isText){ + chapterText += data; + } + }, + onclosetag(name){ + if (name === 'sentence'){ + chapterText += '

' + isText = false; + } + } + }) + parser.write(html); + parser.end(); - const loadedCheerio = parseHTML(body); - - loadedCheerio('.original, script').remove(); - loadedCheerio('sentence.translated').wrap('

'); - - let chapterText = loadedCheerio('.chapter-body').html()?.replace(/„/g, '“'); - - if (!chapterText) { - chapterText = loadedCheerio('.alert.alert-warning').text(); - } - - return chapterText; + return chapterText.replace(/„/g, '“'); } async searchNovels( searchTerm: string, pageNo: number, ): Promise { - const body = await fetchApi(this.site).then(r => r.text()); - const loadedCheerio = parseHTML(body); - - const list = loadedCheerio('footer') - .next() - .next() - .html() - ?.match(/prefetch: '\/(.*json)/)![1]; + const html = await fetchApi(this.site) + .then(b => b.text()) + .then(r => r.replace(/>\s+<')); + let isScript = false; + let isFooter = false; + let list = '' + const parser = new Parser({ + onopentag(name, attribs) { + if(isFooter && name === 'script' && + attribs['type']?.includes('application/javascript')){ + isScript = true; + } + }, + ontext(data){ + if (isScript){ + list = data.match(/prefetch: '\/(.*json)/)![1]; + } + }, + onclosetag(name){ + if (name === 'footer'){ + isFooter = true; + } + if (name === 'script'){ + isScript = false; + } + if (name === 'body'){ + isFooter = false; + } + } + }) + parser.write(html); + parser.end(); const search = await fetch(`${this.site}${list}`); const data = await search.json(); @@ -243,3 +377,8 @@ interface ChapterEntry { slug: string; created_at: string; } + +interface VolumeEntry { + id: string; + title: string; +} diff --git a/plugins/english/ranobes.ts b/plugins/english/ranobes.ts index 11324df..0bcfbf1 100644 --- a/plugins/english/ranobes.ts +++ b/plugins/english/ranobes.ts @@ -1,47 +1,69 @@ -import { CheerioAPI, load as parseHTML } from 'cheerio'; +import { Parser } from 'htmlparser2'; import { fetchApi, fetchFile } from '@libs/fetch'; import { FilterTypes, Filters } from '@libs/filterInputs'; import { Plugin } from '@typings/plugin'; -interface ChapterEntry { - id: number; - title: string; - date: string; - link: string; -} - class RanobesPlugin implements Plugin.PagePlugin { id = 'ranobes'; name = 'Ranobes'; icon = 'src/en/ranobes/icon.png'; site = 'https://ranobes.top'; filters?: Filters | undefined; - version = '1.1.0'; + version = '2.0.0'; async sleep(ms: number) { return new Promise(resolve => setTimeout(resolve, ms)); } - parseNovels(loadedCheerio: CheerioAPI) { + parseNovels(html: string) { const novels: Plugin.NovelItem[] = []; - - loadedCheerio('.short-cont').each((i, el) => { - const novelName = loadedCheerio(el).find('h2.title').text(); - const novelUrl = loadedCheerio(el).find('h2.title a').attr('href'); - const novelCover = loadedCheerio(el) - .find('figure') - .attr('style') - ?.replace(/.*url\((.*?)\)./g, '$1'); - - if (!novelUrl) return; - - const novel = { - name: novelName, - cover: novelCover, - path: novelUrl?.replace(this.site, ''), - }; - novels.push(novel); - }); + let tempNovel = {} as Plugin.NovelItem; + tempNovel.name = ''; + const baseUrl = this.site + let isParsingNovel = false; + let isTitleTag = false; + let isNovelName = false; + const parser = new Parser({ + onopentag(name, attribs) { + if (attribs['class']?.includes('short-cont')){ + isParsingNovel = true; + } + if (isParsingNovel){ + if (name === 'h2' && attribs['class']?.includes('title')){ + isTitleTag = true; + } + if (isTitleTag && name === 'a'){ + tempNovel.path = attribs['href'].slice(baseUrl.length); + isNovelName = true; + } + if (name === 'figure'){ + tempNovel.cover = attribs['style'] + .replace(/.*url\((.*?)\)./g, '$1'); + } + if (tempNovel.path && tempNovel.cover){ + novels.push(tempNovel); + tempNovel = {} as Plugin.NovelItem; + tempNovel.name = ''; + } + } + }, + ontext(data){ + if(isNovelName){ + tempNovel.name += data; + } + }, + onclosetag(name){ + if(name === 'h2'){ + isNovelName = false; + isTitleTag = false; + } + if(name === 'figure'){ + isParsingNovel = false; + } + } + }) + parser.write(html); + parser.end(); return novels; } @@ -51,7 +73,7 @@ class RanobesPlugin implements Plugin.PagePlugin { chapter.push({ name: item.title, releaseTime: new Date(item.date).toISOString(), - path: item.link.replace(this.site, ''), + path: item.link.slice(this.site.length), }); }); return chapter.reverse(); @@ -65,10 +87,9 @@ class RanobesPlugin implements Plugin.PagePlugin { }: Plugin.PopularNovelsOptions, ): Promise { let link = `${this.site}/novels/page/${page}/`; - const body = await fetchApi(link).then(result => result.text()); + const body = await fetchApi(link).then(r => r.text()); - const loadedCheerio = parseHTML(body); - return this.parseNovels(loadedCheerio); + return this.parseNovels(body); } async parseNovel( @@ -76,52 +97,133 @@ class RanobesPlugin implements Plugin.PagePlugin { ): Promise { const url = this.site + novelPath; const result = await fetchApi(url); - const body = await result.text(); - - let loadedCheerio = parseHTML(body); - + const html = await result.text(); const novel: Plugin.SourceNovel & { totalPages: number } = { path: novelPath, - name: loadedCheerio('.poster img').attr('alt') || 'Untitled', - cover: this.site + loadedCheerio('.poster img').attr('src'), - author: loadedCheerio('[itemprop="creator"]').text(), + name: '', + summary: '', chapters: [], totalPages: 1, - }; - - novel.summary = loadedCheerio('.moreless') - .find('br') - .replaceWith('\n') - .end() - .text() - .trim(); - - novel.status = - loadedCheerio('li:contains("Status") span').text() === 'Ongoing' - ? 'Ongoing' - : 'Completed'; - - novel.genres = loadedCheerio('div#mc-fs-genre').text().trim(); - - let chapterListUrl = loadedCheerio('.read-continue:last').attr('href'); - - if (!chapterListUrl?.startsWith('http')) { - chapterListUrl = this.site + chapterListUrl; } + let baseUrl = this.site; + let isCover = false; + let isAuthor = false; + let isSummary = false; + let isStatus = false; + let isStatusText = false; + let isGenres = false; + let isGenresText = false; + let genreArray: string[] = [] + let novelId = ''; + const parser = new Parser ({ + onopentag(name,attribs){ + if (attribs['class'] === 'poster'){ + isCover = true; + } + if (isCover && name === 'img'){ + novel.name = attribs['alt']; + novel.cover = baseUrl + attribs['src']; + } + if (name === 'div' && attribs['class'] === 'moreless'){ + isSummary = true; + } + if (name === 'li' && attribs['title']?.includes('Original status')){ + isStatus = true; + } + if (name === 'input' && attribs['name'] === 'newsid'){ + novelId = attribs['value'] + } + }, + onopentagname(name){ + if (isSummary && name === 'br'){ + novel.summary += `/n`; + } + if (isStatus && name === 'a'){ + isStatusText = true; + } + if (isGenres && name === 'a'){ + isGenresText = true; + } + }, + onattribute(name,value){ + if (name === 'itemprop' && value === 'creator'){ + isAuthor = true; + } + if (name === 'id' && value === 'mc-fs-genre'){ + isGenres = true; + } + }, + ontext(data){ + if(isAuthor){ + novel.author = data; + } + if(isSummary){ + novel.summary += data; + } + if(isStatusText){ + novel.status = data === 'Ongoing' + ? 'Ongoing' + : 'Completed'; + } + if(isGenresText){ + genreArray.push(data); + } + }, + onclosetag(name){ + if(name === 'a'){ + isCover = false; + isAuthor = false; + isStatusText = false; + isGenresText = false; + isStatus = false; + } + if(name === 'div'){ + isSummary = false; + isGenres = false; + } + } + }) + parser.write(html); + parser.end(); + novel.genres = genreArray.join(', '); - const chaptersHtmlToString = await fetchApi(chapterListUrl).then(r => + const chapterListUrl = this.site + '/chapters/' + novelId + '/'; + const chaptersHtml = await fetchApi(chapterListUrl).then(r => r.text(), ); - const pageCheerio = parseHTML(chaptersHtmlToString); + let dataJson: { + pages_count: string, + chapters: ChapterEntry[] + } = { pages_count: '', chapters : [] }; + let isScript = false; + const parser2 = new Parser ({ + ontext(data){ + if(isScript){ + if (data.includes('window.__DATA__ =')){ + dataJson = JSON.parse( + data.replace('window.__DATA__ =', '') + ); + } + } + }, + onclosetag(name){ + if(name === 'main'){ + isScript = true; + } + if (name === 'script'){ + isScript = false; + } + }, + }) + parser2.write(chaptersHtml); + parser2.end(); - const json = pageCheerio('#dle-content main').next().html()!; - const data = JSON.parse(json.replace('window.__DATA__ =', '')); - novel.totalPages = Number(data.pages_count); - novel.chapters = this.parseChapters(data); + novel.totalPages = Number(dataJson.pages_count); + novel.chapters = this.parseChapters(dataJson); - const latestChapterUrl = data.chapters[0].link; - const latestChapterName = data.chapters[0].title; - const latestChapterDate = data.chapters[0].date; + const latestChapterUrl = dataJson.chapters[0].link; + const latestChapterName = dataJson.chapters[0].title; + const latestChapterDate = dataJson.chapters[0].date; novel.latestChapter = latestChapterUrl ? { @@ -138,10 +240,33 @@ class RanobesPlugin implements Plugin.PagePlugin { const firstUrl = this.site + '/chapters' + pagePath.replace('novels/', ''); const pageUrl = firstUrl + '/page/' + page; const pageBody = await fetchApi(pageUrl).then(r => r.text()); - const loadedCheerio = parseHTML(pageBody); - const json = loadedCheerio('#dle-content main').next().html()!; - const data = JSON.parse(json.replace('window.__DATA__ =', '')); - const chapters = this.parseChapters(data); + let isScript = false; + let dataJson: { + pages_count: string, + chapters: ChapterEntry[] + } = { pages_count: '', chapters : [] }; + const parser = new Parser({ + ontext(data){ + if(isScript){ + if (data.includes('window.__DATA__ =')){ + dataJson = JSON.parse( + data.replace('window.__DATA__ =', '') + ); + } + } + }, + onclosetag(name){ + if(name === 'main'){ + isScript = true; + } + if (name === 'script'){ + isScript = false; + } + } + }) + parser.write(pageBody); + parser.end(); + const chapters = this.parseChapters(dataJson); return { chapters, }; @@ -149,11 +274,99 @@ class RanobesPlugin implements Plugin.PagePlugin { async parseChapter(chapterPath: string): Promise { const result = await fetchApi(this.site + chapterPath); - const body = await result.text(); - - const loadedCheerio = parseHTML(body); - - const chapterText = loadedCheerio('#arrticle').html() || ''; + const html = await result.text(); + let chapterText = ''; + let isChapter = false; + let isPtag = false; + let isStyleText = false; + const parser = new Parser({ + onopentag(name,attribs){ + if (isChapter && name === 'div'){ + let stylediv = attribs['style'] + if(stylediv){ + chapterText += `
` + isStyleText = true; + } else { + chapterText += `
` + } + } + if (isChapter && name === 'table'){ + let w = attribs['width']; + if(w) { + chapterText += ``; + } else { + chapterText += `
`; + } + } + if (isChapter && name === 'tbody'){ + chapterText += ``; + } + if (isChapter && name === 'tr'){ + chapterText += ``; + } + if (isChapter && name === 'td'){ + let w1 = attribs['width']; + if(w1) { + chapterText += ``; + } + if (isChapter && name === 'tr'){ + chapterText += ``; + } + if (isChapter && name === 'tbody'){ + chapterText += ``; + } + if (isChapter && name === 'table'){ + chapterText += `
`; + } else { + chapterText += ``; + } + } + }, + onattribute(name, value){ + if (name === 'id' && value === 'arrticle'){ + isChapter = true; + } + if (name === 'class' && value === 'category grey ellipses'){ + isChapter = false; + isPtag = false; + } + }, + onopentagname(name){ + if (isChapter && name === 'p'){ + chapterText += '

'; + isPtag = true; + if(isStyleText){ + isStyleText = false; + } + } + if (isChapter && name === 'br'){ + chapterText += `
`; + } + }, + ontext(data){ + if(isPtag){ + chapterText += data; + } + if(isStyleText){ + chapterText += data; + } + }, + onclosetag(name){ + if (name === 'p'){ + isPtag = false; + chapterText += '

' + } + if (isChapter && name === 'td'){ + chapterText += `
`; + } + if (isChapter && name === 'div'){ + isStyleText = false; + chapterText += `
` + } + } + }) + parser.write(html) + parser.end(); return chapterText; } @@ -163,15 +376,21 @@ class RanobesPlugin implements Plugin.PagePlugin { page: number, ): Promise { let link = `${this.site}/search/${searchTerm}/page/${page}`; - const body = await fetchApi(link).then(r => r.text()); - const loadedCheerio = parseHTML(body); - return this.parseNovels(loadedCheerio); + return this.parseNovels(body); } + async fetchImage(url: string): Promise { return await fetchFile(url); } } export default new RanobesPlugin(); + +interface ChapterEntry { + id: number; + title: string; + date: string; + link: string; +} \ No newline at end of file diff --git a/plugins/english/royalroad.ts b/plugins/english/royalroad.ts index 00216d8..11afdc9 100644 --- a/plugins/english/royalroad.ts +++ b/plugins/english/royalroad.ts @@ -1,34 +1,56 @@ -import { CheerioAPI, load as parseHTML } from 'cheerio'; +import { Parser } from 'htmlparser2'; import { fetchApi, fetchFile } from '@libs/fetch'; import { Plugin } from '@typings/plugin'; import { Filters, FilterTypes } from '@libs/filterInputs'; +import { NovelStatus } from '@libs/novelStatus'; class RoyalRoad implements Plugin.PluginBase { id = 'royalroad'; name = 'Royal Road'; - version = '1.0.1'; + version = '2.0.0'; icon = 'src/en/royalroad/icon.png'; site = 'https://www.royalroad.com/'; - parseNovels(loadedCheerio: CheerioAPI) { + parseNovels(html: string) { const novels: Plugin.NovelItem[] = []; - - loadedCheerio('.fiction-list-item').each((idx, ele) => { - const novelName = loadedCheerio(ele).find('a.bold').text(); - const novelCover = loadedCheerio(ele).find('img').attr('src'); - const novelUrl = loadedCheerio(ele).find('a.bold').attr('href')?.slice(1); - - if (!novelUrl) return; - - const novel = { - name: novelName, - cover: novelCover, - path: novelUrl, - }; - - novels.push(novel); - }); - + let tempNovel = {} as Plugin.NovelItem; + tempNovel.name = ''; + let isParsingNovel = false; + let isNovelName = false; + const parser = new Parser({ + onopentag(name, attribs){ + if (attribs['class']?.includes('fiction-list-item')){ + isParsingNovel = true; + } + if (isParsingNovel){ + if (name === 'a' && attribs['class']?.includes('bold')){ + tempNovel.path = attribs['href'].slice(1); + isNovelName = true; + } + if (name === 'img'){ + tempNovel.cover = attribs['src']; + } + if (tempNovel.path && tempNovel.name){ + novels.push(tempNovel); + tempNovel = {} as Plugin.NovelItem; + tempNovel.name = ''; + } + } + }, + ontext(data){ + if(isNovelName){ + tempNovel.name += data; + } + }, + onclosetag(name){ + if(name === 'h2'){ + isNovelName = false; + isParsingNovel = false; + } + } + }) + parser.write(html); + parser.end(); return novels; } @@ -42,45 +64,136 @@ class RoyalRoad implements Plugin.PluginBase { link += `?page=${page}`; if (filters.genre.value !== '') link += `&genre=${filters.genre.value}`; - + const body = await fetchApi(link).then(r => r.text()); - - const loadedCheerio = parseHTML(body); - return this.parseNovels(loadedCheerio); + + return this.parseNovels(body); } async parseNovel(novelPath: string): Promise { const result = await fetchApi(this.site + novelPath); - const body = await result.text(); - - const loadedCheerio = parseHTML(body); - + const html = await result.text(); const novel: Plugin.SourceNovel = { path: novelPath, - name: loadedCheerio('h1').text() || 'Untitled', - cover: loadedCheerio('.cover-art-container img').attr('src'), - author: loadedCheerio('h4 a').text(), - summary: loadedCheerio('.description').text().trim(), - status: loadedCheerio('span.label-sm:last').text().trim(), + name: '', + summary: '', chapters: [], - }; - - novel.genres = loadedCheerio('span.tags a') - .map((i, el) => loadedCheerio(el).text()) - .toArray() - .join(','); - - const chapterJson = JSON.parse( - loadedCheerio('script:contains("window.chapters")') - .html() - ?.match(/window.chapters = (.+])(?=;)/)![1] || '', - ); - - const volumeJson = JSON.parse( - loadedCheerio('script:contains("window.chapters")') - .html() - ?.match(/window.volumes = (.+])(?=;)/)![1] || '', - ); + } + let isNovelName = false; + let isAuthorName = false; + let isDescription = false; + let isH4 = false; + let isSpan = 0 + let isTags = false; + let isGenres = false; + let genreArray: string[] = [] + let isFooter = false; + let isScript = false; + let chapterJson: ChapterEntry[] = []; + let volumeJson: VolumeEntry[] = []; + const parser = new Parser ({ + onopentag(name,attribs){ + if (name === 'img' && attribs['class']?.includes('thumbnail')){ + novel.cover = attribs['src']; + } + if (name === 'span' && attribs['class']?.includes('label-sm')){ + isSpan++; + } + if (name === 'span' && attribs['class']?.includes('tags')){ + isTags = true; + } + }, + onopentagname(name){ + if (name === 'h1'){ + isNovelName = true; + } + if (isH4 && name === 'a'){ + isAuthorName = true; + } + if(isTags && name === 'a'){ + isGenres = true; + } + if (name === 'label'){ + isDescription = false; + isTags = false; + } + if (isFooter && name === 'script'){ + isScript = true; + } + }, + onattribute(name,value){ + if (name === 'class' && value === 'description'){ + isDescription = true; + } + if (name === 'class' && value === 'page-footer footer'){ + isFooter = true; + } + }, + ontext(data){ + if (isNovelName){ + novel.name = data; + } + if (isAuthorName){ + novel.author = data; + isAuthorName = false; + } + if (isDescription){ + novel.summary += data; + } + if (isSpan === 2){ + novel.status = data.trim(); + isSpan++; + } + if (isGenres){ + genreArray.push(data); + } + if (isScript){ + if (data.includes('window.chapters =')){ + chapterJson = JSON.parse( + data.match(/window.chapters = (.+])(?=;)/)![1] + ) + volumeJson = JSON.parse( + data.match(/window.volumes = (.+])(?=;)/)![1] + ) + } + } + }, + onclosetag(name){ + if (name === 'h1'){ + isNovelName = false; + isH4 = true; + } + if (name === 'h4'){ + isH4 = false; + } + if (name === 'a'){ + isGenres = false; + } + if (name === 'script'){ + isScript = false; + } + if (name === 'body'){ + isFooter = false; + } + } + }) + parser.write(html); + parser.end(); + novel.summary = novel.summary?.trim(); + novel.genres = genreArray.join(', '); + switch(novel.status){ + case 'ONGOING': + novel.status = NovelStatus.Ongoing; + break; + case 'HIATUS': + novel.status = NovelStatus.OnHiatus; + break; + case 'COMPLETED': + novel.status = NovelStatus.Completed; + break; + default: + novel.status = NovelStatus.Unknown; + } const chapter: Plugin.ChapterItem[] = chapterJson.map( (chapter: ChapterEntry) => { @@ -92,7 +205,7 @@ class RoyalRoad implements Plugin.PluginBase { path: chapter.url.slice(1), releaseTime: chapter.date, chapterNumber: chapter?.order, - page: matchingVolume ? matchingVolume.title : null, + page: matchingVolume?.title, }; }, ); @@ -103,11 +216,115 @@ class RoyalRoad implements Plugin.PluginBase { async parseChapter(chapterPath: string): Promise { const result = await fetchApi(this.site + chapterPath); - const body = await result.text(); - - const loadedCheerio = parseHTML(body); - - const chapterText = loadedCheerio('.chapter-content').html() || ''; + const html = await result.text(); + let chapterText = ''; + let isChapter = false; + let isPtag = false; + let isStyle = false; + let isStyleText = false; + let styles = ''; + const parser = new Parser({ + onopentag(name,attribs){ + if (isChapter && name === 'div'){ + let stylediv = attribs['style'] + if(stylediv){ + chapterText += `
` + isStyleText = true; + } else { + chapterText += `
` + } + } + if (isChapter && name === 'table'){ + let w = attribs['width']; + if(w) { + chapterText += ``; + } else { + chapterText += `
`; + } + } + if (isChapter && name === 'tbody'){ + chapterText += ``; + } + if (isChapter && name === 'tr'){ + chapterText += ``; + } + if (isChapter && name === 'td'){ + let w1 = attribs['width']; + if(w1) { + chapterText += ``; + } + if (isChapter && name === 'tr'){ + chapterText += ``; + } + if (isChapter && name === 'tbody'){ + chapterText += ``; + } + if (isChapter && name === 'table'){ + chapterText += `
`; + } else { + chapterText += ``; + } + } + }, + onattribute(name, value){ + if (name === 'class' && value === 'chapter-inner chapter-content'){ + isChapter = true; + } + if (name === 'class' && value === 'portlet light t-center-3'){ + isChapter = false; + isPtag = false; + } + if (name === 'class' && value === styles){ + isPtag = false; + } + }, + onopentagname(name){ + if (isChapter && name === 'p'){ + chapterText += '

'; + isPtag = true; + if(isStyleText){ + isStyleText = false; + } + } + if (name === 'style'){ + isStyle = true; + } + if (isChapter && name === 'br'){ + chapterText += `
`; + } + }, + ontext(data){ + if(isPtag){ + chapterText += data; + } + if(isStyleText){ + chapterText += data; + } + if (isStyle){ + if (data.includes('display: none;')){ + styles = data.match(/\.(.*)\{/)![1]; + } + } + }, + onclosetag(name){ + if (name === 'p'){ + isPtag = false; + chapterText += '

' + } + if (name === 'style'){ + isStyle = false; + } + if (isChapter && name === 'td'){ + chapterText += `
`; + } + if (isChapter && name === 'div'){ + isStyleText = false; + chapterText += `
` + } + } + }) + parser.write(html) + parser.end(); return chapterText; } @@ -118,11 +335,8 @@ class RoyalRoad implements Plugin.PluginBase { ): Promise { const searchUrl = `${this.site}fictions/search?page=${page}&title=${searchTerm}`; - const result = await fetchApi(searchUrl); - const body = await result.text(); - - const loadedCheerio = parseHTML(body); - return this.parseNovels(loadedCheerio); + const body = await fetchApi(searchUrl).then(r => r.text()); + return this.parseNovels(body); } async fetchImage(url: string): Promise {