From 02e7bd06a8659e623288150360d0d3b2d6fa6314 Mon Sep 17 00:00:00 2001 From: Rider21 <58046032+Rider21@users.noreply.github.com> Date: Wed, 27 Nov 2024 04:53:09 +0300 Subject: [PATCH] feat: ranobes multisrc (#1336) * track * ranobes --- .../en => multisrc/ranobes}/ranobes/icon.png | Bin scripts/multisrc/ranobes/generator.js | 31 ++ scripts/multisrc/ranobes/sources.json | 20 + scripts/multisrc/ranobes/template.ts | 453 ++++++++++++++++++ src/plugins/english/ranobes.ts | 377 --------------- 5 files changed, 504 insertions(+), 377 deletions(-) rename public/static/{src/en => multisrc/ranobes}/ranobes/icon.png (100%) create mode 100644 scripts/multisrc/ranobes/generator.js create mode 100644 scripts/multisrc/ranobes/sources.json create mode 100644 scripts/multisrc/ranobes/template.ts delete mode 100644 src/plugins/english/ranobes.ts diff --git a/public/static/src/en/ranobes/icon.png b/public/static/multisrc/ranobes/ranobes/icon.png similarity index 100% rename from public/static/src/en/ranobes/icon.png rename to public/static/multisrc/ranobes/ranobes/icon.png diff --git a/scripts/multisrc/ranobes/generator.js b/scripts/multisrc/ranobes/generator.js new file mode 100644 index 0000000..07229b7 --- /dev/null +++ b/scripts/multisrc/ranobes/generator.js @@ -0,0 +1,31 @@ +import list from './sources.json' with { type: 'json' }; +import { readFileSync } from 'fs'; +import { fileURLToPath } from 'url'; +import { dirname, join } from 'path'; + +const folder = dirname(fileURLToPath(import.meta.url)); + +export const generateAll = function () { + return list.map(metadata => { + console.log(`[ranobes]: Generating`, metadata.id); + return generator(metadata); + }); +}; + +const generator = function generator(metadata) { + const RanobesTemplate = readFileSync(join(folder, 'template.ts'), { + encoding: 'utf-8', + }); + + const pluginScript = ` + ${RanobesTemplate} +const plugin = new RanobesPlugin(${JSON.stringify(metadata)}); +export default plugin; + `.trim(); + + return { + lang: metadata.options.lang, + filename: metadata.sourceName, + pluginScript, + }; +}; diff --git a/scripts/multisrc/ranobes/sources.json b/scripts/multisrc/ranobes/sources.json new file mode 100644 index 0000000..29f2d9a --- /dev/null +++ b/scripts/multisrc/ranobes/sources.json @@ -0,0 +1,20 @@ +[ + { + "id": "ranobes", + "sourceSite": "https://ranobes.top", + "sourceName": "Ranobes", + "options": { + "lang": "English", + "path": "novels" + } + }, + { + "id": "ranobes-ru", + "sourceSite": "https://ranobes.com", + "sourceName": "Ranobes (RU)", + "options": { + "lang": "Russian", + "path": "ranobe" + } + } +] \ No newline at end of file diff --git a/scripts/multisrc/ranobes/template.ts b/scripts/multisrc/ranobes/template.ts new file mode 100644 index 0000000..238f9ce --- /dev/null +++ b/scripts/multisrc/ranobes/template.ts @@ -0,0 +1,453 @@ +import { Parser } from 'htmlparser2'; +import { fetchApi } from '@libs/fetch'; +import { Plugin } from '@typings/plugin'; +import { NovelStatus } from '@libs/novelStatus'; + +type RanobesOptions = { + lang?: string; + path: string; +}; + +export type RanobesMetadata = { + id: string; + sourceSite: string; + sourceName: string; + options?: RanobesOptions; +}; + +class RanobesPlugin implements Plugin.PluginBase { + id: string; + name: string; + icon: string; + site: string; + version: string; + options: RanobesOptions; + + constructor(metadata: RanobesMetadata) { + this.id = metadata.id; + this.name = metadata.sourceName; + this.icon = 'multisrc/ranobes/ranobes/icon.png'; + this.site = metadata.sourceSite; + this.version = '2.0.1'; + this.options = metadata.options as RanobesOptions; + } + + parseNovels(html: string) { + const novels: Plugin.NovelItem[] = []; + let tempNovel = {} as Plugin.NovelItem; + tempNovel.name = ''; + const baseUrl = this.site; + let isParsingNovel = false; + let isTitleTag = false; + let isNovelName = false; + const parser = new Parser({ + onopentag(name, attribs) { + if (attribs['class']?.includes('short-cont')) { + isParsingNovel = true; + } + if (isParsingNovel) { + if (name === 'h2' && attribs['class']?.includes('title')) { + isTitleTag = true; + } + if (isTitleTag && name === 'a') { + tempNovel.path = attribs['href'].slice(baseUrl.length); + isNovelName = true; + } + if (name === 'figure') { + tempNovel.cover = attribs['style'].replace( + /.*url\((.*?)\)./g, + '$1', + ); + } + if (tempNovel.path && tempNovel.cover) { + novels.push(tempNovel); + tempNovel = {} as Plugin.NovelItem; + tempNovel.name = ''; + } + } + }, + ontext(data) { + if (isNovelName) { + tempNovel.name += data; + } + }, + onclosetag(name) { + if (name === 'h2') { + isNovelName = false; + isTitleTag = false; + } + if (name === 'figure') { + isParsingNovel = false; + } + }, + }); + parser.write(html); + parser.end(); + return novels; + } + + parseChapters(data: { chapters: ChapterEntry[] }) { + const chapter: Plugin.ChapterItem[] = []; + data.chapters.map((item: ChapterEntry) => { + chapter.push({ + name: item.title, + releaseTime: new Date(item.date).toISOString(), + path: item.link.slice(this.site.length), + }); + }); + return chapter.reverse(); + } + + parseDate = (date: string) => { + const now = new Date(); + if (!date) return now.toISOString(); + if (this.id === 'ranobes-ru') { + if (date.includes(' в ')) return date.replace(' в ', ' г., '); + + const [when, time] = date.split(', '); + if (!time) return now.toISOString(); + const [h, m] = time.split(':'); + + switch (when) { + case 'Сегодня': + now.setHours(parseInt(h, 10)); + now.setMinutes(parseInt(m, 10)); + break; + case 'Вчера': + now.setDate(now.getDate() - 1); + now.setHours(parseInt(h, 10)); + now.setMinutes(parseInt(m, 10)); + break; + default: + return now.toISOString(); + } + } else { + const [num, xz, ago] = date.split(' '); + if (ago !== 'ago') return now.toISOString(); + + switch (xz) { + case 'minutes': + now.setMinutes(parseInt(num, 10)); + break; + case 'hour': + case 'hours': + now.setHours(parseInt(num, 10)); + break; + case 'day': + case 'days': + now.setDate(now.getDate() - parseInt(num, 10)); + break; + case 'month': + case 'months': + now.setMonth(now.getMonth() - parseInt(num, 10)); + break; + case 'year': + case 'years': + now.setFullYear(now.getFullYear() - parseInt(num, 10)); + break; + default: + return now.toISOString(); + } + } + return now.toISOString(); + }; + + async popularNovels(page: number): Promise { + const link = `${this.site}/${this.options.path}/page/${page}/`; + const body = await fetchApi(link).then(r => r.text()); + return this.parseNovels(body); + } + + async parseNovel( + novelPath: string, + ): Promise { + const baseUrl = this.site; + const result = await fetchApi(baseUrl + novelPath); + const html = await result.text(); + const novel: Plugin.SourceNovel & { totalPages: number } = { + path: novelPath, + name: '', + summary: '', + chapters: [], + totalPages: 1, + }; + let isCover = false; + let isAuthor = false; + let isSummary = false; + let isStatus = false; + let isStatusText = false; + let isGenres = false; + let isGenresText = false; + let isMaxChapters = false; + let isChapter = false; + let isChapterTitle = false; + let isChapterDate = false; + const genreArray: string[] = []; + const chapters: Plugin.ChapterItem[] = []; + let tempchapter: Plugin.ChapterItem = {}; + let maxChapters = 0; + const fixDate = this.parseDate; + const parser = new Parser({ + onopentag(name, attribs) { + if (attribs['class'] === 'poster') { + isCover = true; + } + if (isCover && name === 'img') { + novel.name = attribs['alt']; + novel.cover = baseUrl + attribs['src']; + } + if ( + (name === 'div' && + attribs['class'] === 'moreless cont-text showcont-h') || + (attribs['class'] === 'cont-text showcont-h' && + attribs['itemprop'] === 'description') + ) { + isSummary = true; + } + if ( + name === 'li' && + attribs['title'] && + (attribs['title'].includes('Original status') || + attribs['title'].includes('Статус оригинала')) + ) { + isStatus = true; + } + if (name === 'a' && attribs['rel'] === 'chapter') { + isChapter = true; + tempchapter.path = attribs['href'].replace(baseUrl, ''); + } + if ( + isChapter && + name === 'span' && + attribs['class'] === 'title ellipses' + ) { + isChapterTitle = true; + } + if (isChapter && name === 'span' && attribs['class'] === 'grey') { + isChapterDate = true; + } + if ( + name === 'li' && + (attribs['title'] == + 'Glossary + illustrations + division of chapters, etc.' || + attribs['title'] === + 'Глоссарий + иллюстраций + разделение глав и т.д.') + ) { + isMaxChapters = true; + } + }, + onopentagname(name) { + if (isSummary && name === 'br') { + novel.summary += '\n'; + } + if (isStatus && name === 'a') { + isStatusText = true; + } + if (isGenres && name === 'a') { + isGenresText = true; + } + }, + onattribute(name, value) { + if (name === 'itemprop' && value === 'creator') { + isAuthor = true; + } + if (name === 'id' && value === 'mc-fs-genre') { + isGenres = true; + } + }, + ontext(data) { + if (isAuthor) { + novel.author = data; + } + if (isSummary) { + novel.summary += data.trim(); + } + if (isStatusText) { + novel.status = + data === 'Ongoing' || data == 'В процессе' + ? NovelStatus.Ongoing + : NovelStatus.Completed; + } + if (isGenresText) { + genreArray.push(data); + } + if (isMaxChapters) { + const isNumber = data.replace(/\D/g, ''); + if (isNumber) { + maxChapters = parseInt(isNumber, 10); + } + } + if (isChapter) { + if (isChapterTitle) tempchapter.name = data.trim(); + if (isChapterDate) tempchapter.releaseTime = fixDate(data.trim()); + } + }, + onclosetag(name) { + if (name === 'a') { + isCover = false; + isAuthor = false; + isStatusText = false; + isGenresText = false; + isStatus = false; + } + if (name === 'div') { + isSummary = false; + isGenres = false; + } + if (name === 'li') { + isMaxChapters = false; + } + if (name === 'a') { + isChapter = false; + if (tempchapter.name) { + chapters.push({ ...tempchapter, page: '1' }); + tempchapter = {}; + } + } + if (name === 'span') { + if (isChapterTitle) isChapterTitle = false; + if (isChapterDate) isChapterDate = false; + } + }, + }); + parser.write(html); + parser.end(); + novel.genres = genreArray.join(', '); + novel.totalPages = Math.ceil((maxChapters || 1) / 25); + novel.chapters = chapters; + + if (novel.chapters[0].path) { + novel.latestChapter = novel.chapters[0]; + } + + return novel; + } + + async parsePage(novelPath: string, page: string): Promise { + const pagePath = + this.id == 'ranobes' + ? novelPath.split('-')[0] + : '/' + novelPath.split('-').slice(1).join('-').split('.')[0]; + const firstUrl = + this.site + '/chapters' + pagePath.replace(this.options.path + '/', ''); + const pageBody = await fetchApi(firstUrl + '/page/' + page).then(r => + r.text(), + ); + + const baseUrl = this.site; + let isScript = false; + let isChapter = false; + let isChapterInfo = false; + let isChapterDate = false; + + let chapters: Plugin.ChapterItem[] = []; + let tempchapter: Plugin.ChapterItem = {}; + const fixDate = this.parseDate; + + let dataJson: { + pages_count: string; + chapters: ChapterEntry[]; + } = { pages_count: '', chapters: [] }; + + const parser = new Parser({ + onopentag(name, attribs) { + if (name === 'div' && attribs['class'] === 'cat_block cat_line') { + isChapter = true; + } + if (isChapter && name === 'a' && attribs['title'] && attribs['href']) { + tempchapter.name = attribs['title']; + tempchapter.path = attribs['href'].replace(baseUrl, ''); + } + if (name === 'span' && attribs['class'] === 'grey small') { + isChapterInfo = true; + } + if (name === 'small' && isChapterInfo) { + isChapterDate = true; + } + }, + ontext(data) { + if (isChapterDate) tempchapter.releaseTime = fixDate(data.trim()); + if (isScript) { + if (data.includes('window.__DATA__ =')) { + dataJson = JSON.parse(data.replace('window.__DATA__ =', '')); + } + } + }, + onclosetag(name) { + if (name === 'a' && tempchapter.name) { + chapters.push(tempchapter); + tempchapter = {}; + } + if (name === 'div') { + isChapter = false; + } + if (name === 'span') { + isChapterInfo = false; + } + if (name === 'small') { + isChapterDate = false; + } + if (name === 'main') { + isScript = true; + } + if (name === 'script') { + isScript = false; + } + }, + }); + parser.write(pageBody); + parser.end(); + + if (dataJson.chapters?.length) { + chapters = this.parseChapters(dataJson); + } + + return { + chapters, + }; + } + + async parseChapter(chapterPath: string): Promise { + const result = await fetchApi(this.site + chapterPath); + const html = await result.text(); + + const indexA = html.indexOf('
'); + const indexB = html.indexOf('
', indexA); + + const chapterText = html.substring(indexA, indexB); + return chapterText; + } + + async searchNovels( + searchTerm: string, + page: number, + ): Promise { + let html; + if (this.id === 'ranobes-ru') { + html = await fetchApi(this.site + '/index.php?do=search', { + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + Referer: this.site + '/', + }, + method: 'POST', + body: new URLSearchParams({ + do: 'search', + subaction: 'search', + search_start: page.toString(), + story: searchTerm, + }).toString(), + }).then(res => res.text()); + } else { + const link = `${this.site}/search/${searchTerm}/page/${page}`; + html = await fetchApi(link).then(r => r.text()); + } + return this.parseNovels(html); + } +} + +type ChapterEntry = { + id: number; + title: string; + date: string; + link: string; +}; diff --git a/src/plugins/english/ranobes.ts b/src/plugins/english/ranobes.ts deleted file mode 100644 index 4d21fe9..0000000 --- a/src/plugins/english/ranobes.ts +++ /dev/null @@ -1,377 +0,0 @@ -import { Parser } from 'htmlparser2'; -import { fetchApi } from '@libs/fetch'; -import { Plugin } from '@typings/plugin'; - -class RanobesPlugin implements Plugin.PagePlugin { - id = 'ranobes'; - name = 'Ranobes'; - icon = 'src/en/ranobes/icon.png'; - site = 'https://ranobes.top'; - version = '2.0.0'; - - async sleep(ms: number) { - return new Promise(resolve => setTimeout(resolve, ms)); - } - - parseNovels(html: string) { - const novels: Plugin.NovelItem[] = []; - let tempNovel = {} as Plugin.NovelItem; - tempNovel.name = ''; - const baseUrl = this.site; - let isParsingNovel = false; - let isTitleTag = false; - let isNovelName = false; - const parser = new Parser({ - onopentag(name, attribs) { - if (attribs['class']?.includes('short-cont')) { - isParsingNovel = true; - } - if (isParsingNovel) { - if (name === 'h2' && attribs['class']?.includes('title')) { - isTitleTag = true; - } - if (isTitleTag && name === 'a') { - tempNovel.path = attribs['href'].slice(baseUrl.length); - isNovelName = true; - } - if (name === 'figure') { - tempNovel.cover = attribs['style'].replace( - /.*url\((.*?)\)./g, - '$1', - ); - } - if (tempNovel.path && tempNovel.cover) { - novels.push(tempNovel); - tempNovel = {} as Plugin.NovelItem; - tempNovel.name = ''; - } - } - }, - ontext(data) { - if (isNovelName) { - tempNovel.name += data; - } - }, - onclosetag(name) { - if (name === 'h2') { - isNovelName = false; - isTitleTag = false; - } - if (name === 'figure') { - isParsingNovel = false; - } - }, - }); - parser.write(html); - parser.end(); - return novels; - } - - parseChapters(data: { chapters: ChapterEntry[] }) { - const chapter: Plugin.ChapterItem[] = []; - data.chapters.map((item: ChapterEntry) => { - chapter.push({ - name: item.title, - releaseTime: new Date(item.date).toISOString(), - path: item.link.slice(this.site.length), - }); - }); - return chapter.reverse(); - } - - async popularNovels(page: number): Promise { - const link = `${this.site}/novels/page/${page}/`; - const body = await fetchApi(link).then(r => r.text()); - - return this.parseNovels(body); - } - - async parseNovel( - novelPath: string, - ): Promise { - const url = this.site + novelPath; - const result = await fetchApi(url); - const html = await result.text(); - const novel: Plugin.SourceNovel & { totalPages: number } = { - path: novelPath, - name: '', - summary: '', - chapters: [], - totalPages: 1, - }; - const baseUrl = this.site; - let isCover = false; - let isAuthor = false; - let isSummary = false; - let isStatus = false; - let isStatusText = false; - let isGenres = false; - let isGenresText = false; - const genreArray: string[] = []; - let novelId = ''; - const parser = new Parser({ - onopentag(name, attribs) { - if (attribs['class'] === 'poster') { - isCover = true; - } - if (isCover && name === 'img') { - novel.name = attribs['alt']; - novel.cover = baseUrl + attribs['src']; - } - if (name === 'div' && attribs['class'] === 'moreless') { - isSummary = true; - } - if (name === 'li' && attribs['title']?.includes('Original status')) { - isStatus = true; - } - if (name === 'input' && attribs['name'] === 'newsid') { - novelId = attribs['value']; - } - }, - onopentagname(name) { - if (isSummary && name === 'br') { - novel.summary += `/n`; - } - if (isStatus && name === 'a') { - isStatusText = true; - } - if (isGenres && name === 'a') { - isGenresText = true; - } - }, - onattribute(name, value) { - if (name === 'itemprop' && value === 'creator') { - isAuthor = true; - } - if (name === 'id' && value === 'mc-fs-genre') { - isGenres = true; - } - }, - ontext(data) { - if (isAuthor) { - novel.author = data; - } - if (isSummary) { - novel.summary += data; - } - if (isStatusText) { - novel.status = data === 'Ongoing' ? 'Ongoing' : 'Completed'; - } - if (isGenresText) { - genreArray.push(data); - } - }, - onclosetag(name) { - if (name === 'a') { - isCover = false; - isAuthor = false; - isStatusText = false; - isGenresText = false; - isStatus = false; - } - if (name === 'div') { - isSummary = false; - isGenres = false; - } - }, - }); - parser.write(html); - parser.end(); - novel.genres = genreArray.join(', '); - - const chapterListUrl = this.site + '/chapters/' + novelId + '/'; - const chaptersHtml = await fetchApi(chapterListUrl).then(r => r.text()); - let dataJson: { - pages_count: string; - chapters: ChapterEntry[]; - } = { pages_count: '', chapters: [] }; - let isScript = false; - const parser2 = new Parser({ - ontext(data) { - if (isScript) { - if (data.includes('window.__DATA__ =')) { - dataJson = JSON.parse(data.replace('window.__DATA__ =', '')); - } - } - }, - onclosetag(name) { - if (name === 'main') { - isScript = true; - } - if (name === 'script') { - isScript = false; - } - }, - }); - parser2.write(chaptersHtml); - parser2.end(); - - novel.totalPages = Number(dataJson.pages_count); - novel.chapters = this.parseChapters(dataJson); - - const latestChapterUrl = dataJson.chapters[0].link; - const latestChapterName = dataJson.chapters[0].title; - const latestChapterDate = dataJson.chapters[0].date; - - novel.latestChapter = latestChapterUrl - ? { - path: latestChapterUrl.replace(this.site, ''), - name: latestChapterName, - releaseTime: new Date(latestChapterDate).toISOString(), - } - : undefined; - return novel; - } - - async parsePage(novelPath: string, page: string): Promise { - const pagePath = novelPath.split('-')[0]; - const firstUrl = this.site + '/chapters' + pagePath.replace('novels/', ''); - const pageUrl = firstUrl + '/page/' + page; - const pageBody = await fetchApi(pageUrl).then(r => r.text()); - let isScript = false; - let dataJson: { - pages_count: string; - chapters: ChapterEntry[]; - } = { pages_count: '', chapters: [] }; - const parser = new Parser({ - ontext(data) { - if (isScript) { - if (data.includes('window.__DATA__ =')) { - dataJson = JSON.parse(data.replace('window.__DATA__ =', '')); - } - } - }, - onclosetag(name) { - if (name === 'main') { - isScript = true; - } - if (name === 'script') { - isScript = false; - } - }, - }); - parser.write(pageBody); - parser.end(); - const chapters = this.parseChapters(dataJson); - return { - chapters, - }; - } - - async parseChapter(chapterPath: string): Promise { - const result = await fetchApi(this.site + chapterPath); - const html = await result.text(); - let chapterText = ''; - let isChapter = false; - let isPtag = false; - let isStyleText = false; - const parser = new Parser({ - onopentag(name, attribs) { - if (isChapter && name === 'div') { - const stylediv = attribs['style']; - if (stylediv) { - chapterText += `
`; - isStyleText = true; - } else { - chapterText += `
`; - } - } - if (isChapter && name === 'table') { - const w = attribs['width']; - if (w) { - chapterText += ``; - } else { - chapterText += `
`; - } - } - if (isChapter && name === 'tbody') { - chapterText += ``; - } - if (isChapter && name === 'tr') { - chapterText += ``; - } - if (isChapter && name === 'td') { - const w1 = attribs['width']; - if (w1) { - chapterText += ``; - } - if (isChapter && name === 'tr') { - chapterText += ``; - } - if (isChapter && name === 'tbody') { - chapterText += ``; - } - if (isChapter && name === 'table') { - chapterText += `
`; - } else { - chapterText += ``; - } - } - }, - onattribute(name, value) { - if (name === 'id' && value === 'arrticle') { - isChapter = true; - } - if (name === 'class' && value === 'category grey ellipses') { - isChapter = false; - isPtag = false; - } - }, - onopentagname(name) { - if (isChapter && name === 'p') { - chapterText += '

'; - isPtag = true; - if (isStyleText) { - isStyleText = false; - } - } - if (isChapter && name === 'br') { - chapterText += `
`; - } - }, - ontext(data) { - if (isPtag) { - chapterText += data; - } - if (isStyleText) { - chapterText += data; - } - }, - onclosetag(name) { - if (name === 'p') { - isPtag = false; - chapterText += '

'; - } - if (isChapter && name === 'td') { - chapterText += `
`; - } - if (isChapter && name === 'div') { - isStyleText = false; - chapterText += `
`; - } - }, - }); - parser.write(html); - parser.end(); - - return chapterText; - } - - async searchNovels( - searchTerm: string, - page: number, - ): Promise { - const link = `${this.site}/search/${searchTerm}/page/${page}`; - const body = await fetchApi(link).then(r => r.text()); - - return this.parseNovels(body); - } -} -export default new RanobesPlugin(); - -type ChapterEntry = { - id: number; - title: string; - date: string; - link: string; -};