From 4c00b2bc23f7450c7d9c7e8efdba577385e3d3dc Mon Sep 17 00:00:00 2001 From: K1ngfish3r Date: Sat, 15 Jul 2023 15:45:32 +0800 Subject: [PATCH] fix(source): Arnovel (#671) * fix arnovel * update 1stkissnovel url * fix images 1stkissnovel * QoL fixes for Agitoon and Pawread * Partial fix for sakuranovel/indowebnovel chapnames * Experimental lnp and pawread array * I know how commas work * fix inoveltranslation actual * startCase * fix bug --- src/sources/en/lightnovelpub.js | 33 +++++-------------- src/sources/en/novelhall.js | 10 +++--- src/sources/en/novelupdates.js | 10 +++++- src/sources/en/pawread.js | 31 ++++++++--------- src/sources/id/indowebnovel.js | 29 ++++++++-------- src/sources/id/sakuranovel.js | 29 ++++++++-------- src/sources/kr/Agitoon.js | 12 ++----- .../multisrc/madara/MadaraGenerator.ts | 2 +- src/sources/multisrc/madara/MadaraScraper.js | 22 +++++++++++-- .../multisrc/madara/MadaraSources.json | 2 +- 10 files changed, 86 insertions(+), 94 deletions(-) diff --git a/src/sources/en/lightnovelpub.js b/src/sources/en/lightnovelpub.js index ecb20c87..72906eae 100644 --- a/src/sources/en/lightnovelpub.js +++ b/src/sources/en/lightnovelpub.js @@ -53,43 +53,28 @@ const parseNovelAndChapters = async novelUrl => { sourceName: 'LightNovelPub', }; - novel.novelName = loadedCheerio('h1.novel-title') - .text() - .replace(/[\t\n]/g, '') - .trim(); + novel.novelName = loadedCheerio('h1.novel-title').text().trim(); novel.novelCover = loadedCheerio('figure.cover > img').attr('data-src'); - novel.genre = ''; + novel.genre = loadedCheerio('.categories li') + .find('a') + .map((i, el) => loadedCheerio(el).text()) + .toArray() + .join(','); - loadedCheerio('div.categories > ul > li').each(function () { - novel.genre += - loadedCheerio(this) - .text() - .replace(/[\t\n]/g, '') + ','; - }); - - loadedCheerio('div.header-stats > span').each(function () { - if (loadedCheerio(this).find('small').text() === 'Status') { - novel.status = loadedCheerio(this).find('strong').text(); - } - }); - - novel.genre = novel.genre.slice(0, -1); + novel.status = loadedCheerio('small:contains("Status")').prev().text().trim(); novel.author = loadedCheerio('.author > a > span').text(); + loadedCheerio('.expand').remove(); novel.summary = loadedCheerio('.summary > .content').text().trim(); const delay = ms => new Promise(res => setTimeout(res, ms)); let lastPage = 1; - lastPage = loadedCheerio( - '#novel > header > div.header-body.container > div.novel-info > div.header-stats > span:nth-child(1) > strong', - ) - .text() - ?.trim(); + lastPage = loadedCheerio('small:contains("Chapters")').prev().text().trim(); lastPage = Math.ceil(lastPage / 100); diff --git a/src/sources/en/novelhall.js b/src/sources/en/novelhall.js index b8794b4f..7f3f44f9 100644 --- a/src/sources/en/novelhall.js +++ b/src/sources/en/novelhall.js @@ -56,18 +56,16 @@ const parseNovelAndChapters = async novelUrl => { .text() ?.replace('back<<', ''); - novel.author = loadedCheerio('span.blue') - .first() + loadedCheerio('p[style="display: none;"]').remove(); + novel.author = loadedCheerio('span.blue:contains("Author")') .text() - .replace('Author：', ''); + .replace(/Author：|\n+\t+/g, ''); novel.genre = loadedCheerio('a.red').text(); novel.artist = null; - novel.status = loadedCheerio('span.blue') - .first() - .next() + novel.status = loadedCheerio('span.blue:contains("Status")') .text() .replace('Status：', ''); diff --git a/src/sources/en/novelupdates.js b/src/sources/en/novelupdates.js index 50ccb51a..045b12fc 100644 --- a/src/sources/en/novelupdates.js +++ b/src/sources/en/novelupdates.js @@ -258,7 +258,15 @@ const parseChapter = async (novelUrl, chapterUrl) => { } else if (isLightNovelsTls) { chapterText = loadedCheerio('.text_story').html(); } else if (isiNovelTranslation) { - chapterText = loadedCheerio('.chakra-skeleton').html(); + const link = 'https://api.' + result.url.slice(8); + const json = await fetchApi({ + url: link, + sourceId, + }).then(r => r.json()); + chapterText = + json.content.replace(/\n/g, '
') + + '

TL Notes:
' + + json.notes.replace(/\n/g, '
'); } else if (isWordPress) { /** * Remove wordpress bloat tags diff --git a/src/sources/en/pawread.js b/src/sources/en/pawread.js index 4a6193ec..eab6c0e4 100644 --- a/src/sources/en/pawread.js +++ b/src/sources/en/pawread.js @@ -47,34 +47,28 @@ const parseNovelAndChapters = async novelUrl => { .attr('style') .replace(/.*\((.*?)\)/g, '$1'); - novel.author = loadedCheerio( - '#views_info > div:nth-child(4) > span:nth-child(2)', - ) + novel.author = loadedCheerio('#views_info > div:last > span:last') .text() - .replace(/(Author: )/g, ''); + .replace('Author: ', ''); novel.status = loadedCheerio('.label').text().trim(); - novel.genre = loadedCheerio('div.mt20:nth-child(4)') - .text() - .trim() - .replace(/\s/g, ','); + novel.genre = loadedCheerio('.col-md-9 .mt20') + .find('a') + .map((i, el) => loadedCheerio(el).text()) + .toArray() + .join(','); novel.summary = loadedCheerio('#simple-des').text().trim(); let chapters = []; - loadedCheerio('div.filtr-item').each(function () { + loadedCheerio('.item-box').each(function () { const chapterName = loadedCheerio(this).find('.c_title').text(); - const releaseDate = loadedCheerio(this) - .find('.c_title') - .next() - .next() - .text(); + const releaseDate = loadedCheerio(this).find('span:last').text(); const chapterUrl = novelUrl + loadedCheerio(this) - .find('.item-box') .attr('onclick') .replace(/.*'(.*)'.*/g, '$1') + '.html'; @@ -95,9 +89,10 @@ const parseChapter = async (novelUrl, chapterUrl) => { let loadedCheerio = cheerio.load(body); - const chapterName = loadedCheerio( - 'div.panel:nth-child(2) > div > div > h3', - ).text(); + const chapterName = loadedCheerio('.chapter-content h3').text(); + + const steal = ['bit.ly', 'tinyurl', 'pawread']; + steal.map(tag => loadedCheerio(`p:icontains(${tag})`).remove()); const chapterText = loadedCheerio('#chapter_item').html(); const chapter = { diff --git a/src/sources/id/indowebnovel.js b/src/sources/id/indowebnovel.js index 0409ed2b..26f8780e 100644 --- a/src/sources/id/indowebnovel.js +++ b/src/sources/id/indowebnovel.js @@ -1,5 +1,6 @@ import * as cheerio from 'cheerio'; import { fetchHtml } from '@utils/fetch/fetch'; +import { startCase } from 'lodash-es'; const sourceId = 87; const sourceName = 'IndoWebNovel'; @@ -50,7 +51,7 @@ const parseNovelAndChapters = async novelUrl => { novelUrl, }; - novel.novelName = loadedCheerio('.series-title h2').text().trim(); + novel.novelName = startCase(loadedCheerio('.series-title h2').text().trim()); novel.novelCover = loadedCheerio('.series-thumb img').attr('src'); @@ -78,22 +79,18 @@ const parseNovelAndChapters = async novelUrl => { let chapters = []; - loadedCheerio('.series-chapterlist li').each(function () { - const chapterName = loadedCheerio(this) - .find('a span') - .first() - .text() - .replace(/.*?(Chapter.|[0-9])/g, '$1') - .replace(/Bahasa Indonesia/g, '') - .replace(/\s+/g, ' ') - .trim(); + loadedCheerio('.series-chapterlist li a').each(function () { + let titles = startCase( + loadedCheerio(this) + .attr('title') + .replace(/Bahasa Indonesia/g, '') + .replace(/\s\s+/g, ' ') + .trim(), + ); - const releaseDate = loadedCheerio(this) - .find('a span') - .first() - .next() - .text(); - const chapterUrl = loadedCheerio(this).find('a').attr('href'); + const chapterName = titles.replace(`${novel.novelName}`, ''); + const releaseDate = loadedCheerio(this).find('span:last').text(); + const chapterUrl = loadedCheerio(this).attr('href'); chapters.push({ chapterName, releaseDate, chapterUrl }); }); diff --git a/src/sources/id/sakuranovel.js b/src/sources/id/sakuranovel.js index f8d47690..fb6becf1 100644 --- a/src/sources/id/sakuranovel.js +++ b/src/sources/id/sakuranovel.js @@ -1,5 +1,6 @@ import { fetchHtml } from '@utils/fetch/fetch'; import * as cheerio from 'cheerio'; +import { startCase } from 'lodash-es'; const sourceId = 103; const sourceName = 'SakuraNovel'; @@ -45,7 +46,7 @@ const parseNovelAndChapters = async novelUrl => { novelUrl, }; - novel.novelName = loadedCheerio('.series-title h2').text().trim(); + novel.novelName = startCase(loadedCheerio('.series-title h2').text().trim()); novel.novelCover = loadedCheerio('.series-thumb img').attr('src'); @@ -73,22 +74,18 @@ const parseNovelAndChapters = async novelUrl => { let chapters = []; - loadedCheerio('.series-chapterlist li').each(function () { - const chapterName = loadedCheerio(this) - .find('a span') - .first() - .text() - .replace(/.*?(Chapter.|[0-9])/g, '$1') - .replace(/Bahasa Indonesia/g, '') - .replace(/\s+/g, ' ') - .trim(); + loadedCheerio('.series-chapterlist li a').each(function () { + let titles = startCase( + loadedCheerio(this) + .attr('title') + .replace(/Bahasa Indonesia/g, '') + .replace(/\s\s+/g, ' ') + .trim(), + ); - const releaseDate = loadedCheerio(this) - .find('a span') - .first() - .next() - .text(); - const chapterUrl = loadedCheerio(this).find('a').attr('href'); + const chapterName = titles.replace(`${novel.novelName}`, ''); + const releaseDate = loadedCheerio(this).find('span:last').text(); + const chapterUrl = loadedCheerio(this).attr('href'); chapters.push({ chapterName, releaseDate, chapterUrl }); }); diff --git a/src/sources/kr/Agitoon.js b/src/sources/kr/Agitoon.js index 5f2d0035..3c1e7c20 100644 --- a/src/sources/kr/Agitoon.js +++ b/src/sources/kr/Agitoon.js @@ -102,16 +102,10 @@ const parseChapter = async (novelUrl, chapterUrl) => { const loadedCheerio = cheerio.load(body); const title = loadedCheerio('div > div.col-12 > h2').text(); - const contentTag = loadedCheerio('#id_wr_content > p'); - - let content = ''; - contentTag.each((_, element) => { - content += loadedCheerio(element).text(); - content += '
'; - }); + let chapterText = loadedCheerio('#id_wr_content').html(); // gets rid of the popup thingy - content = content.replace( + chapterText = chapterText.replace( '팝업메뉴는 빈공간을 더치하거나 스크룰시 사라집니다', '', ); @@ -121,7 +115,7 @@ const parseChapter = async (novelUrl, chapterUrl) => { novelUrl, chapterUrl, chapterName: title, - chapterText: content, + chapterText, }; return chapter; diff --git a/src/sources/multisrc/madara/MadaraGenerator.ts b/src/sources/multisrc/madara/MadaraGenerator.ts index 7f682544..43a36178 100644 --- a/src/sources/multisrc/madara/MadaraGenerator.ts +++ b/src/sources/multisrc/madara/MadaraGenerator.ts @@ -53,7 +53,7 @@ export const FreeNovelMeScraper = new MadaraScraper( export const FirstKissNovelScraper = new MadaraScraper( 46, - 'https://1stkissnovel.love/', + 'https://1stkissnovel.org/', 'FirstKissNovel', { 'useNewChapterEndpoint': true }, ); diff --git a/src/sources/multisrc/madara/MadaraScraper.js b/src/sources/multisrc/madara/MadaraScraper.js index 6370ba0c..d637b929 100644 --- a/src/sources/multisrc/madara/MadaraScraper.js +++ b/src/sources/multisrc/madara/MadaraScraper.js @@ -44,6 +44,14 @@ class MadaraScraper { .attr('href') .split('/')[4]; + if (sourceId === 59) { + novelUrl = loadedCheerio(this) + .find('.post-title') + .find('a') + .attr('href') + .split('/')[5]; + } + const novel = { sourceId, novelName, @@ -79,11 +87,12 @@ class MadaraScraper { novel.novelName = loadedCheerio('.post-title h1').text().trim(); novel.novelCover = + loadedCheerio('.summary_image > a > img').attr('data-lazy-src') || loadedCheerio('.summary_image > a > img').attr('data-src') || loadedCheerio('.summary_image > a > img').attr('src') || defaultCoverUri; - loadedCheerio('.post-content_item', '.post-content').each(function () { + loadedCheerio('.post-content_item, .post-content').each(function () { const detailName = loadedCheerio(this).find('h5').text().trim(); const detail = loadedCheerio(this).find('.summary-content').text().trim(); @@ -94,6 +103,7 @@ class MadaraScraper { break; case 'Author(s)': case 'المؤلف': + case 'المؤلف (ين)': novel.author = detail; break; case 'Status': @@ -106,7 +116,7 @@ class MadaraScraper { } }); - loadedCheerio('div.summary__content .code-block').remove(); + loadedCheerio('div.summary__content .code-block,script').remove(); novel.summary = loadedCheerio('div.summary__content').text().trim(); let novelChapters = []; @@ -237,6 +247,14 @@ class MadaraScraper { .attr('href') .split('/')[4]; + if (sourceId === 59) { + novelUrl = loadedCheerio(this) + .find('.post-title') + .find('a') + .attr('href') + .split('/')[5]; + } + const novel = { sourceId, novelName, diff --git a/src/sources/multisrc/madara/MadaraSources.json b/src/sources/multisrc/madara/MadaraSources.json index b849ef0c..44ec97b9 100644 --- a/src/sources/multisrc/madara/MadaraSources.json +++ b/src/sources/multisrc/madara/MadaraSources.json @@ -48,7 +48,7 @@ }, { "sourceId": 46, - "baseUrl": "https://1stkissnovel.love/", + "baseUrl": "https://1stkissnovel.org/", "sourceName": "FirstKissNovel", "options": { "useNewChapterEndpoint": true