From c30e2b5966b5189c1f2e6466946b3378a07423af Mon Sep 17 00:00:00 2001
From: Lukas Richter <53831711+Batorian@users.noreply.github.com>
Date: Mon, 29 Apr 2024 04:32:38 +0200
Subject: [PATCH] Enhancement: Novel Updates (#915)
* refactored and updated Novel Updates' parseChapter()
* added `ippotranslations` and `shanghaifantasy` to Novel Updates
* small changes
* removed switch cases and reverted back to `let` and `if`/`else if` structure
* added lots of translation group support
* added more site support and tried cleaning chapter content where possible
* quick fix
* quick fix
* logic overhaul; implementation of switch cases (again)
* small changes to `bloatClasses` and `chapterContent`
* added Anomalously Creative (`anotivereads`) to outliers and added new `bloatClasses` to Wuxia World (`wuxiaworld`)
---
plugins/english/novelupdates.ts | 394 ++++++++++++++++++++++----------
1 file changed, 276 insertions(+), 118 deletions(-)
diff --git a/plugins/english/novelupdates.ts b/plugins/english/novelupdates.ts
index 183fabe..bb96a14 100644
--- a/plugins/english/novelupdates.ts
+++ b/plugins/english/novelupdates.ts
@@ -6,7 +6,7 @@ import { Plugin } from '@typings/plugin';
class NovelUpdates implements Plugin.PluginBase {
id = 'novelupdates';
name = 'Novel Updates';
- version = '0.5.2';
+ version = '0.5.3';
icon = 'src/en/novelupdates/icon.png';
site = 'https://www.novelupdates.com/';
@@ -165,58 +165,200 @@ class NovelUpdates implements Plugin.PluginBase {
return match && `${match[1]}//${match[3]}`;
}
+ async getChapterBody(
+ loadedCheerio: CheerioAPI,
+ domain: string[],
+ url: string,
+ ) {
+ let bloatClasses = [];
+ let chapterTitle = '';
+ let chapterContent = '';
+ let chapterText = '';
+
+ const unwanted = ['blogspot', 'wordpress', 'www'];
+ const targetDomain = domain.find(d => !unwanted.includes(d));
+
+ switch (targetDomain) {
+ case 'anotivereads':
+ chapterTitle = loadedCheerio('#comic-nav-name').first().text()!;
+ chapterContent = loadedCheerio('#spliced-comic').html()!;
+ chapterText = `${chapterTitle}
${chapterContent}`;
+ break;
+ case 'asuratls':
+ let titleElementAsura = loadedCheerio('.post-body div b').first();
+ chapterTitle = titleElementAsura.first().text();
+ titleElementAsura.remove();
+ chapterContent = loadedCheerio('.post-body').html()!;
+ chapterText = `${chapterTitle}
${chapterContent}`;
+ break;
+ case 'hiraethtranslation':
+ chapterTitle = loadedCheerio('li.active').first().text()!;
+ chapterContent = loadedCheerio('.text-left').html()!;
+ chapterText = `${chapterTitle}
${chapterContent}`;
+ break;
+ case 'hostednovel':
+ chapterTitle = loadedCheerio('#chapter-title').first().text()!;
+ chapterContent = loadedCheerio('#chapter-content').html()!;
+ chapterText = `${chapterTitle}
${chapterContent}`;
+ break;
+ case 'inoveltranslation':
+ const link_inovel = 'https://api.' + url.slice(8);
+ const json_inovel = await fetchApi(link_inovel).then(r => r.json());
+ chapterTitle =
+ 'Chapter ' + json_inovel.chapter + ' | ' + json_inovel.title;
+ chapterContent = json_inovel.content.replace(/\n/g, '
');
+ if (json_inovel.notes) {
+ chapterContent +=
+ '
TL Notes:
' +
+ json_inovel.notes.replace(/\n/g, '
');
+ }
+ chapterText = `${chapterTitle}
${chapterContent}`;
+ break;
+ case 'ko-fi':
+ chapterText = loadedCheerio('script:contains("shadowDom.innerHTML")')
+ .html()
+ ?.match(/shadowDom\.innerHTML \+= '( loadedCheerio(tag).remove());
+ chapterTitle = loadedCheerio('.entry-title').first().text()!;
+ chapterContent = loadedCheerio('.entry-content p span span').html()!;
+ let chapterContentArray = chapterContent.split('\n').map(paragraph => {
+ return paragraph.replace(/ /g, '').trim() + '
';
+ });
+ chapterContent = chapterContentArray.join('').trim();
+ chapterText = `${chapterTitle}
${chapterContent}`;
+ break;
+ case 'raeitranslations':
+ const parts = url.split('/');
+ const link_raei = `${parts[0]}//api.${parts[2]}/api/chapters/?id=${parts[3]}&num=${parts[4]}`;
+ const json = await fetchApi(link_raei).then(r => r.json());
+ chapterTitle =
+ json.currentChapter.novTitle +
+ ' | ' +
+ 'Chapter ' +
+ json.currentChapter.num;
+ chapterContent =
+ json.currentChapter.head +
+ `
` +
+ json.currentChapter.body +
+ `
Translator's Note:
` +
+ json.currentChapter.note;
+ chapterContent = chapterContent.replace(/\n/g, '
');
+ chapterText = `${chapterTitle}
${chapterContent}`;
+ break;
+ case 'rainofsnow':
+ loadedCheerio('.responsivevoice-button').remove();
+ chapterText = loadedCheerio('div.content').html()!;
+ break;
+ case 'sacredtexttranslations':
+ bloatClasses = ['.entry-content blockquote', '.entry-content div'];
+ bloatClasses.map(tag => loadedCheerio(tag).remove());
+ chapterTitle = loadedCheerio('.entry-title').first().text()!;
+ chapterContent = loadedCheerio('.entry-content').html()!;
+ chapterText = `${chapterTitle}
${chapterContent}`;
+ break;
+ case 'scribblehub':
+ chapterTitle = loadedCheerio('.chapter-title').first().text()!;
+ chapterContent = loadedCheerio('div.chp_raw').html()!;
+ chapterText = `${chapterTitle}
${chapterContent}`;
+ break;
+ case 'stabbingwithasyringe':
+ /**
+ * Get the chapter link from the main page
+ */
+ const linkSyringe = loadedCheerio('.entry-content a').attr('href')!;
+ const resultSyringe = await fetchApi(linkSyringe);
+ const bodySyringe = await resultSyringe.text();
+ const loadedCheerioSyringe = parseHTML(bodySyringe);
+ bloatClasses = ['.wp-block-buttons', '#jp-post-flair'];
+ bloatClasses.map(tag => loadedCheerioSyringe(tag).remove());
+ chapterContent = loadedCheerioSyringe('.entry-content').html()!;
+ let titleElementSyringe =
+ loadedCheerioSyringe('.entry-content h3').first();
+ if (titleElementSyringe.length) {
+ chapterTitle = titleElementSyringe.first().text();
+ titleElementSyringe.remove();
+ chapterContent = loadedCheerioSyringe('.entry-content').html()!;
+ chapterText = `${chapterTitle}
${chapterContent}`;
+ } else {
+ chapterText = chapterContent;
+ }
+ break;
+ case 'tinytranslation':
+ bloatClasses = [
+ '.google_translate_element',
+ '.navigate',
+ '.post-views',
+ ];
+ bloatClasses.map(tag => loadedCheerio(tag).remove());
+ chapterTitle = loadedCheerio('.title-content').first().text()!;
+ loadedCheerio('.title-content').first().remove();
+ chapterContent = loadedCheerio('.content').html()!;
+ chapterText = `${chapterTitle}
${chapterContent}`;
+ break;
+ case 'tumblr':
+ chapterText = loadedCheerio('.post').html()!;
+ break;
+ case 'wattpad':
+ chapterTitle = loadedCheerio('.h2').first().text()!;
+ chapterContent = loadedCheerio('.part-content pre').html()!;
+ chapterText = `${chapterTitle}
${chapterContent}`;
+ break;
+ case 'webnovel':
+ chapterTitle = loadedCheerio('.cha-tit .pr .dib').first().text()!;
+ chapterContent = loadedCheerio('.cha-words').html()!;
+ if (!chapterContent) {
+ chapterContent = loadedCheerio('._content').html()!;
+ }
+ chapterText = `${chapterTitle}
${chapterContent}`;
+ break;
+ case 'wuxiaworld':
+ bloatClasses = ['.MuiLink-root'];
+ bloatClasses.map(tag => loadedCheerio(tag).remove());
+ chapterTitle = loadedCheerio('h4 span').first().text()!;
+ chapterContent = loadedCheerio('.chapter-content').html()!;
+ chapterText = `${chapterTitle}
${chapterContent}`;
+ break;
+ }
+ return chapterText;
+ }
+
async parseChapter(chapterPath: string): Promise {
+ let bloatClasses = [];
+ let chapterTitle = '';
+ let chapterContent = '';
let chapterText = '';
const result = await fetchApi(this.site + chapterPath);
const body = await result.text();
+ const url = result.url.toLowerCase();
+ const domain = url.split('/')[2].split('.');
// console.log(result.url);
// console.log('Redirected URL: ', result.chapterUrl);
const loadedCheerio = parseHTML(body);
-
- let isWuxiaWorld = result.url.toLowerCase().includes('wuxiaworld');
-
- let isBlogspot = result.url.toLowerCase().includes('blogspot');
-
- let isTumblr = result.url.toLowerCase().includes('tumblr');
-
- let isWattpad = result.url.toLowerCase().includes('wattpad');
-
- let isAnomalously = result.url.toLowerCase().includes('anotivereads');
-
- let kofi = result.url.toLowerCase().includes('ko-fi');
-
- let isBlossomTranslation = result.url
- .toLowerCase()
- .includes('blossomtranslation');
-
- let isLightNovelsTls = result.url
- .toLowerCase()
- .includes('lightnovelstranslations');
-
- let isiNovelTranslation = result.url
- .toLowerCase()
- .includes('inoveltranslation');
-
- let isTravisTranslation = result.url
- .toLowerCase()
- .includes('travistranslations');
-
- let isRaeiTranslation = result.url
- .toLowerCase()
- .includes('raeitranslations');
/**
- * Checks if its a wordpress site
+ * Generators are last
*/
+ let isBlogspotStr = loadedCheerio(
+ 'meta[name="google-adsense-platform-domain"]',
+ ).attr('content');
+
+ let isBlogspot = false;
+ if (isBlogspotStr) {
+ isBlogspot = isBlogspotStr.toLowerCase().includes('blogspot');
+ }
+
let isWordPressStr =
+ loadedCheerio('#dcl_comments-js-extra').html()! ||
loadedCheerio('meta[name="generator"]').attr('content') ||
loadedCheerio('footer').text();
let isWordPress = false;
-
if (isWordPressStr) {
isWordPress =
isWordPressStr.toLowerCase().includes('wordpress') ||
@@ -224,109 +366,125 @@ class NovelUpdates implements Plugin.PluginBase {
loadedCheerio('.powered-by').text().toLowerCase().includes('wordpress');
}
- let isRainOfSnow = result.url.toLowerCase().includes('rainofsnow');
-
- let isWebNovel = result.url.toLowerCase().includes('webnovel');
-
- let isHostedNovel = result.url.toLowerCase().includes('hostednovel');
-
- let isScribbleHub = result.url.toLowerCase().includes('scribblehub');
-
- if (isWuxiaWorld) {
- chapterText = loadedCheerio('#chapter-content').html()!;
- } else if (isRainOfSnow) {
- chapterText = loadedCheerio('div.content').html()!;
- } else if (isTumblr) {
- chapterText = loadedCheerio('.post').html()!;
+ const outliers = [
+ 'anotivereads',
+ 'asuratls',
+ 'novelworldtranslations',
+ 'sacredtexttranslations',
+ 'stabbingwithasyringe',
+ 'tinytranslation',
+ ];
+ if (domain.find(d => outliers.includes(d))) {
+ isWordPress = false;
+ isBlogspot = false;
+ }
+ /**
+ * Blogspot Novels:
+ * - AssedTL
+ * - AsuraTls (Outlier)
+ * - Novel World Translations (Outlier)
+ * - SacredText Translation (Outlier)
+ *
+ * WordPress Novels:
+ * - A Novel Reader Attempts Translating
+ * - Anomalously Creative (Outlier)
+ * - Arcane Translations
+ * - Blossom Translation
+ * - Dumah's Translations
+ * - ElloTL
+ * - Gem Novels Translations
+ * - Genesis Translations
+ * - Goblinslate
+ * - Hel Scans
+ * - Ippo Translations
+ * - JATranslations
+ * - Light Novels Translations
+ * - Neosekai Translations
+ * - Shanghai Fantasy
+ * - Soafp Translations
+ * - Stabbing with a Syringe (Outlier)
+ * - Stone Scape
+ * - Tiny Translation (Outlier)
+ * - Wonder Novels
+ * - Yong Library
+ * - Zetro Translation
+ */
+ if (!isWordPress && !isBlogspot) {
+ chapterText = await this.getChapterBody(loadedCheerio, domain, url);
} else if (isBlogspot) {
- loadedCheerio('.post-share-buttons').remove();
- chapterText =
- loadedCheerio('.entry-content').html() ||
- loadedCheerio('#post-body').html()!;
- } else if (isHostedNovel) {
- chapterText = loadedCheerio('.chapter').html()!;
- } else if (isScribbleHub) {
- chapterText = loadedCheerio('div.chp_raw').html()!;
- } else if (isWattpad) {
- chapterText = loadedCheerio('.container pre').html()!;
- } else if (isTravisTranslation) {
- chapterText = loadedCheerio('.reader-content').html()!;
- } else if (isLightNovelsTls) {
- chapterText = loadedCheerio('.text_story').html()!;
- } else if (isBlossomTranslation) {
- chapterText = loadedCheerio('.manga-child-content').html()!;
- } else if (isAnomalously) {
- chapterText = loadedCheerio('#comic').html()!;
- } else if (isRaeiTranslation) {
- const parts = result.url.split('/');
- const link = `${parts[0]}//api.${parts[2]}/api/chapters/?id=${parts[3]}&num=${parts[4]}`;
- const json = await fetchApi(link).then(r => r.json());
- chapterText =
- json.currentChapter.head +
- `
` +
- json.currentChapter.body +
- `
Translator's Note:
` +
- json.currentChapter.note;
- chapterText = chapterText.replace(/\n/g, '
');
- } else if (kofi) {
- chapterText = loadedCheerio('script:contains("shadowDom.innerHTML")')
- .html()
- ?.match(/shadowDom\.innerHTML \+= '( r.json());
- chapterText = json.content.replace(/\n/g, '
');
- if (json.notes) {
- chapterText +=
- '
TL Notes:
' + json.notes.replace(/\n/g, '
');
- }
- } else if (isWordPress) {
/**
- * Remove wordpress bloat tags
+ * Generators are last
*/
-
- const bloatClasses = [
- '.c-ads',
- '#madara-comments',
- '#comments',
- '.content-comments',
+ bloatClasses = ['.button-container', '.separator'];
+ bloatClasses.map(tag => loadedCheerio(tag).remove());
+ chapterTitle =
+ loadedCheerio('.entry-title').first().text() ||
+ loadedCheerio('.post-title').first().text()!;
+ chapterContent =
+ loadedCheerio('.entry-content').html() ||
+ loadedCheerio('.post-body').html()!;
+ chapterText = `${chapterTitle}
${chapterContent}`;
+ } else if (isWordPress) {
+ bloatClasses = [
+ '.ad',
+ '.author-avatar',
+ '.chapter-warning',
+ '.entry-content .has-text-align-center',
+ '.entry-meta',
+ '.ezoic-ad',
+ '.mb-center',
+ '.patreon-widget',
+ '.post-cats',
+ '.pre-bar',
'.sharedaddy',
- '.wp-dark-mode-switcher',
- '.wp-next-post-navi',
+ '.sidebar',
'.wp-block-buttons',
'.wp-block-columns',
- '.post-cats',
- '.sidebar',
- '.author-avatar',
- '.ezoic-ad',
+ '.wp-block-image',
+ '.wp-dark-mode-switcher',
+ '.wp-next-post-navi',
+ '#jp-post-flair',
+ '#textbox',
];
-
bloatClasses.map(tag => loadedCheerio(tag).remove());
-
- chapterText =
+ chapterTitle =
+ loadedCheerio('.entry-title').first().text() ||
+ loadedCheerio('.entry-title-main').first().text() ||
+ loadedCheerio('.chapter__title').first().text() ||
+ loadedCheerio('.sp-title').first().text() ||
+ loadedCheerio('.title-content').first().text() ||
+ loadedCheerio('.wp-block-post-title').first().text() ||
+ loadedCheerio('.title_story').first().text() ||
+ loadedCheerio('.active').first().text() ||
+ loadedCheerio('head title').first().text()!;
+ let chapterSubtitle = loadedCheerio('.cat-series').first().text() || '';
+ chapterContent =
+ loadedCheerio('.rdminimal').html() ||
loadedCheerio('.entry-content').html() ||
+ loadedCheerio('.chapter__content').html() ||
+ loadedCheerio('.prevent-select').html() ||
+ loadedCheerio('.text_story').html() ||
+ loadedCheerio('.contenta').html() ||
loadedCheerio('.single_post').html() ||
loadedCheerio('.post-entry').html() ||
loadedCheerio('.main-content').html() ||
- loadedCheerio('article.post').html() ||
loadedCheerio('.content').html() ||
- loadedCheerio('#content').html() ||
loadedCheerio('.page-body').html() ||
- loadedCheerio('.td-page-content').html()!;
- } else if (isWebNovel) {
- chapterText = loadedCheerio('.cha-words').html()!;
-
- if (!chapterText) {
- chapterText = loadedCheerio('._content').html()!;
+ loadedCheerio('.td-page-content').html() ||
+ loadedCheerio('#content').html() ||
+ loadedCheerio('article.post').html()!;
+ if (chapterSubtitle) {
+ chapterTitle = `${chapterTitle} | ${chapterSubtitle}`;
}
- } else {
+ chapterText = `${chapterTitle}
${chapterContent}`;
+ }
+
+ if (!chapterText) {
/**
* Remove unnecessary tags
*/
const tags = ['nav', 'header', 'footer', '.hidden'];
-
tags.map(tag => loadedCheerio(tag).remove());
-
chapterText = loadedCheerio('body').html()!;
}