Enhancement: Novel Updates (#915)
* refactored and updated Novel Updates' parseChapter() * added `ippotranslations` and `shanghaifantasy` to Novel Updates * small changes * removed switch cases and reverted back to `let` and `if`/`else if` structure * added lots of translation group support * added more site support and tried cleaning chapter content where possible * quick fix * quick fix * logic overhaul; implementation of switch cases (again) * small changes to `bloatClasses` and `chapterContent` * added Anomalously Creative (`anotivereads`) to outliers and added new `bloatClasses` to Wuxia World (`wuxiaworld`)
This commit is contained in:
Lukas Richter
+275
-117
@@ -6,7 +6,7 @@ import { Plugin } from '@typings/plugin';
|
||||
class NovelUpdates implements Plugin.PluginBase {
|
||||
id = 'novelupdates';
|
||||
name = 'Novel Updates';
|
||||
version = '0.5.2';
|
||||
version = '0.5.3';
|
||||
icon = 'src/en/novelupdates/icon.png';
|
||||
site = 'https://www.novelupdates.com/';
|
||||
|
||||
@@ -165,58 +165,200 @@ class NovelUpdates implements Plugin.PluginBase {
|
||||
return match && `${match[1]}//${match[3]}`;
|
||||
}
|
||||
|
||||
async getChapterBody(
|
||||
loadedCheerio: CheerioAPI,
|
||||
domain: string[],
|
||||
url: string,
|
||||
) {
|
||||
let bloatClasses = [];
|
||||
let chapterTitle = '';
|
||||
let chapterContent = '';
|
||||
let chapterText = '';
|
||||
|
||||
const unwanted = ['blogspot', 'wordpress', 'www'];
|
||||
const targetDomain = domain.find(d => !unwanted.includes(d));
|
||||
|
||||
switch (targetDomain) {
|
||||
case 'anotivereads':
|
||||
chapterTitle = loadedCheerio('#comic-nav-name').first().text()!;
|
||||
chapterContent = loadedCheerio('#spliced-comic').html()!;
|
||||
chapterText = `<h1>${chapterTitle}</h1><br>${chapterContent}`;
|
||||
break;
|
||||
case 'asuratls':
|
||||
let titleElementAsura = loadedCheerio('.post-body div b').first();
|
||||
chapterTitle = titleElementAsura.first().text();
|
||||
titleElementAsura.remove();
|
||||
chapterContent = loadedCheerio('.post-body').html()!;
|
||||
chapterText = `<h1>${chapterTitle}</h1><br>${chapterContent}`;
|
||||
break;
|
||||
case 'hiraethtranslation':
|
||||
chapterTitle = loadedCheerio('li.active').first().text()!;
|
||||
chapterContent = loadedCheerio('.text-left').html()!;
|
||||
chapterText = `<h1>${chapterTitle}</h1><br>${chapterContent}`;
|
||||
break;
|
||||
case 'hostednovel':
|
||||
chapterTitle = loadedCheerio('#chapter-title').first().text()!;
|
||||
chapterContent = loadedCheerio('#chapter-content').html()!;
|
||||
chapterText = `<h1>${chapterTitle}</h1><br>${chapterContent}`;
|
||||
break;
|
||||
case 'inoveltranslation':
|
||||
const link_inovel = 'https://api.' + url.slice(8);
|
||||
const json_inovel = await fetchApi(link_inovel).then(r => r.json());
|
||||
chapterTitle =
|
||||
'Chapter ' + json_inovel.chapter + ' | ' + json_inovel.title;
|
||||
chapterContent = json_inovel.content.replace(/\n/g, '<br>');
|
||||
if (json_inovel.notes) {
|
||||
chapterContent +=
|
||||
'<br><hr><br>TL Notes:<br>' +
|
||||
json_inovel.notes.replace(/\n/g, '<br>');
|
||||
}
|
||||
chapterText = `<h1>${chapterTitle}</h1><br>${chapterContent}`;
|
||||
break;
|
||||
case 'ko-fi':
|
||||
chapterText = loadedCheerio('script:contains("shadowDom.innerHTML")')
|
||||
.html()
|
||||
?.match(/shadowDom\.innerHTML \+= '(<div.*?)';/)![1]!;
|
||||
break;
|
||||
case 'novelworldtranslations':
|
||||
bloatClasses = ['.separator', 'p[dir="ltr"]'];
|
||||
bloatClasses.map(tag => loadedCheerio(tag).remove());
|
||||
chapterTitle = loadedCheerio('.entry-title').first().text()!;
|
||||
chapterContent = loadedCheerio('.entry-content p span span').html()!;
|
||||
let chapterContentArray = chapterContent.split('\n').map(paragraph => {
|
||||
return paragraph.replace(/ /g, '').trim() + '<br>';
|
||||
});
|
||||
chapterContent = chapterContentArray.join('').trim();
|
||||
chapterText = `<h1>${chapterTitle}</h1><br>${chapterContent}`;
|
||||
break;
|
||||
case 'raeitranslations':
|
||||
const parts = url.split('/');
|
||||
const link_raei = `${parts[0]}//api.${parts[2]}/api/chapters/?id=${parts[3]}&num=${parts[4]}`;
|
||||
const json = await fetchApi(link_raei).then(r => r.json());
|
||||
chapterTitle =
|
||||
json.currentChapter.novTitle +
|
||||
' | ' +
|
||||
'Chapter ' +
|
||||
json.currentChapter.num;
|
||||
chapterContent =
|
||||
json.currentChapter.head +
|
||||
`<br><hr><br>` +
|
||||
json.currentChapter.body +
|
||||
`<br><hr><br>Translator's Note:<br>` +
|
||||
json.currentChapter.note;
|
||||
chapterContent = chapterContent.replace(/\n/g, '<br>');
|
||||
chapterText = `<h1>${chapterTitle}</h1><br>${chapterContent}`;
|
||||
break;
|
||||
case 'rainofsnow':
|
||||
loadedCheerio('.responsivevoice-button').remove();
|
||||
chapterText = loadedCheerio('div.content').html()!;
|
||||
break;
|
||||
case 'sacredtexttranslations':
|
||||
bloatClasses = ['.entry-content blockquote', '.entry-content div'];
|
||||
bloatClasses.map(tag => loadedCheerio(tag).remove());
|
||||
chapterTitle = loadedCheerio('.entry-title').first().text()!;
|
||||
chapterContent = loadedCheerio('.entry-content').html()!;
|
||||
chapterText = `<h1>${chapterTitle}</h1><br>${chapterContent}`;
|
||||
break;
|
||||
case 'scribblehub':
|
||||
chapterTitle = loadedCheerio('.chapter-title').first().text()!;
|
||||
chapterContent = loadedCheerio('div.chp_raw').html()!;
|
||||
chapterText = `<h1>${chapterTitle}</h1><br>${chapterContent}`;
|
||||
break;
|
||||
case 'stabbingwithasyringe':
|
||||
/**
|
||||
* Get the chapter link from the main page
|
||||
*/
|
||||
const linkSyringe = loadedCheerio('.entry-content a').attr('href')!;
|
||||
const resultSyringe = await fetchApi(linkSyringe);
|
||||
const bodySyringe = await resultSyringe.text();
|
||||
const loadedCheerioSyringe = parseHTML(bodySyringe);
|
||||
bloatClasses = ['.wp-block-buttons', '#jp-post-flair'];
|
||||
bloatClasses.map(tag => loadedCheerioSyringe(tag).remove());
|
||||
chapterContent = loadedCheerioSyringe('.entry-content').html()!;
|
||||
let titleElementSyringe =
|
||||
loadedCheerioSyringe('.entry-content h3').first();
|
||||
if (titleElementSyringe.length) {
|
||||
chapterTitle = titleElementSyringe.first().text();
|
||||
titleElementSyringe.remove();
|
||||
chapterContent = loadedCheerioSyringe('.entry-content').html()!;
|
||||
chapterText = `<h1>${chapterTitle}</h1><br>${chapterContent}`;
|
||||
} else {
|
||||
chapterText = chapterContent;
|
||||
}
|
||||
break;
|
||||
case 'tinytranslation':
|
||||
bloatClasses = [
|
||||
'.google_translate_element',
|
||||
'.navigate',
|
||||
'.post-views',
|
||||
];
|
||||
bloatClasses.map(tag => loadedCheerio(tag).remove());
|
||||
chapterTitle = loadedCheerio('.title-content').first().text()!;
|
||||
loadedCheerio('.title-content').first().remove();
|
||||
chapterContent = loadedCheerio('.content').html()!;
|
||||
chapterText = `<h1>${chapterTitle}</h1><br>${chapterContent}`;
|
||||
break;
|
||||
case 'tumblr':
|
||||
chapterText = loadedCheerio('.post').html()!;
|
||||
break;
|
||||
case 'wattpad':
|
||||
chapterTitle = loadedCheerio('.h2').first().text()!;
|
||||
chapterContent = loadedCheerio('.part-content pre').html()!;
|
||||
chapterText = `<h1>${chapterTitle}</h1><br>${chapterContent}`;
|
||||
break;
|
||||
case 'webnovel':
|
||||
chapterTitle = loadedCheerio('.cha-tit .pr .dib').first().text()!;
|
||||
chapterContent = loadedCheerio('.cha-words').html()!;
|
||||
if (!chapterContent) {
|
||||
chapterContent = loadedCheerio('._content').html()!;
|
||||
}
|
||||
chapterText = `<h1>${chapterTitle}</h1><br>${chapterContent}`;
|
||||
break;
|
||||
case 'wuxiaworld':
|
||||
bloatClasses = ['.MuiLink-root'];
|
||||
bloatClasses.map(tag => loadedCheerio(tag).remove());
|
||||
chapterTitle = loadedCheerio('h4 span').first().text()!;
|
||||
chapterContent = loadedCheerio('.chapter-content').html()!;
|
||||
chapterText = `<h1>${chapterTitle}</h1><br>${chapterContent}`;
|
||||
break;
|
||||
}
|
||||
return chapterText;
|
||||
}
|
||||
|
||||
async parseChapter(chapterPath: string): Promise<string> {
|
||||
let bloatClasses = [];
|
||||
let chapterTitle = '';
|
||||
let chapterContent = '';
|
||||
let chapterText = '';
|
||||
|
||||
const result = await fetchApi(this.site + chapterPath);
|
||||
const body = await result.text();
|
||||
const url = result.url.toLowerCase();
|
||||
const domain = url.split('/')[2].split('.');
|
||||
|
||||
// console.log(result.url);
|
||||
|
||||
// console.log('Redirected URL: ', result.chapterUrl);
|
||||
|
||||
const loadedCheerio = parseHTML(body);
|
||||
|
||||
let isWuxiaWorld = result.url.toLowerCase().includes('wuxiaworld');
|
||||
|
||||
let isBlogspot = result.url.toLowerCase().includes('blogspot');
|
||||
|
||||
let isTumblr = result.url.toLowerCase().includes('tumblr');
|
||||
|
||||
let isWattpad = result.url.toLowerCase().includes('wattpad');
|
||||
|
||||
let isAnomalously = result.url.toLowerCase().includes('anotivereads');
|
||||
|
||||
let kofi = result.url.toLowerCase().includes('ko-fi');
|
||||
|
||||
let isBlossomTranslation = result.url
|
||||
.toLowerCase()
|
||||
.includes('blossomtranslation');
|
||||
|
||||
let isLightNovelsTls = result.url
|
||||
.toLowerCase()
|
||||
.includes('lightnovelstranslations');
|
||||
|
||||
let isiNovelTranslation = result.url
|
||||
.toLowerCase()
|
||||
.includes('inoveltranslation');
|
||||
|
||||
let isTravisTranslation = result.url
|
||||
.toLowerCase()
|
||||
.includes('travistranslations');
|
||||
|
||||
let isRaeiTranslation = result.url
|
||||
.toLowerCase()
|
||||
.includes('raeitranslations');
|
||||
/**
|
||||
* Checks if its a wordpress site
|
||||
* Generators are last
|
||||
*/
|
||||
let isBlogspotStr = loadedCheerio(
|
||||
'meta[name="google-adsense-platform-domain"]',
|
||||
).attr('content');
|
||||
|
||||
let isBlogspot = false;
|
||||
if (isBlogspotStr) {
|
||||
isBlogspot = isBlogspotStr.toLowerCase().includes('blogspot');
|
||||
}
|
||||
|
||||
let isWordPressStr =
|
||||
loadedCheerio('#dcl_comments-js-extra').html()! ||
|
||||
loadedCheerio('meta[name="generator"]').attr('content') ||
|
||||
loadedCheerio('footer').text();
|
||||
|
||||
let isWordPress = false;
|
||||
|
||||
if (isWordPressStr) {
|
||||
isWordPress =
|
||||
isWordPressStr.toLowerCase().includes('wordpress') ||
|
||||
@@ -224,109 +366,125 @@ class NovelUpdates implements Plugin.PluginBase {
|
||||
loadedCheerio('.powered-by').text().toLowerCase().includes('wordpress');
|
||||
}
|
||||
|
||||
let isRainOfSnow = result.url.toLowerCase().includes('rainofsnow');
|
||||
|
||||
let isWebNovel = result.url.toLowerCase().includes('webnovel');
|
||||
|
||||
let isHostedNovel = result.url.toLowerCase().includes('hostednovel');
|
||||
|
||||
let isScribbleHub = result.url.toLowerCase().includes('scribblehub');
|
||||
|
||||
if (isWuxiaWorld) {
|
||||
chapterText = loadedCheerio('#chapter-content').html()!;
|
||||
} else if (isRainOfSnow) {
|
||||
chapterText = loadedCheerio('div.content').html()!;
|
||||
} else if (isTumblr) {
|
||||
chapterText = loadedCheerio('.post').html()!;
|
||||
} else if (isBlogspot) {
|
||||
loadedCheerio('.post-share-buttons').remove();
|
||||
chapterText =
|
||||
loadedCheerio('.entry-content').html() ||
|
||||
loadedCheerio('#post-body').html()!;
|
||||
} else if (isHostedNovel) {
|
||||
chapterText = loadedCheerio('.chapter').html()!;
|
||||
} else if (isScribbleHub) {
|
||||
chapterText = loadedCheerio('div.chp_raw').html()!;
|
||||
} else if (isWattpad) {
|
||||
chapterText = loadedCheerio('.container pre').html()!;
|
||||
} else if (isTravisTranslation) {
|
||||
chapterText = loadedCheerio('.reader-content').html()!;
|
||||
} else if (isLightNovelsTls) {
|
||||
chapterText = loadedCheerio('.text_story').html()!;
|
||||
} else if (isBlossomTranslation) {
|
||||
chapterText = loadedCheerio('.manga-child-content').html()!;
|
||||
} else if (isAnomalously) {
|
||||
chapterText = loadedCheerio('#comic').html()!;
|
||||
} else if (isRaeiTranslation) {
|
||||
const parts = result.url.split('/');
|
||||
const link = `${parts[0]}//api.${parts[2]}/api/chapters/?id=${parts[3]}&num=${parts[4]}`;
|
||||
const json = await fetchApi(link).then(r => r.json());
|
||||
chapterText =
|
||||
json.currentChapter.head +
|
||||
`<br><hr><br>` +
|
||||
json.currentChapter.body +
|
||||
`<br><hr><br>Translator's Note:<br>` +
|
||||
json.currentChapter.note;
|
||||
chapterText = chapterText.replace(/\n/g, '<br>');
|
||||
} else if (kofi) {
|
||||
chapterText = loadedCheerio('script:contains("shadowDom.innerHTML")')
|
||||
.html()
|
||||
?.match(/shadowDom\.innerHTML \+= '(<div.*?)';/)![1]!;
|
||||
} else if (isiNovelTranslation) {
|
||||
const link = 'https://api.' + result.url.slice(8);
|
||||
const json = await fetchApi(link).then(r => r.json());
|
||||
chapterText = json.content.replace(/\n/g, '<br>');
|
||||
if (json.notes) {
|
||||
chapterText +=
|
||||
'<br><hr><br>TL Notes:<br>' + json.notes.replace(/\n/g, '<br>');
|
||||
const outliers = [
|
||||
'anotivereads',
|
||||
'asuratls',
|
||||
'novelworldtranslations',
|
||||
'sacredtexttranslations',
|
||||
'stabbingwithasyringe',
|
||||
'tinytranslation',
|
||||
];
|
||||
if (domain.find(d => outliers.includes(d))) {
|
||||
isWordPress = false;
|
||||
isBlogspot = false;
|
||||
}
|
||||
} else if (isWordPress) {
|
||||
/**
|
||||
* Remove wordpress bloat tags
|
||||
* Blogspot Novels:
|
||||
* - AssedTL
|
||||
* - AsuraTls (Outlier)
|
||||
* - Novel World Translations (Outlier)
|
||||
* - SacredText Translation (Outlier)
|
||||
*
|
||||
* WordPress Novels:
|
||||
* - A Novel Reader Attempts Translating
|
||||
* - Anomalously Creative (Outlier)
|
||||
* - Arcane Translations
|
||||
* - Blossom Translation
|
||||
* - Dumah's Translations
|
||||
* - ElloTL
|
||||
* - Gem Novels Translations
|
||||
* - Genesis Translations
|
||||
* - Goblinslate
|
||||
* - Hel Scans
|
||||
* - Ippo Translations
|
||||
* - JATranslations
|
||||
* - Light Novels Translations
|
||||
* - Neosekai Translations
|
||||
* - Shanghai Fantasy
|
||||
* - Soafp Translations
|
||||
* - Stabbing with a Syringe (Outlier)
|
||||
* - Stone Scape
|
||||
* - Tiny Translation (Outlier)
|
||||
* - Wonder Novels
|
||||
* - Yong Library
|
||||
* - Zetro Translation
|
||||
*/
|
||||
|
||||
const bloatClasses = [
|
||||
'.c-ads',
|
||||
'#madara-comments',
|
||||
'#comments',
|
||||
'.content-comments',
|
||||
if (!isWordPress && !isBlogspot) {
|
||||
chapterText = await this.getChapterBody(loadedCheerio, domain, url);
|
||||
} else if (isBlogspot) {
|
||||
/**
|
||||
* Generators are last
|
||||
*/
|
||||
bloatClasses = ['.button-container', '.separator'];
|
||||
bloatClasses.map(tag => loadedCheerio(tag).remove());
|
||||
chapterTitle =
|
||||
loadedCheerio('.entry-title').first().text() ||
|
||||
loadedCheerio('.post-title').first().text()!;
|
||||
chapterContent =
|
||||
loadedCheerio('.entry-content').html() ||
|
||||
loadedCheerio('.post-body').html()!;
|
||||
chapterText = `<h1>${chapterTitle}</h1><br>${chapterContent}`;
|
||||
} else if (isWordPress) {
|
||||
bloatClasses = [
|
||||
'.ad',
|
||||
'.author-avatar',
|
||||
'.chapter-warning',
|
||||
'.entry-content .has-text-align-center',
|
||||
'.entry-meta',
|
||||
'.ezoic-ad',
|
||||
'.mb-center',
|
||||
'.patreon-widget',
|
||||
'.post-cats',
|
||||
'.pre-bar',
|
||||
'.sharedaddy',
|
||||
'.wp-dark-mode-switcher',
|
||||
'.wp-next-post-navi',
|
||||
'.sidebar',
|
||||
'.wp-block-buttons',
|
||||
'.wp-block-columns',
|
||||
'.post-cats',
|
||||
'.sidebar',
|
||||
'.author-avatar',
|
||||
'.ezoic-ad',
|
||||
'.wp-block-image',
|
||||
'.wp-dark-mode-switcher',
|
||||
'.wp-next-post-navi',
|
||||
'#jp-post-flair',
|
||||
'#textbox',
|
||||
];
|
||||
|
||||
bloatClasses.map(tag => loadedCheerio(tag).remove());
|
||||
|
||||
chapterText =
|
||||
chapterTitle =
|
||||
loadedCheerio('.entry-title').first().text() ||
|
||||
loadedCheerio('.entry-title-main').first().text() ||
|
||||
loadedCheerio('.chapter__title').first().text() ||
|
||||
loadedCheerio('.sp-title').first().text() ||
|
||||
loadedCheerio('.title-content').first().text() ||
|
||||
loadedCheerio('.wp-block-post-title').first().text() ||
|
||||
loadedCheerio('.title_story').first().text() ||
|
||||
loadedCheerio('.active').first().text() ||
|
||||
loadedCheerio('head title').first().text()!;
|
||||
let chapterSubtitle = loadedCheerio('.cat-series').first().text() || '';
|
||||
chapterContent =
|
||||
loadedCheerio('.rdminimal').html() ||
|
||||
loadedCheerio('.entry-content').html() ||
|
||||
loadedCheerio('.chapter__content').html() ||
|
||||
loadedCheerio('.prevent-select').html() ||
|
||||
loadedCheerio('.text_story').html() ||
|
||||
loadedCheerio('.contenta').html() ||
|
||||
loadedCheerio('.single_post').html() ||
|
||||
loadedCheerio('.post-entry').html() ||
|
||||
loadedCheerio('.main-content').html() ||
|
||||
loadedCheerio('article.post').html() ||
|
||||
loadedCheerio('.content').html() ||
|
||||
loadedCheerio('#content').html() ||
|
||||
loadedCheerio('.page-body').html() ||
|
||||
loadedCheerio('.td-page-content').html()!;
|
||||
} else if (isWebNovel) {
|
||||
chapterText = loadedCheerio('.cha-words').html()!;
|
||||
loadedCheerio('.td-page-content').html() ||
|
||||
loadedCheerio('#content').html() ||
|
||||
loadedCheerio('article.post').html()!;
|
||||
if (chapterSubtitle) {
|
||||
chapterTitle = `${chapterTitle} | ${chapterSubtitle}`;
|
||||
}
|
||||
chapterText = `<h1>${chapterTitle}</h1><br>${chapterContent}`;
|
||||
}
|
||||
|
||||
if (!chapterText) {
|
||||
chapterText = loadedCheerio('._content').html()!;
|
||||
}
|
||||
} else {
|
||||
/**
|
||||
* Remove unnecessary tags
|
||||
*/
|
||||
const tags = ['nav', 'header', 'footer', '.hidden'];
|
||||
|
||||
tags.map(tag => loadedCheerio(tag).remove());
|
||||
|
||||
chapterText = loadedCheerio('body').html()!;
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user