Enhancement: Novel Updates (#915)

* refactored and updated Novel Updates' parseChapter()

* added `ippotranslations` and `shanghaifantasy` to Novel Updates

* small changes

* removed switch cases and reverted back to `let` and `if`/`else if` structure

* added lots of translation group support

* added more site support and tried cleaning chapter content where possible

* quick fix

* quick fix

* logic overhaul; implementation of switch cases (again)

* small changes to `bloatClasses` and `chapterContent`

* added Anomalously Creative (`anotivereads`) to outliers and added new `bloatClasses` to Wuxia World (`wuxiaworld`)

plugins/english/novelupdates.ts

@@ -6,7 +6,7 @@ import { Plugin } from '@typings/plugin';
 class NovelUpdates implements Plugin.PluginBase {
   id = 'novelupdates';
   name = 'Novel Updates';
-  version = '0.5.2';
+  version = '0.5.3';
   icon = 'src/en/novelupdates/icon.png';
   site = 'https://www.novelupdates.com/';
 
@@ -165,58 +165,200 @@ class NovelUpdates implements Plugin.PluginBase {
     return match && `${match[1]}//${match[3]}`;
   }
 
+  async getChapterBody(
+    loadedCheerio: CheerioAPI,
+    domain: string[],
+    url: string,
+  ) {
+    let bloatClasses = [];
+    let chapterTitle = '';
+    let chapterContent = '';
+    let chapterText = '';
+
+    const unwanted = ['blogspot', 'wordpress', 'www'];
+    const targetDomain = domain.find(d => !unwanted.includes(d));
+
+    switch (targetDomain) {
+      case 'anotivereads':
+        chapterTitle = loadedCheerio('#comic-nav-name').first().text()!;
+        chapterContent = loadedCheerio('#spliced-comic').html()!;
+        chapterText = `<h1>${chapterTitle}</h1><br>${chapterContent}`;
+        break;
+      case 'asuratls':
+        let titleElementAsura = loadedCheerio('.post-body div b').first();
+        chapterTitle = titleElementAsura.first().text();
+        titleElementAsura.remove();
+        chapterContent = loadedCheerio('.post-body').html()!;
+        chapterText = `<h1>${chapterTitle}</h1><br>${chapterContent}`;
+        break;
+      case 'hiraethtranslation':
+        chapterTitle = loadedCheerio('li.active').first().text()!;
+        chapterContent = loadedCheerio('.text-left').html()!;
+        chapterText = `<h1>${chapterTitle}</h1><br>${chapterContent}`;
+        break;
+      case 'hostednovel':
+        chapterTitle = loadedCheerio('#chapter-title').first().text()!;
+        chapterContent = loadedCheerio('#chapter-content').html()!;
+        chapterText = `<h1>${chapterTitle}</h1><br>${chapterContent}`;
+        break;
+      case 'inoveltranslation':
+        const link_inovel = 'https://api.' + url.slice(8);
+        const json_inovel = await fetchApi(link_inovel).then(r => r.json());
+        chapterTitle =
+          'Chapter ' + json_inovel.chapter + ' | ' + json_inovel.title;
+        chapterContent = json_inovel.content.replace(/\n/g, '<br>');
+        if (json_inovel.notes) {
+          chapterContent +=
+            '<br><hr><br>TL Notes:<br>' +
+            json_inovel.notes.replace(/\n/g, '<br>');
+        }
+        chapterText = `<h1>${chapterTitle}</h1><br>${chapterContent}`;
+        break;
+      case 'ko-fi':
+        chapterText = loadedCheerio('script:contains("shadowDom.innerHTML")')
+          .html()
+          ?.match(/shadowDom\.innerHTML \+= '(<div.*?)';/)![1]!;
+        break;
+      case 'novelworldtranslations':
+        bloatClasses = ['.separator', 'p[dir="ltr"]'];
+        bloatClasses.map(tag => loadedCheerio(tag).remove());
+        chapterTitle = loadedCheerio('.entry-title').first().text()!;
+        chapterContent = loadedCheerio('.entry-content p span span').html()!;
+        let chapterContentArray = chapterContent.split('\n').map(paragraph => {
+          return paragraph.replace(/&nbsp;/g, '').trim() + '<br>';
+        });
+        chapterContent = chapterContentArray.join('').trim();
+        chapterText = `<h1>${chapterTitle}</h1><br>${chapterContent}`;
+        break;
+      case 'raeitranslations':
+        const parts = url.split('/');
+        const link_raei = `${parts[0]}//api.${parts[2]}/api/chapters/?id=${parts[3]}&num=${parts[4]}`;
+        const json = await fetchApi(link_raei).then(r => r.json());
+        chapterTitle =
+          json.currentChapter.novTitle +
+          ' | ' +
+          'Chapter ' +
+          json.currentChapter.num;
+        chapterContent =
+          json.currentChapter.head +
+          `<br><hr><br>` +
+          json.currentChapter.body +
+          `<br><hr><br>Translator's Note:<br>` +
+          json.currentChapter.note;
+        chapterContent = chapterContent.replace(/\n/g, '<br>');
+        chapterText = `<h1>${chapterTitle}</h1><br>${chapterContent}`;
+        break;
+      case 'rainofsnow':
+        loadedCheerio('.responsivevoice-button').remove();
+        chapterText = loadedCheerio('div.content').html()!;
+        break;
+      case 'sacredtexttranslations':
+        bloatClasses = ['.entry-content blockquote', '.entry-content div'];
+        bloatClasses.map(tag => loadedCheerio(tag).remove());
+        chapterTitle = loadedCheerio('.entry-title').first().text()!;
+        chapterContent = loadedCheerio('.entry-content').html()!;
+        chapterText = `<h1>${chapterTitle}</h1><br>${chapterContent}`;
+        break;
+      case 'scribblehub':
+        chapterTitle = loadedCheerio('.chapter-title').first().text()!;
+        chapterContent = loadedCheerio('div.chp_raw').html()!;
+        chapterText = `<h1>${chapterTitle}</h1><br>${chapterContent}`;
+        break;
+      case 'stabbingwithasyringe':
+        /**
+         * Get the chapter link from the main page
+         */
+        const linkSyringe = loadedCheerio('.entry-content a').attr('href')!;
+        const resultSyringe = await fetchApi(linkSyringe);
+        const bodySyringe = await resultSyringe.text();
+        const loadedCheerioSyringe = parseHTML(bodySyringe);
+        bloatClasses = ['.wp-block-buttons', '#jp-post-flair'];
+        bloatClasses.map(tag => loadedCheerioSyringe(tag).remove());
+        chapterContent = loadedCheerioSyringe('.entry-content').html()!;
+        let titleElementSyringe =
+          loadedCheerioSyringe('.entry-content h3').first();
+        if (titleElementSyringe.length) {
+          chapterTitle = titleElementSyringe.first().text();
+          titleElementSyringe.remove();
+          chapterContent = loadedCheerioSyringe('.entry-content').html()!;
+          chapterText = `<h1>${chapterTitle}</h1><br>${chapterContent}`;
+        } else {
+          chapterText = chapterContent;
+        }
+        break;
+      case 'tinytranslation':
+        bloatClasses = [
+          '.google_translate_element',
+          '.navigate',
+          '.post-views',
+        ];
+        bloatClasses.map(tag => loadedCheerio(tag).remove());
+        chapterTitle = loadedCheerio('.title-content').first().text()!;
+        loadedCheerio('.title-content').first().remove();
+        chapterContent = loadedCheerio('.content').html()!;
+        chapterText = `<h1>${chapterTitle}</h1><br>${chapterContent}`;
+        break;
+      case 'tumblr':
+        chapterText = loadedCheerio('.post').html()!;
+        break;
+      case 'wattpad':
+        chapterTitle = loadedCheerio('.h2').first().text()!;
+        chapterContent = loadedCheerio('.part-content pre').html()!;
+        chapterText = `<h1>${chapterTitle}</h1><br>${chapterContent}`;
+        break;
+      case 'webnovel':
+        chapterTitle = loadedCheerio('.cha-tit .pr .dib').first().text()!;
+        chapterContent = loadedCheerio('.cha-words').html()!;
+        if (!chapterContent) {
+          chapterContent = loadedCheerio('._content').html()!;
+        }
+        chapterText = `<h1>${chapterTitle}</h1><br>${chapterContent}`;
+        break;
+      case 'wuxiaworld':
+        bloatClasses = ['.MuiLink-root'];
+        bloatClasses.map(tag => loadedCheerio(tag).remove());
+        chapterTitle = loadedCheerio('h4 span').first().text()!;
+        chapterContent = loadedCheerio('.chapter-content').html()!;
+        chapterText = `<h1>${chapterTitle}</h1><br>${chapterContent}`;
+        break;
+    }
+    return chapterText;
+  }
+
   async parseChapter(chapterPath: string): Promise<string> {
+    let bloatClasses = [];
+    let chapterTitle = '';
+    let chapterContent = '';
     let chapterText = '';
 
     const result = await fetchApi(this.site + chapterPath);
     const body = await result.text();
+    const url = result.url.toLowerCase();
+    const domain = url.split('/')[2].split('.');
 
     // console.log(result.url);
 
     // console.log('Redirected URL: ', result.chapterUrl);
 
     const loadedCheerio = parseHTML(body);
-
-    let isWuxiaWorld = result.url.toLowerCase().includes('wuxiaworld');
-
-    let isBlogspot = result.url.toLowerCase().includes('blogspot');
-
-    let isTumblr = result.url.toLowerCase().includes('tumblr');
-
-    let isWattpad = result.url.toLowerCase().includes('wattpad');
-
-    let isAnomalously = result.url.toLowerCase().includes('anotivereads');
-
-    let kofi = result.url.toLowerCase().includes('ko-fi');
-
-    let isBlossomTranslation = result.url
-      .toLowerCase()
-      .includes('blossomtranslation');
-
-    let isLightNovelsTls = result.url
-      .toLowerCase()
-      .includes('lightnovelstranslations');
-
-    let isiNovelTranslation = result.url
-      .toLowerCase()
-      .includes('inoveltranslation');
-
-    let isTravisTranslation = result.url
-      .toLowerCase()
-      .includes('travistranslations');
-
-    let isRaeiTranslation = result.url
-      .toLowerCase()
-      .includes('raeitranslations');
     /**
-     * Checks if its a wordpress site
+     * Generators are last
      */
+    let isBlogspotStr = loadedCheerio(
+      'meta[name="google-adsense-platform-domain"]',
+    ).attr('content');
+
+    let isBlogspot = false;
+    if (isBlogspotStr) {
+      isBlogspot = isBlogspotStr.toLowerCase().includes('blogspot');
+    }
+
     let isWordPressStr =
+      loadedCheerio('#dcl_comments-js-extra').html()! ||
       loadedCheerio('meta[name="generator"]').attr('content') ||
       loadedCheerio('footer').text();
 
     let isWordPress = false;
-
     if (isWordPressStr) {
       isWordPress =
         isWordPressStr.toLowerCase().includes('wordpress') ||
@@ -224,109 +366,125 @@ class NovelUpdates implements Plugin.PluginBase {
         loadedCheerio('.powered-by').text().toLowerCase().includes('wordpress');
     }
 
-    let isRainOfSnow = result.url.toLowerCase().includes('rainofsnow');
+    const outliers = [
-
+      'anotivereads',
-    let isWebNovel = result.url.toLowerCase().includes('webnovel');
+      'asuratls',
-
+      'novelworldtranslations',
-    let isHostedNovel = result.url.toLowerCase().includes('hostednovel');
+      'sacredtexttranslations',
-
+      'stabbingwithasyringe',
-    let isScribbleHub = result.url.toLowerCase().includes('scribblehub');
+      'tinytranslation',
-
+    ];
-    if (isWuxiaWorld) {
+    if (domain.find(d => outliers.includes(d))) {
-      chapterText = loadedCheerio('#chapter-content').html()!;
+      isWordPress = false;
-    } else if (isRainOfSnow) {
+      isBlogspot = false;
-      chapterText = loadedCheerio('div.content').html()!;
+    }
-    } else if (isTumblr) {
+    /**
-      chapterText = loadedCheerio('.post').html()!;
+     * Blogspot Novels:
+     * - AssedTL
+     * - AsuraTls (Outlier)
+     * - Novel World Translations (Outlier)
+     * - SacredText Translation (Outlier)
+     *
+     * WordPress Novels:
+     * - A Novel Reader Attempts Translating
+     * - Anomalously Creative (Outlier)
+     * - Arcane Translations
+     * - Blossom Translation
+     * - Dumah's Translations
+     * - ElloTL
+     * - Gem Novels Translations
+     * - Genesis Translations
+     * - Goblinslate
+     * - Hel Scans
+     * - Ippo Translations
+     * - JATranslations
+     * - Light Novels Translations
+     * - Neosekai Translations
+     * - Shanghai Fantasy
+     * - Soafp Translations
+     * - Stabbing with a Syringe (Outlier)
+     * - Stone Scape
+     * - Tiny Translation (Outlier)
+     * - Wonder Novels
+     * - Yong Library
+     * - Zetro Translation
+     */
+    if (!isWordPress && !isBlogspot) {
+      chapterText = await this.getChapterBody(loadedCheerio, domain, url);
     } else if (isBlogspot) {
-      loadedCheerio('.post-share-buttons').remove();
-      chapterText =
-        loadedCheerio('.entry-content').html() ||
-        loadedCheerio('#post-body').html()!;
-    } else if (isHostedNovel) {
-      chapterText = loadedCheerio('.chapter').html()!;
-    } else if (isScribbleHub) {
-      chapterText = loadedCheerio('div.chp_raw').html()!;
-    } else if (isWattpad) {
-      chapterText = loadedCheerio('.container  pre').html()!;
-    } else if (isTravisTranslation) {
-      chapterText = loadedCheerio('.reader-content').html()!;
-    } else if (isLightNovelsTls) {
-      chapterText = loadedCheerio('.text_story').html()!;
-    } else if (isBlossomTranslation) {
-      chapterText = loadedCheerio('.manga-child-content').html()!;
-    } else if (isAnomalously) {
-      chapterText = loadedCheerio('#comic').html()!;
-    } else if (isRaeiTranslation) {
-      const parts = result.url.split('/');
-      const link = `${parts[0]}//api.${parts[2]}/api/chapters/?id=${parts[3]}&num=${parts[4]}`;
-      const json = await fetchApi(link).then(r => r.json());
-      chapterText =
-        json.currentChapter.head +
-        `<br><hr><br>` +
-        json.currentChapter.body +
-        `<br><hr><br>Translator's Note:<br>` +
-        json.currentChapter.note;
-      chapterText = chapterText.replace(/\n/g, '<br>');
-    } else if (kofi) {
-      chapterText = loadedCheerio('script:contains("shadowDom.innerHTML")')
-        .html()
-        ?.match(/shadowDom\.innerHTML \+= '(<div.*?)';/)![1]!;
-    } else if (isiNovelTranslation) {
-      const link = 'https://api.' + result.url.slice(8);
-      const json = await fetchApi(link).then(r => r.json());
-      chapterText = json.content.replace(/\n/g, '<br>');
-      if (json.notes) {
-        chapterText +=
-          '<br><hr><br>TL Notes:<br>' + json.notes.replace(/\n/g, '<br>');
-      }
-    } else if (isWordPress) {
       /**
-       * Remove wordpress bloat tags
+       * Generators are last
        */
-
+      bloatClasses = ['.button-container', '.separator'];
-      const bloatClasses = [
+      bloatClasses.map(tag => loadedCheerio(tag).remove());
-        '.c-ads',
+      chapterTitle =
-        '#madara-comments',
+        loadedCheerio('.entry-title').first().text() ||
-        '#comments',
+        loadedCheerio('.post-title').first().text()!;
-        '.content-comments',
+      chapterContent =
+        loadedCheerio('.entry-content').html() ||
+        loadedCheerio('.post-body').html()!;
+      chapterText = `<h1>${chapterTitle}</h1><br>${chapterContent}`;
+    } else if (isWordPress) {
+      bloatClasses = [
+        '.ad',
+        '.author-avatar',
+        '.chapter-warning',
+        '.entry-content .has-text-align-center',
+        '.entry-meta',
+        '.ezoic-ad',
+        '.mb-center',
+        '.patreon-widget',
+        '.post-cats',
+        '.pre-bar',
         '.sharedaddy',
-        '.wp-dark-mode-switcher',
+        '.sidebar',
-        '.wp-next-post-navi',
         '.wp-block-buttons',
         '.wp-block-columns',
-        '.post-cats',
+        '.wp-block-image',
-        '.sidebar',
+        '.wp-dark-mode-switcher',
-        '.author-avatar',
+        '.wp-next-post-navi',
-        '.ezoic-ad',
+        '#jp-post-flair',
+        '#textbox',
       ];
-
       bloatClasses.map(tag => loadedCheerio(tag).remove());
-
+      chapterTitle =
-      chapterText =
+        loadedCheerio('.entry-title').first().text() ||
+        loadedCheerio('.entry-title-main').first().text() ||
+        loadedCheerio('.chapter__title').first().text() ||
+        loadedCheerio('.sp-title').first().text() ||
+        loadedCheerio('.title-content').first().text() ||
+        loadedCheerio('.wp-block-post-title').first().text() ||
+        loadedCheerio('.title_story').first().text() ||
+        loadedCheerio('.active').first().text() ||
+        loadedCheerio('head title').first().text()!;
+      let chapterSubtitle = loadedCheerio('.cat-series').first().text() || '';
+      chapterContent =
+        loadedCheerio('.rdminimal').html() ||
         loadedCheerio('.entry-content').html() ||
+        loadedCheerio('.chapter__content').html() ||
+        loadedCheerio('.prevent-select').html() ||
+        loadedCheerio('.text_story').html() ||
+        loadedCheerio('.contenta').html() ||
         loadedCheerio('.single_post').html() ||
         loadedCheerio('.post-entry').html() ||
         loadedCheerio('.main-content').html() ||
-        loadedCheerio('article.post').html() ||
         loadedCheerio('.content').html() ||
-        loadedCheerio('#content').html() ||
         loadedCheerio('.page-body').html() ||
-        loadedCheerio('.td-page-content').html()!;
+        loadedCheerio('.td-page-content').html() ||
-    } else if (isWebNovel) {
+        loadedCheerio('#content').html() ||
-      chapterText = loadedCheerio('.cha-words').html()!;
+        loadedCheerio('article.post').html()!;
-
+      if (chapterSubtitle) {
-      if (!chapterText) {
+        chapterTitle = `${chapterTitle} | ${chapterSubtitle}`;
-        chapterText = loadedCheerio('._content').html()!;
       }
-    } else {
+      chapterText = `<h1>${chapterTitle}</h1><br>${chapterContent}`;
+    }
+
+    if (!chapterText) {
       /**
        * Remove unnecessary tags
        */
       const tags = ['nav', 'header', 'footer', '.hidden'];
-
       tags.map(tag => loadedCheerio(tag).remove());
-
       chapterText = loadedCheerio('body').html()!;
     }