Fix/Upgrade xiaowaz (#934)

* xiaowaz:
 - get all novels from all categories
 - cover on search (slow TT)
 - better image finding
 - fix summary

* xiaowaz:
 - more autor regex patterns
 - restored 'Rebirth of The Thief Who Roamed The World' weird fields
 - return false; instead of return;  ¯\_(ツ)_/¯

* Xiaowaz:
 - Static novels list, for less fetch
 - Popular novels split based on articles count instead of page max number
 - async images fetching
 - added status to cancelled novels
 - added original tag

plugins/french/xiaowaz.ts

@@ -1,4 +1,4 @@
-import { CheerioAPI, load } from 'cheerio';
+import { Cheerio, CheerioAPI, load, Element } from 'cheerio';
 import { fetchApi, fetchFile } from '@libs/fetch';
 import { Plugin } from '@typings/plugin';
 import { Filters, FilterTypes } from '@libs/filterInputs';
@@ -10,52 +10,89 @@ class XiaowazPlugin implements Plugin.PluginBase {
   name = 'Xiaowaz';
   icon = 'src/fr/xiaowaz/icon.png';
   site = 'https://xiaowaz.fr';
-  version = '1.0.0';
+  version = '1.0.1';
   filters: Filters | undefined = undefined;
+  static novels: Plugin.NovelItem[] | undefined;
 
   async getCheerio(url: string): Promise<CheerioAPI> {
-    const r = await fetchApi(url, {
+    let retries = 5; // when fetching for images the sites sometimes terminates the connection
-      headers: { 'Accept-Encoding': 'deflate' },
+    let returnError: any;
-    });
+    while (retries > 0) {
-    const body = await r.text();
+      try {
-    const $ = load(body);
+        const r = await fetchApi(url);
-    return $;
+        const body = await r.text();
+        const $ = load(body);
+        return $;
+      } catch (error) {
+        console.error(error);
+        returnError = error;
+        retries--;
+        // wait 1 second before retrying
+        await new Promise(resolve => setTimeout(resolve, 1000));
+      }
+    }
+    throw new Error(returnError ? returnError : 'Error fetching the page');
   }
 
-  async popularNovels(
+  async getAllNovels(): Promise<Plugin.NovelItem[]> {
-    pageNo: number,
+    let novels: Plugin.NovelItem[] = [];
-    {
-      showLatestNovels,
-      filters,
-    }: Plugin.PopularNovelsOptions<typeof this.filters>,
-  ): Promise<Plugin.NovelItem[]> {
-    if (pageNo > 1) return [];
-
-    const novels: Plugin.NovelItem[] = [];
     let novel: Plugin.NovelItem;
-    let url = this.site;
+    let $ = await this.getCheerio(this.site);
-    let $ = await this.getCheerio(url);
+
-    $('li.page_item.page-item-866.page_item_has_children ul.children li').each(
+    const categories: Cheerio<Element>[] = [
-      (i, elem) => {
+      $('li.page_item').find('a:contains("Séries")').parent(),
-        let novelName = $(elem).first().text().trim();
+      $('li.page_item').find('a:contains("Créations")').parent(),
-        if (novelName.charAt(novelName.length - 1) === '✔') {
+      $('li.page_item').find('a:contains("†")').parent(),
-          novelName = novelName.slice(0, -1);
+    ];
-        }
+
+    categories.forEach(cheerio => {
+      cheerio.find('ul.children li').each((i, elem) => {
+        const novelName = $(elem).first().text().trim().replace('✔', '');
         const novelUrl = $(elem).find('a').attr('href');
 
-        if (novelUrl && novelName) {
+        // Douluo Dalu is no good
+        if (novelUrl && novelName && novelName !== 'Douluo Dalu') {
           novel = {
             name: novelName,
-            cover: defaultCover,
             path: novelUrl.replace(this.site, ''),
           };
           novels.push(novel);
         }
-      },
+      });
+    });
+    return novels;
+  }
+
+  async getNovelsCovers(
+    novels: Plugin.NovelItem[],
+  ): Promise<Plugin.NovelItem[]> {
+    await Promise.all(
+      novels.map(async novel => {
+        const $novel = await this.getCheerio(this.site + novel.path);
+        novel.cover =
+          $novel('img[fetchpriority = "high"]').first().attr('src') ||
+          $novel('img.aligncenter').first().attr('src') ||
+          defaultCover;
+      }),
     );
     return novels;
   }
 
+  async popularNovels(pageNo: number, {}: any): Promise<Plugin.NovelItem[]> {
+    const PAGE_SIZE = 5;
+
+    if (!XiaowazPlugin.novels) XiaowazPlugin.novels = await this.getAllNovels();
+    let novels: Plugin.NovelItem[] = XiaowazPlugin.novels;
+
+    const totalPages = Math.ceil(novels.length / PAGE_SIZE);
+    if (pageNo > totalPages) return [];
+
+    // splitting novel list to make fewer requests for getting images
+    novels = novels.slice(PAGE_SIZE * (pageNo - 1), PAGE_SIZE * pageNo);
+
+    return await this.getNovelsCovers(novels);
+  }
+
   async parseNovel(novelPath: string): Promise<Plugin.SourceNovel> {
     const novel: Plugin.SourceNovel = {
       path: novelPath,
@@ -66,42 +103,51 @@ class XiaowazPlugin implements Plugin.PluginBase {
 
     novel.name = $('.card_title').text().trim();
     novel.cover =
-      $('.entry-content p img').first().attr('src') ||
+      $('img[fetchpriority = "high"]').first().attr('src') ||
-      $('.entry-content h4 img').first().attr('src') ||
+      $('img.aligncenter').first().attr('src') ||
       defaultCover;
 
     novel.status = NovelStatus.Ongoing;
     if (novel.name.charAt(novel.name.length - 1) === '✔') {
       novel.status = NovelStatus.Completed;
       novel.name = novel.name.slice(0, -1);
+    } else if (novelPath.startsWith('/series-abandonnees')) {
+      novel.status = NovelStatus.Cancelled;
     }
 
     const entryContentText = $('.entry-content').text();
     novel.author = this.getAuthor(entryContentText);
     novel.genres = this.getGenres(entryContentText);
+    if (novelPath.startsWith('/oeuvres-originales')) {
+      novel.genres += novel.genres ? ', Oeuvre originale' : 'Oeuvre originale';
+    }
 
     const listeParagraphe: string[] = [];
 
     const PARAGRAPH_EXCLUDE_LIST = [
       'Écrit par',
       'Ecrit par',
-      'Sorties régulières le mardi et vendredi\u00A0!',
+      'Sorties régulières',
       'Auteur\u00A0:',
       'Statut VO\u00A0:',
       'Nom utilisé\u00A0:',
       'Auteur original\u00A0:',
       'Auteur original de l’oeuvre',
       '851 chapitres en',
+      'Index\u00A0:',
     ];
 
-    $('.entry-content p').each((index, element) => {
+    $('.entry-content > p').each((index, element) => {
       const balise = $(element);
-      const textbalise = balise.text();
+
-      if (
+      // remove chapter links
-        PARAGRAPH_EXCLUDE_LIST.some(keyword => textbalise.includes(keyword))
+      if (balise.find('a[href*="xiaowaz.fr/articles"]').length !== 0)
-      ) {
         return false;
-      }
+
+      const textbalise = balise.text();
+      if (PARAGRAPH_EXCLUDE_LIST.some(keyword => textbalise.includes(keyword)))
+        return false;
+
       if (
         !textbalise.includes('Genre') &&
         !textbalise.includes('Synopsis') &&
@@ -111,9 +157,8 @@ class XiaowazPlugin implements Plugin.PluginBase {
         !textbalise.includes(
           'Romance, Comédie, Action, VRMMO, Réincarnation, Futuriste',
         )
-      ) {
+      )
         listeParagraphe.push(textbalise);
-      }
     });
     novel.summary = listeParagraphe.join('\n').trim();
 
@@ -143,11 +188,13 @@ class XiaowazPlugin implements Plugin.PluginBase {
   getAuthor(text: string) {
     const regexAuthors = [
       /Écrit par([^\n]*). Traduction/i,
+      /Écrit par([^\n]*)./i,
       /Auteur original de l’œuvre\u00A0:([^\n]*)VO/i,
       /Auteur\u00A0:([^\n]*)sur/,
       /Auteur\u00A0:([^\n]*)/,
       /Auteure\u00A0:\u00A0([^\n]*)/,
       /Auteur original de l’oeuvre\u00A0:([^\n]*)/i,
+      /Auteur original\u00A0:([^\n]*)/i,
     ];
 
     for (const regex of regexAuthors) {
@@ -224,12 +271,12 @@ class XiaowazPlugin implements Plugin.PluginBase {
   ): Promise<Plugin.NovelItem[]> {
     if (pageNo !== 1) return [];
 
-    let popularNovels = this.popularNovels(1, {
+    if (!XiaowazPlugin.novels) XiaowazPlugin.novels = await this.getAllNovels();
-      showLatestNovels: true,
+    let popularNovels = XiaowazPlugin.novels;
-      filters: undefined,
-    });
 
-    let novels = (await popularNovels).filter(novel =>
+    // Normalize the text to remove accents and other special characters
+    // This ensures that the search term and novel names are compared accurately
+    let novels = popularNovels.filter(novel =>
       novel.name
         .toLowerCase()
         .normalize('NFD')
@@ -244,7 +291,7 @@ class XiaowazPlugin implements Plugin.PluginBase {
         ),
     );
 
-    return novels;
+    return await this.getNovelsCovers(novels);
   }
 
   async fetchImage(url: string): Promise<string | undefined> {