fix(fictioneer): migrate Illusia to the generator and add per-source overrides (#2567)

plugins/multisrc/fictioneer/sources.json

@@ -65,5 +65,25 @@
       "versionIncrements": 1,
       "browsePage": "stories"
     }
+  },
+  {
+    "id": "illusia",
+    "sourceSite": "https://illusia.com.br",
+    "sourceName": "Illusia",
+    "options": {
+      "browsePage": "historias",
+      "lang": "Portuguese",
+      "trimTrailingSlash": true,
+      "selectors": {
+        "browseCard": "#list-of-stories > li.illusia-card",
+        "searchCard": "#search-result-list > li.illusia-card",
+        "cardTitle": "h2.illusia-card__title > a",
+        "cardCover": "img.illusia-card__cover-img",
+        "novelTitle": "h1.illusia-single-story__title",
+        "novelAuthor": "span.illusia-single-story__credits-primary a.author",
+        "novelCover": "img.illusia-single-story__cover-img",
+        "novelSummary": "div.illusia-single-story__description"
+      }
+    }
   }
 ]

plugins/multisrc/fictioneer/template.ts

@@ -4,10 +4,35 @@ import { Plugin } from '@/types/plugin';
 import { NovelStatus } from '@libs/novelStatus';
 import { Filters } from '@libs/filterInputs';
 
+type FictioneerSelectors = {
+  browseCard: string;
+  searchCard: string;
+  cardTitle: string;
+  cardCover: string;
+  novelTitle: string;
+  novelAuthor: string;
+  novelCover: string;
+  novelSummary: string;
+};
+
+const defaultSelectors: FictioneerSelectors = {
+  browseCard:
+    '#featured-list > li > div > div, #list-of-stories > li > div > div',
+  searchCard: '#search-result-list > li > div > div',
+  cardTitle: 'h3 > a',
+  cardCover: 'a.cell-img:has(img)',
+  novelTitle: 'h1.story__identity-title',
+  novelAuthor: 'div.story__identity-meta',
+  novelCover: 'figure.story__thumbnail > a',
+  novelSummary: 'section.story__summary',
+};
+
 type FictioneerOptions = {
   browsePage: string;
   lang?: string;
   versionIncrements?: number;
+  trimTrailingSlash?: boolean;
+  selectors?: Partial<FictioneerSelectors>;
 };
 
 export type FictioneerMetadata = {
@@ -24,6 +49,7 @@ export class FictioneerPlugin implements Plugin.PluginBase {
   site: string;
   version: string;
   options: FictioneerOptions;
+  selectors: FictioneerSelectors;
   filters: Filters | undefined = undefined;
 
   constructor(metadata: FictioneerMetadata) {
@@ -32,8 +58,14 @@ export class FictioneerPlugin implements Plugin.PluginBase {
     this.icon = `multisrc/fictioneer/${metadata.id.toLowerCase()}/icon.png`;
     this.site = metadata.sourceSite;
     const versionIncrements = metadata.options?.versionIncrements || 0;
-    this.version = `1.1.${0 + versionIncrements}`;
+    this.version = `1.2.${0 + versionIncrements}`;
     this.options = metadata.options;
+    this.selectors = { ...defaultSelectors, ...metadata.options?.selectors };
+  }
+
+  private toPath(url: string): string {
+    const path = new URL(url, this.site).pathname.substring(1);
+    return this.options.trimTrailingSlash ? path.replace(/\/$/, '') : path;
   }
 
   private parseNovels(
@@ -43,16 +75,19 @@ export class FictioneerPlugin implements Plugin.PluginBase {
     return loadedCheerio(selector)
       .map((i, el) => {
         const element = loadedCheerio(el);
-        const novelName = element.find('h3 > a').text();
+        const title = element.find(this.selectors.cardTitle);
-        const novelCover = element.find('a.cell-img:has(img)').attr('href');
+        const novelName = title.text();
-        const novelUrl = element.find('h3 > a').attr('href');
+        const cover = element.find(this.selectors.cardCover);
+        const novelCover =
+          cover.attr('data-src') || cover.attr('src') || cover.attr('href');
+        const novelUrl = title.attr('href');
 
         if (!novelUrl) return;
 
         return {
           name: novelName,
           cover: novelCover,
-          path: new URL(novelUrl, this.site).pathname.substring(1),
+          path: this.toPath(novelUrl),
         };
       })
       .toArray();
@@ -60,11 +95,20 @@ export class FictioneerPlugin implements Plugin.PluginBase {
 
   async popularNovels(
     pageNo: number,
-    // {
+    { showLatestNovels }: Plugin.PopularNovelsOptions<typeof this.filters>,
-    //   showLatestNovels,
-    //   filters,
-    // }: Plugin.PopularNovelsOptions<typeof this.filters>,
   ): Promise<Plugin.NovelItem[]> {
+    if (showLatestNovels) {
+      // Latest updates come from a search results page, so use searchCard.
+      const req = await fetchApi(
+        this.site +
+          `/${pageNo === 1 ? '' : 'page/' + pageNo + '/'}?s=&post_type=fcn_story&orderby=modified&order=desc`,
+      );
+      const body = await req.text();
+      const loadedCheerio = loadCheerio(body);
+
+      return this.parseNovels(loadedCheerio, this.selectors.searchCard);
+    }
+
     const req = await fetchApi(
       this.site +
         '/' +
@@ -75,10 +119,7 @@ export class FictioneerPlugin implements Plugin.PluginBase {
     const body = await req.text();
     const loadedCheerio = loadCheerio(body);
 
-    return this.parseNovels(
+    return this.parseNovels(loadedCheerio, this.selectors.browseCard);
-      loadedCheerio,
-      '#featured-list > li > div > div, #list-of-stories > li > div > div',
-    );
   }
 
   async parseNovel(novelPath: string): Promise<Plugin.SourceNovel> {
@@ -88,52 +129,78 @@ export class FictioneerPlugin implements Plugin.PluginBase {
 
     const novel: Plugin.SourceNovel = {
       path: novelPath,
-      name: loadedCheerio('h1.story__identity-title').text(),
+      name: loadedCheerio(this.selectors.novelTitle).text(),
     };
 
     // novel.artist = '';
-    novel.author = loadedCheerio('div.story__identity-meta')
+    const author = loadedCheerio(this.selectors.novelAuthor).text();
-      .text()
+    // The default selector matches the "Author: X | ..." meta line, while an
-      .split('|')[0]
+    // override is expected to match the author name itself.
-      .replace('Author: ', '')
+    novel.author = this.options.selectors?.novelAuthor
-      .replace('by ', '')
+      ? author.trim()
-      .trim();
+      : author.split('|')[0].replace('Author: ', '').replace('by ', '').trim();
-    novel.cover = loadedCheerio('figure.story__thumbnail > a').attr('href');
+    const cover = loadedCheerio(this.selectors.novelCover);
+    novel.cover =
+      cover.attr('data-src') || cover.attr('src') || cover.attr('href');
     novel.genres = loadedCheerio('div.tag-group > a, section.tag-group > a')
       .map((i, el) => loadedCheerio(el).text())
       .toArray()
       .join(',');
 
-    loadedCheerio('section.story__summary .related-stories-block').remove();
+    const summary = loadedCheerio(this.selectors.novelSummary);
-    novel.summary = loadedCheerio('section.story__summary').text();
+    summary.find('.related-stories-block, section.small-card-block').remove();
+    summary.find('p').after('\n\n');
+    summary.find('br').after('\n');
+    novel.summary = summary
+      .text()
+      .trim()
+      .replace(/\n{3,}/g, '\n\n');
 
     novel.chapters = loadedCheerio('li.chapter-group__list-item._publish')
       .filter((i, el) => !el.attribs['class'].includes('_password'))
       .filter(
         (i, el) =>
-          !loadedCheerio(el)
+          !(loadedCheerio(el).find('i').first().attr('class') || '').includes(
-            .find('i')
+            'fa-lock',
-            .first()!
+          ),
-            .attr('class')!
-            .includes('fa-lock'),
       )
       .map((i, el) => {
         const chapterName = loadedCheerio(el).find('a').text();
         const chapterUrl = loadedCheerio(el).find('a').attr('href');
 
         if (!chapterUrl) return;
-        return {
+        const chapter: Plugin.ChapterItem = {
           name: chapterName,
-          path: new URL(chapterUrl, this.site).pathname.substring(1),
+          path: this.toPath(chapterUrl),
         };
+        const chapterNumber = chapterName.match(
+          /^(?:cap[íi]tulo|chapter|ch\.?)\s*(\d+(?:\.\d+)?)/i,
+        );
+        if (chapterNumber) chapter.chapterNumber = Number(chapterNumber[1]);
+        return chapter;
       })
       .toArray();
 
-    const status = loadedCheerio('span.story__status').text().trim();
+    // The status class ("_" + lowercase Fictioneer status) is
-    if (status === 'Ongoing') novel.status = NovelStatus.Ongoing;
+    // language-independent; the English text is a fallback.
-    if (status === 'Completed') novel.status = NovelStatus.Completed;
+    const statusElement = loadedCheerio('span.story__status');
-    if (status === 'Cancelled') novel.status = NovelStatus.Cancelled;
+    const status = statusElement.text().trim();
-    if (status === 'Hiatus') novel.status = NovelStatus.OnHiatus;
+    if (statusElement.hasClass('_ongoing') || status === 'Ongoing')
+      novel.status = NovelStatus.Ongoing;
+    if (
+      statusElement.hasClass('_completed') ||
+      statusElement.hasClass('_oneshot') ||
+      status === 'Completed'
+    )
+      novel.status = NovelStatus.Completed;
+    if (
+      statusElement.hasClass('_canceled') ||
+      status === 'Canceled' ||
+      status === 'Cancelled'
+    )
+      novel.status = NovelStatus.Cancelled;
+    if (statusElement.hasClass('_hiatus') || status === 'Hiatus')
+      novel.status = NovelStatus.OnHiatus;
 
     return novel;
   }
@@ -146,6 +213,10 @@ export class FictioneerPlugin implements Plugin.PluginBase {
 
     // chapterTransformJs HERE
 
+    loadedCheerio('section#chapter-content')
+      .find('script, style, iframe')
+      .remove();
+
     return loadedCheerio('section#chapter-content > div').html() || '';
   }
 
@@ -160,12 +231,9 @@ export class FictioneerPlugin implements Plugin.PluginBase {
     const body = await req.text();
     const loadedCheerio = loadCheerio(body);
 
-    return this.parseNovels(
+    return this.parseNovels(loadedCheerio, this.selectors.searchCard);
-      loadedCheerio,
-      '#search-result-list > li > div > div',
-    );
   }
 
-  // resolveUrl = (path: string, isNovel?: boolean) =>
+  resolveUrl = (path: string) =>
-  //   this.site + '/' + path + '/';
+    this.site.replace(/\/+$/, '') + '/' + path.replace(/^\/+|\/+$/g, '') + '/';
 }

plugins/portuguese/illusia.ts

@@ -1,329 +0,0 @@
-import { fetchApi } from '@libs/fetch';
-import { Plugin } from '@/types/plugin';
-import { Filters } from '@libs/filterInputs';
-import { load as loadCheerio } from 'cheerio';
-import { NovelStatus } from '@libs/novelStatus';
-import { defaultCover } from '@libs/defaultCover';
-
-class Illusia implements Plugin.PluginBase {
-  id = 'illusia';
-  name = 'Illusia';
-  icon = 'src/pt-br/illusia/icon.png';
-  site = 'https://illusia.com.br';
-  version = '1.0.2';
-  filters: Filters | undefined = undefined;
-
-  headers = {
-    'User-Agent':
-      'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36',
-  };
-
-  async popularNovels(
-    pageNo: number,
-    { showLatestNovels }: Plugin.PopularNovelsOptions<typeof this.filters>,
-  ): Promise<Plugin.NovelItem[]> {
-    const orderBy = showLatestNovels ? 'modified' : 'comment_count';
-    const pagePath = pageNo === 1 ? '' : `page/${pageNo}/`;
-
-    const url = `${this.site}/${pagePath}?s=&post_type=fcn_story&sentence=0&orderby=${orderBy}&order=desc&age_rating=Any&story_status=Any&miw=0&maw=0&genres=&fandoms=&characters=&tags=&warnings=&authors=&ex_genres=&ex_fandoms=&ex_characters=&ex_tags=&ex_warnings=&ex_authors=`;
-
-    const req = await fetchApi(url, { headers: this.headers });
-    const body = await req.text();
-    const loadedCheerio = loadCheerio(body);
-
-    const novels = loadedCheerio(
-      '#search-result-list > li, article.story, article.post, .card, .story-card, .ranking-item, ul.ranking-list li, .bsx, .book-item, .fcn-story',
-    )
-      .map((i, el) => {
-        const item = loadedCheerio(el);
-        const titleEl = item
-          .find(
-            '.card__title a, h2 a, h3 a, h4 a, .card-title a, .story-title a, .story__title a, .ranking-title a, .entry-title a, .tt',
-          )
-          .first();
-        const novelName = titleEl.text().trim();
-        const novelUrl =
-          titleEl.attr('href') || item.find('a').first().attr('href');
-
-        let novelCover =
-          item.find('img').attr('data-src') ||
-          item.find('img').attr('data-lazy-src') ||
-          item.find('img').attr('src') ||
-          item.find('.ranking-cover, .story-cover, .img-cover').attr('data-bg');
-
-        if (!novelCover) {
-          const bgElement = item.find('[style*="url("]');
-          const styleAttr = bgElement.length
-            ? bgElement.attr('style')
-            : item.attr('style');
-
-          if (styleAttr) {
-            const match = styleAttr.match(/url\(['"]?([^'"]+)['"]?\)/i);
-            if (match) novelCover = match[1];
-          }
-        }
-
-        if (!novelName || !novelUrl) return null;
-
-        if (novelCover && novelCover.startsWith('/')) {
-          novelCover = this.site + novelCover;
-        }
-
-        return {
-          name: novelName,
-          cover: novelCover || defaultCover,
-          path: novelUrl
-            .replace(this.site, '')
-            .replace(/^\//, '')
-            .replace(/\/$/, ''),
-        } as Plugin.NovelItem;
-      })
-      .toArray()
-      .filter(novel => novel !== null) as Plugin.NovelItem[];
-
-    const uniqueNovels = Array.from(
-      new Map(novels.map(item => [item.path, item])).values(),
-    );
-    return uniqueNovels;
-  }
-
-  async parseNovel(novelPath: string): Promise<Plugin.SourceNovel> {
-    const req = await fetchApi(`${this.site}/${novelPath}/`, {
-      headers: this.headers,
-    });
-    const body = await req.text();
-    const loadedCheerio = loadCheerio(body);
-
-    const novel: Plugin.SourceNovel = {
-      path: novelPath,
-      name: loadedCheerio('h1.story__identity-title, h1.post-title')
-        .text()
-        .trim(),
-    };
-
-    let author =
-      loadedCheerio(
-        'span.custom-story-info a.author, a[href*="/author/"], a[rel="author"]',
-      )
-        .first()
-        .text()
-        .trim() ||
-      loadedCheerio(
-        '.story__author, .story-author, .author-name, .post-author, [class*="__author"]',
-      )
-        .first()
-        .text()
-        .trim();
-
-    if (!author) {
-      const metaText = loadedCheerio(
-        '.story__identity-meta, .story-meta, .custom-story-info',
-      )
-        .text()
-        .trim();
-      if (metaText) {
-        author = metaText
-          .split('|')[0]
-          .replace(/^(Autor[a]?|Por|Author|by)[\s:]*/i, '')
-          .trim();
-      }
-    }
-    novel.author = author || 'Desconhecido';
-
-    novel.cover =
-      loadedCheerio('figure.story__thumbnail img').attr('data-src') ||
-      loadedCheerio('figure.story__thumbnail img').attr('src') ||
-      loadedCheerio('.story__thumbnail img').attr('data-src') ||
-      loadedCheerio('.story__thumbnail img').attr('src') ||
-      loadedCheerio('figure.story__thumbnail > a').attr('href') ||
-      defaultCover;
-
-    novel.genres = loadedCheerio(
-      'div.tag-group > a, section.tag-group > a, .genres a',
-    )
-      .map((i, el) => loadedCheerio(el).text().trim())
-      .toArray()
-      .join(',');
-
-    let summaryHtml =
-      loadedCheerio(
-        'section.story__summary, div.story__summary, .summary',
-      ).html() || '';
-    summaryHtml = summaryHtml
-      .replace(/<br\s*\/?>/gi, '\n')
-      .replace(/<\/p>/gi, '\n\n')
-      .replace(/<\/div>/gi, '\n');
-    novel.summary = loadCheerio(summaryHtml)
-      .text()
-      .trim()
-      .replace(/\n{3,}/g, '\n\n');
-
-    const chapterElements = loadedCheerio(
-      'li.chapter-group__list-item, ul.chapter-list li, .chapters li, .chapter-item',
-    );
-
-    novel.chapters = chapterElements
-      .map((i, el) => {
-        const item = loadedCheerio(el);
-
-        const aTag = item.find('a').first();
-        const chapterName = aTag.text().trim();
-        const chapterUrl = aTag.attr('href');
-
-        if (!chapterUrl) return null;
-
-        const chapterNumberMatch =
-          chapterName.match(/(?:cap[íi]tulo|cap\.?|ch\.?)\s*(\d+(\.\d+)?)/i) ||
-          chapterName.match(/^(\d+(\.\d+)?)/);
-        const chapterNumber = chapterNumberMatch
-          ? Number(chapterNumberMatch[1])
-          : undefined;
-
-        const chapter: Plugin.ChapterItem = {
-          name: chapterName,
-          path: chapterUrl
-            .replace(this.site, '')
-            .replace(/^\//, '')
-            .replace(/\/$/, ''),
-        };
-
-        if (chapterNumber !== undefined) {
-          chapter.chapterNumber = chapterNumber;
-        }
-
-        return chapter;
-      })
-      .toArray()
-      .filter(chapter => chapter !== null) as Plugin.ChapterItem[];
-
-    const metaBlockText =
-      loadedCheerio('div.story__identity-meta, .story-meta').text() || '';
-    const metaParts = metaBlockText.split('|').map(p => p.trim());
-
-    let statusText = loadedCheerio('span.story__status')
-      .text()
-      .trim()
-      .toLowerCase();
-    if (!statusText && metaParts.length > 1) {
-      statusText = metaBlockText.toLowerCase();
-    }
-
-    if (
-      statusText.includes('ongoing') ||
-      statusText.includes('andamento') ||
-      statusText.includes('lançando') ||
-      statusText.includes('ativa')
-    )
-      novel.status = NovelStatus.Ongoing;
-    else if (
-      statusText.includes('completed') ||
-      statusText.includes('completo')
-    )
-      novel.status = NovelStatus.Completed;
-    else if (
-      statusText.includes('cancelled') ||
-      statusText.includes('cancelado') ||
-      statusText.includes('dropado')
-    )
-      novel.status = NovelStatus.Cancelled;
-    else if (
-      statusText.includes('hiatus') ||
-      statusText.includes('hiato') ||
-      statusText.includes('pausado')
-    )
-      novel.status = NovelStatus.OnHiatus;
-    else novel.status = NovelStatus.Unknown;
-
-    return novel;
-  }
-
-  async parseChapter(chapterPath: string): Promise<string> {
-    const req = await fetchApi(`${this.site}/${chapterPath}/`, {
-      headers: this.headers,
-    });
-    const body = await req.text();
-    const loadedCheerio = loadCheerio(body);
-
-    const chapterContent = loadedCheerio(
-      'section#chapter-content > div, div.chapter-content',
-    );
-    chapterContent
-      .find(
-        'script, style, iframe, .patreon-popup, .fcn-notice, .fictioneer-notice, div.card',
-      )
-      .remove();
-
-    return chapterContent.html() || '';
-  }
-
-  async searchNovels(
-    searchTerm: string,
-    pageNo: number,
-  ): Promise<Plugin.NovelItem[]> {
-    const pagePath = pageNo === 1 ? '' : `page/${pageNo}/`;
-
-    const url = `${this.site}/${pagePath}?s=${encodeURIComponent(searchTerm)}&post_type=fcn_story&sentence=0&orderby=relevance&order=desc&age_rating=Any&story_status=Any&miw=0&maw=0&genres=&fandoms=&characters=&tags=&warnings=&authors=&ex_genres=&ex_fandoms=&ex_characters=&ex_tags=&ex_warnings=&ex_authors=`;
-
-    const req = await fetchApi(url, { headers: this.headers });
-    const body = await req.text();
-    const loadedCheerio = loadCheerio(body);
-
-    const novels = loadedCheerio(
-      '#search-result-list > li, article.story, article.post, .card, .story-card, .ranking-item, ul.ranking-list li, .bsx, .book-item, .fcn-story',
-    )
-      .map((i, el) => {
-        const item = loadedCheerio(el);
-        const titleEl = item
-          .find(
-            '.card__title a, h2 a, h3 a, h4 a, .card-title a, .story-title a, .story__title a, .ranking-title a, .entry-title a, .tt',
-          )
-          .first();
-        const novelName = titleEl.text().trim();
-        const novelUrl =
-          titleEl.attr('href') || item.find('a').first().attr('href');
-
-        let novelCover =
-          item.find('img').attr('data-src') ||
-          item.find('img').attr('data-lazy-src') ||
-          item.find('img').attr('src') ||
-          item.find('.ranking-cover, .story-cover, .img-cover').attr('data-bg');
-
-        if (!novelCover) {
-          const bgElement = item.find('[style*="url("]');
-          const styleAttr = bgElement.length
-            ? bgElement.attr('style')
-            : item.attr('style');
-          if (styleAttr) {
-            const match = styleAttr.match(/url\(['"]?([^'"]+)['"]?\)/i);
-            if (match) novelCover = match[1];
-          }
-        }
-
-        if (!novelName || !novelUrl) return null;
-
-        if (novelCover && novelCover.startsWith('/')) {
-          novelCover = this.site + novelCover;
-        }
-
-        return {
-          name: novelName,
-          cover: novelCover || defaultCover,
-          path: novelUrl
-            .replace(this.site, '')
-            .replace(/^\//, '')
-            .replace(/\/$/, ''),
-        } as Plugin.NovelItem;
-      })
-      .toArray()
-      .filter(novel => novel !== null) as Plugin.NovelItem[];
-
-    const uniqueNovels = Array.from(
-      new Map(novels.map(item => [item.path, item])).values(),
-    );
-    return uniqueNovels;
-  }
-
-  // resolveUrl = (path: string, isNovel?: boolean) => `${this.site}/${path}/`;
-}
-
-export default new Illusia();