feat: Add Option to Revert Non-Page Plugin in NovelFire (#1908)

plugins/english/novelfire.ts

@@ -4,14 +4,25 @@ import { Plugin } from '@/types/plugin';
 import { NovelStatus } from '@libs/novelStatus';
 import { Filters, FilterTypes } from '@libs/filterInputs';
 import { defaultCover } from '@/types/constants';
+import { storage } from '@libs/storage';
 
 class NovelFire implements Plugin.PagePlugin {
   id = 'novelfire';
   name = 'Novel Fire';
-  version = '1.1.0';
+  version = '1.1.1';
   icon = 'src/en/novelfire/icon.png';
   site = 'https://novelfire.net/';
 
+  singlePage = storage.get('singlePage');
+  pluginSettings = {
+    singlePage: {
+      value: '',
+      label:
+        'Have all the chapters in a single page (Will be slower & use more internet data)',
+      type: FilterTypes.Switch,
+    },
+  };
+
   async getCheerio(url: string, search: boolean): Promise<CheerioAPI> {
     const r = await fetchApi(url);
     if (!r.ok && search != true)
@@ -79,6 +90,107 @@ class NovelFire implements Plugin.PagePlugin {
       .filter(novel => novel !== null);
   }
 
+  async getAllChapters(
+    novelPath: string,
+    pages: number,
+  ): Promise<Plugin.ChapterItem[]> {
+    const pagesArray = Array.from({ length: pages }, (_, i) => i + 1);
+    const allChapters: Plugin.ChapterItem[] = [];
+
+    // When pages > ~30, we get rate limited. To mitigate, split into chunks and retry chunk on rate limit with delay.
+    const chunkSize = 5; // 5 pages per chunk was tested to be a good balance between speed and rate limiting.
+    const retryCount = 10;
+    const sleepTime = 3.5; // Rate limit seems to be around ~10s, so usually 3 retries should be enough for another ~30 pages.
+
+    const chaptersArray: Plugin.ChapterItem[][] = [];
+
+    for (let i = 0; i < pagesArray.length; i += chunkSize) {
+      const pagesArrayChunk = pagesArray.slice(i, i + chunkSize);
+
+      const firstPage = pagesArrayChunk[0];
+      const lastPage = pagesArrayChunk[pagesArrayChunk.length - 1];
+
+      let attempt = 0;
+
+      while (attempt < retryCount) {
+        try {
+          // Parse all pages in chunk in parallel
+          const chaptersArrayChunk = await Promise.all(
+            pagesArrayChunk.map(page =>
+              this.getChaptersOnPage(novelPath, page.toString()),
+            ),
+          );
+
+          chaptersArray.push(...chaptersArrayChunk);
+          break;
+        } catch (err) {
+          if (err instanceof NovelFireThrottlingError) {
+            attempt += 1;
+            console.warn(
+              `[pages=${firstPage}-${lastPage}] Novel Fire is rate limiting requests. Retry attempt ${attempt + 1} in ${sleepTime} seconds...`,
+            );
+            if (attempt === retryCount) {
+              throw err;
+            }
+
+            // Sleep for X second before retrying
+            await new Promise(resolve => setTimeout(resolve, sleepTime * 1000));
+          } else {
+            throw err;
+          }
+        }
+      }
+    }
+
+    // Merge all chapters into a single array
+    for (const chapters of chaptersArray) {
+      allChapters.push(...chapters);
+    }
+
+    return allChapters;
+  }
+
+  async getChaptersOnPage(
+    novelPath: string,
+    page: string,
+  ): Promise<Plugin.ChapterItem[]> {
+    const url = `${this.site}${novelPath}/chapters?page=${page}`;
+    const result = await fetchApi(url);
+    const body = await result.text();
+
+    const loadedCheerio = load(body);
+
+    if (loadedCheerio.text().includes('You are being rate limited')) {
+      throw new NovelFireThrottlingError();
+    }
+
+    const chapters = loadedCheerio('.chapter-list li')
+      .map((index, ele) => {
+        const chapterName =
+          loadedCheerio(ele).find('a').attr('title') || 'No Title Found';
+        const chapterPath = loadedCheerio(ele).find('a').attr('href');
+
+        if (!chapterPath) return null;
+
+        return {
+          name: chapterName,
+          path: chapterPath.replace(this.site, ''),
+        };
+      })
+      .get()
+      .filter(chapter => chapter !== null) as Plugin.ChapterItem[];
+
+    return chapters;
+  }
+
+  getTotalPages($: CheerioAPI): number {
+    const totalChapters = $('.header-stats .icon-book-open')
+      .parent()
+      .text()
+      .trim();
+    return Math.ceil(parseInt(totalChapters) / 100);
+  }
+
   async parseNovel(
     novelPath: string,
   ): Promise<Plugin.SourceNovel & { totalPages: number }> {
@@ -134,38 +246,24 @@ class NovelFire implements Plugin.PagePlugin {
 
     novel.rating = parseFloat($('.nub').text().trim());
 
-    const totalChapters = $('.header-stats .icon-book-open')
-      .parent()
-      .text()
-      .trim();
-
-    novel.totalPages = Math.ceil(parseInt(totalChapters) / 100);
+    if (this.singlePage) {
+      novel.totalPages = 1;
+    } else {
+      novel.totalPages = this.getTotalPages($);
+    }
 
     return novel as Plugin.SourceNovel & { totalPages: number };
   }
 
   async parsePage(novelPath: string, page: string): Promise<Plugin.SourcePage> {
-    const url = `${this.site}${novelPath}/chapters?page=${page}`;
-    const result = await fetchApi(url);
-    const body = await result.text();
+    let chapters: Plugin.ChapterItem[];
 
-    const loadedCheerio = load(body);
-
-    const chapters = loadedCheerio('.chapter-list li')
-      .map((index, ele) => {
-        const chapterName =
-          loadedCheerio(ele).find('a').attr('title') || 'No Title Found';
-        const chapterPath = loadedCheerio(ele).find('a').attr('href');
-
-        if (!chapterPath) return null;
-
-        return {
-          name: chapterName,
-          path: chapterPath.replace(this.site, ''),
-        };
-      })
-      .get()
-      .filter(chapter => chapter !== null) as Plugin.ChapterItem[];
+    if (this.singlePage)
+      chapters = await this.getAllChapters(
+        novelPath,
+        this.getTotalPages(await this.getCheerio(this.site + novelPath, false)),
+      );
+    else chapters = await this.getChaptersOnPage(novelPath, page);
 
     return {
       chapters,