Plugins fixed (#903)

* test

* fast fix

* fix

* fix

* Update ranobelib.ts

scripts/multisrc/hotnovelpub/template.ts

@@ -2,7 +2,6 @@ import { fetchFile, fetchApi } from '@libs/fetch';
 import { Filters, FilterTypes } from '@libs/filterInputs';
 import { Plugin } from '@typings/plugin';
 import { NovelStatus } from '@libs/novelStatus';
-import { load as parseHTML } from 'cheerio';
 import dayjs from 'dayjs';
 
 export interface HotNovelPubMetadata {
@@ -33,7 +32,7 @@ class HotNovelPubPlugin implements Plugin.PluginBase {
     this.icon = `multisrc/hotnovelpub/${metadata.id.toLowerCase()}/icon.png`;
     this.site = metadata.sourceSite;
     this.apiSite = metadata.sourceSite.replace('://', '://api.');
-    this.version = '1.0.0';
+    this.version = '1.0.1';
     this.filters = metadata.filters;
     this.lang = metadata.options?.lang || 'en';
   }
@@ -50,20 +49,19 @@ class HotNovelPubPlugin implements Plugin.PluginBase {
 
     url += '/?page=' + (pageNo - 1) + '&limit=20';
 
-    const result = await fetchApi(url, {
+    const result: responseNovels = await fetchApi(url, {
       headers: {
         lang: this.lang,
       },
-    });
-    const json = (await result.json()) as responseNovels;
+    }).then(res => res.json());
     const novels: Plugin.NovelItem[] = [];
 
-    if (json.status && json.data.books.data?.length) {
-      json.data.books.data.forEach(novel =>
+    if (result.status && result.data.books.data?.length) {
+      result.data.books.data.forEach(novel =>
         novels.push({
           name: novel.name,
           cover: this.site + novel.image,
-          path: '/' + novel.slug,
+          path: novel.slug,
         }),
       );
     }
@@ -71,12 +69,14 @@ class HotNovelPubPlugin implements Plugin.PluginBase {
   }
 
   async parseNovel(novelPath: string): Promise<Plugin.SourceNovel> {
-    const result = await fetchApi(this.apiSite + '/book' + novelPath, {
-      headers: {
-        lang: this.lang,
+    const json: responseNovel = await fetchApi(
+      this.apiSite + '/book/' + novelPath,
+      {
+        headers: {
+          lang: this.lang,
+        },
       },
-    });
-    const json = (await result.json()) as responseNovel;
+    ).then(res => res.json());
 
     const novel: Plugin.SourceNovel = {
       name: json.data.book.name,
@@ -99,7 +99,7 @@ class HotNovelPubPlugin implements Plugin.PluginBase {
       json.data.chapters.forEach((chapter, chapterIndex) =>
         chapters.push({
           name: chapter.title,
-          path: '/' + chapter.slug,
+          path: chapter.slug,
           releaseTime: undefined,
           chapterNumber: (chapter.index || chapterIndex) + 1,
         }),
@@ -111,12 +111,13 @@ class HotNovelPubPlugin implements Plugin.PluginBase {
   }
 
   async parseChapter(chapterPath: string): Promise<string> {
-    const body = await fetchApi(this.site + chapterPath).then(res =>
+    const body = await fetchApi(this.resolveUrl(chapterPath)).then(res =>
       res.text(),
     );
-    const loadedCheerio = parseHTML(body);
 
-    let chapterText = loadedCheerio('#content-item').html() || '';
+    let chapterText =
+      body.match(/<div id="content-item" ([\s\S]*?)<\/div>/g)?.[0] || '';
+
     if (chapterText) {
       const result = await fetchApi(
         this.site + '/server/getContent?slug=' + chapterPath,
@@ -138,7 +139,7 @@ class HotNovelPubPlugin implements Plugin.PluginBase {
     searchTerm: string,
     pageNo: number,
   ): Promise<Plugin.NovelItem[]> {
-    const result = await fetchApi(this.apiSite + '/search', {
+    const result: responseSearch = await fetchApi(this.apiSite + '/search', {
       headers: {
         'Content-Type': 'application/json;charset=utf-8',
         Referer: this.site,
@@ -147,16 +148,14 @@ class HotNovelPubPlugin implements Plugin.PluginBase {
       },
       method: 'POST',
       body: JSON.stringify({ key_search: searchTerm }),
-    });
-    const json = (await result.json()) as responseSearch;
+    }).then(res => res.json());
     const novels: Plugin.NovelItem[] = [];
 
-    if (json.status && json.data.books?.length) {
-      json.data.books.forEach(novel =>
+    if (result.status && result.data.books?.length) {
+      result.data.books.forEach(novel =>
         novels.push({
           name: novel.name,
-          cover: undefined,
-          path: '/' + novel.slug,
+          path: novel.slug,
         }),
       );
     }
@@ -165,6 +164,7 @@ class HotNovelPubPlugin implements Plugin.PluginBase {
   }
 
   fetchImage = fetchFile;
+  resolveUrl = (path: string, isNovel?: boolean) => this.site + '/' + path;
 }
 
 interface responseNovels {

scripts/multisrc/ifreedom/template.ts

@@ -25,7 +25,7 @@ class IfreedomPlugin implements Plugin.PluginBase {
     this.name = metadata.sourceName;
     this.icon = `multisrc/ifreedom/${metadata.id.toLowerCase()}/icon.png`;
     this.site = metadata.sourceSite;
-    this.version = '1.0.0';
+    this.version = '1.0.1';
     this.filters = metadata.filters;
   }
 
@@ -139,20 +139,26 @@ class IfreedomPlugin implements Plugin.PluginBase {
     const body = await fetchApi(this.site + chapterPath).then(res =>
       res.text(),
     );
-    const loadedCheerio = parseHTML(body);
+    let chapterText =
+      body.match(/<article id="([\s\S]*?)<\/article>/g)?.[0] || '';
+    chapterText = chapterText.replace(/<script[^>]*>[\s\S]*?<\/script>/gim, '');
 
-    loadedCheerio('.entry-content img').each((index, element) => {
-      const srcset = loadedCheerio(element).attr('srcset')?.split?.(' ');
-      if (srcset?.length) {
-        loadedCheerio(element).removeAttr('srcset');
-        const bestlink: string[] = srcset.filter(url => url.startsWith('http'));
-        if (bestlink[bestlink.length - 1]) {
-          loadedCheerio(element).attr('src', bestlink[bestlink.length - 1]);
+    if (chapterText.includes('<img')) {
+      return chapterText.replace(/srcset="([^"]+)"/g, (match, src) => {
+        if (!src) return match;
+        const bestlink = src
+          .split(' ')
+          .filter((url: string) => url.startsWith('http'))
+          .pop();
+
+        if (bestlink) {
+          if (bestlink.startsWith('http')) return `src="${bestlink}"`;
+          return `src="${this.site}${bestlink}"`;
         }
-      }
-    });
+        return match;
+      });
+    }
 
-    const chapterText = loadedCheerio('.entry-content').html() || '';
     return chapterText;
   }
 

scripts/multisrc/readwn/generator.ts

@@ -1,37 +1,20 @@
-import { FilterTypes, Filters } from '../../../libs/filterInputs';
 import { ScrpitGeneratorFunction } from '../generate';
-import list from './sources.json';
-import defaultSettings from './settings.json';
 import { ReadwnMetadata } from './template';
+import list from './sources.json';
 import { readFileSync } from 'fs';
 import path from 'path';
 
 export const generateAll: ScrpitGeneratorFunction = function (name) {
-  return list
-    .map<ReadwnMetadata>(p => {
-      p.filters.tags.options.unshift(...defaultSettings.filters.tags.options);
-      p.filters.genres.options.unshift(
-        ...defaultSettings.filters.genres.options,
-      );
-      p.filters = Object.assign(defaultSettings.filters, p.filters);
-
-      const d = false;
-      const filters: Filters = {};
-      for (const k in p.filters) {
-        const f = p.filters[k as keyof typeof p.filters];
-        if (f) {
-          filters[k] = {
-            ...f,
-            type: FilterTypes.Picker,
-          };
-        }
-      }
-      return { ...p, filters: d ? undefined : filters };
-    })
-    .map((metadata: ReadwnMetadata) => {
-      console.log(`[${name}]: Generating`, metadata.id);
-      return generator(metadata);
-    });
+  return list.map((metadata: ReadwnMetadata) => {
+    try {
+      const { filters }: any = require(`./filters/${metadata.id}`);
+      metadata.filters = filters;
+    } catch (e) {}
+    console.log(
+      `[${name}] Generating: ${metadata.id}${' '.repeat(20 - metadata.id.length)} ${metadata.filters ? '🔎with filters🔍' : '🚫no filters🚫'}`,
+    );
+    return generator(metadata);
+  });
 };
 
 const generator = function generator(metadata: ReadwnMetadata) {
@@ -39,18 +22,9 @@ const generator = function generator(metadata: ReadwnMetadata) {
     encoding: 'utf-8',
   });
 
-  //metadata.filters = {
-  //  sort: { ...defaultSettings.filters.sort, type: FilterTypes.Picker },
-  //  status: { ...defaultSettings.filters.status, type: FilterTypes.Picker },
-  //  ...metadata.filters,
-  //};
-
   const pluginScript = `
-    ${readwnTemplate}
-const plugin = new ReadwnPlugin(${JSON.stringify(metadata).replace(
-    /"type":"([^"]+)"/g,
-    '"type":FilterTypes.$1',
-  )});
+${readwnTemplate}
+const plugin = new ReadwnPlugin(${JSON.stringify(metadata)});
 export default plugin;
     `.trim();
 

scripts/multisrc/readwn/get_filters.ts

@@ -0,0 +1,155 @@
+require('module-alias/register');
+import * as fs from 'fs';
+import * as cheerio from 'cheerio';
+import * as path from 'path';
+import { FilterTypes, FilterOption } from '@libs/filterInputs';
+const type: string[] = ['genres', 'status', 'sort'];
+
+async function getFilters(name: string, url: string) {
+  const html = await fetch(url + '/list/all/all-newstime-0.html').then(res =>
+    res.text(),
+  );
+  const $: cheerio.CheerioAPI = cheerio.load(html);
+  const filters: any = {
+    'sort': {
+      type: FilterTypes.Picker,
+      label: 'Sort By',
+      value: 'onclick',
+      options: [],
+    },
+    'status': {
+      type: FilterTypes.Picker,
+      label: 'Status',
+      value: 'all',
+      options: [],
+    },
+    'genres': {
+      type: FilterTypes.Picker,
+      label: 'Genre / Category',
+      value: '',
+      options: [],
+    },
+    'tags': {
+      type: FilterTypes.Picker,
+      label: 'Tags',
+      value: '',
+      options: [{ label: 'NONE', value: '' }],
+    },
+  };
+
+  $('ul.proplist')?.each?.(function (index, ulElement) {
+    if (!type[index]) return;
+    $(ulElement)
+      .find('li > a')
+      .each((indx, liElement) =>
+        filters[type[index]].options.push({
+          label: $(liElement).text(),
+          value: $(liElement).attr('href'),
+        }),
+      );
+    filters[type[index]].options = filters[type[index]].options.map(
+      (item: FilterOption) => {
+        let res = item.value;
+        if (index == 0) res = item.value.split('/')[2];
+        if (index == 1)
+          res = item.value.replace(/\/list\/all\/(.*?)-.*$/, '$1');
+        if (index == 2) res = item.value.replace(/.*\/all-(.*?)-.*/, '$1');
+        return {
+          label: item.label,
+          value: res,
+        };
+      },
+    ) as FilterOption[];
+    filters[type[index]].options.sort((a: FilterOption, b: FilterOption) => {
+      if (a.label === 'All') {
+        return -1;
+      } else if (b.label === 'All') {
+        return 1;
+      }
+      return a.label.localeCompare(b.label);
+    });
+  });
+
+  const response = await fetch(url + '/browsetags/').then(res => res.text());
+  const loadedCheerio: cheerio.CheerioAPI = cheerio.load(response);
+  const allPage = loadedCheerio('.tag-letters > a')
+    .map((index, element) => loadedCheerio(element).attr('href'))
+    .get();
+  // ===================== tags ======================
+  for (let i = 0; i < allPage.length; i++) {
+    console.log('fetch', url + allPage[i]);
+    const resTags = await fetch(url + allPage[i]).then(res => res.text());
+    const $: cheerio.CheerioAPI = cheerio.load(resTags);
+    $('.tag-items > li > a').each((index, element) =>
+      filters['tags'].options.push({
+        label: $(element).text()?.trim(),
+        value: $(element)
+          .attr('href')
+          ?.split('/')
+          ?.pop()
+          ?.replace('-0.html', ''),
+      }),
+    );
+    let nextPage = $('.pagination > li:last-child > a').attr('href');
+    if (nextPage) {
+      const allpage = parseInt(nextPage.replace(/[^0-9]/g, '') || '0', 10);
+      for (let pageNo = 0; pageNo < allpage; pageNo++) {
+        await sleep(3000);
+        console.log(
+          'fetch',
+          url + allPage[i].replace('-0.html', `-${pageNo + 1}.html`),
+        );
+        const resTags = await fetch(
+          url + allPage[i].replace('-0.html', `-${pageNo + 1}.html`),
+        ).then(res => res.text());
+        const $: cheerio.CheerioAPI = cheerio.load(resTags);
+
+        $('.tag-items > li > a').each((index, element) =>
+          filters['tags'].options.push({
+            label: $(element).text()?.trim(),
+            value: $(element).attr('href')?.split('/')?.pop()?.split('-')?.[0],
+          }),
+        );
+      }
+    }
+    await sleep(3000);
+  }
+
+  fs.writeFileSync(
+    path.join(__dirname, 'filters', name + '.json'),
+    JSON.stringify({ filters }, null, 2),
+  );
+  console.log(`✅Filters created successfully for ${name}✅`);
+}
+
+async function sleep(ms: number) {
+  return new Promise(resolve => setTimeout(resolve, ms));
+}
+
+async function askGetFilter() {
+  const readline = require('readline').createInterface({
+    input: process.stdin,
+    output: process.stdout,
+  });
+
+  const EREASE_PREV_LINE = '\x1b[1A\r\x1b[2K';
+  await readline.question(
+    'Enter the id of the site (same one as in sources.json): ',
+    async (name: string) => {
+      await readline.question(
+        EREASE_PREV_LINE + 'Enter the URL (same one as in sources.json): ',
+        async (url: string) => {
+          try {
+            await getFilters(name, url);
+          } catch (e: any) {
+            console.error('Error while getting filters from', url);
+            console.log(e.message || e);
+          }
+          readline.close();
+        },
+      );
+    },
+  );
+}
+
+askGetFilter();

scripts/multisrc/readwn/template.ts

@@ -10,7 +10,7 @@ export interface ReadwnMetadata {
   id: string;
   sourceSite: string;
   sourceName: string;
-  filters?: Filters;
+  filters?: any;
 }
 
 class ReadwnPlugin implements Plugin.PluginBase {
@@ -26,7 +26,7 @@ class ReadwnPlugin implements Plugin.PluginBase {
     this.name = metadata.sourceName;
     this.icon = `multisrc/readwn/${metadata.id.toLowerCase()}/icon.png`;
     this.site = metadata.sourceSite;
-    this.version = '1.0.0';
+    this.version = '1.0.1';
     this.filters = metadata.filters;
   }
 

scripts/multisrc/rulate/template.ts

@@ -25,7 +25,7 @@ class RulatePlugin implements Plugin.PluginBase {
     this.name = metadata.sourceName;
     this.icon = `multisrc/rulate/${metadata.id.toLowerCase()}/icon.png`;
     this.site = metadata.sourceSite;
-    this.version = '1.0.0';
+    this.version = '1.0.1';
     this.filters = metadata.filters;
   }
 
@@ -80,12 +80,10 @@ class RulatePlugin implements Plugin.PluginBase {
       formData.append('path', novelPath);
       formData.append('ok', 'Да');
 
-      await fetchApi(result.url, {
+      result = await fetchApi(result.url, {
         method: 'POST',
         body: formData,
       });
-
-      result = await fetchApi(this.site + novelPath);
     }
     const body = await result.text();
     const loadedCheerio = parseHTML(body);
@@ -183,14 +181,13 @@ class RulatePlugin implements Plugin.PluginBase {
     let result = await fetchApi(this.site + chapterPath);
     if (result.url.includes('mature?path=')) {
       const formData = new FormData();
+      formData.append('path', chapterPath.split('/').slice(0, 3).join('/'));
       formData.append('ok', 'Да');
 
-      await fetchApi(result.url, {
+      result = await fetchApi(result.url, {
         method: 'POST',
         body: formData,
       });
-
-      result = await fetchApi(this.site + chapterPath);
     }
     const body = await result.text();
     const loadedCheerio = parseHTML(body);
@@ -208,12 +205,11 @@ class RulatePlugin implements Plugin.PluginBase {
 
   async searchNovels(searchTerm: string): Promise<Plugin.NovelItem[]> {
     const novels: Plugin.NovelItem[] = [];
-    const result = await fetchApi(
+    const result: response[] = await fetchApi(
       this.site + '/search/autocomplete?query=' + searchTerm,
-    );
-    const json = (await result.json()) as response[];
+    ).then(res => res.json());
 
-    json.forEach(novel => {
+    result.forEach(novel => {
       const name = novel.title_one + ' / ' + novel.title_two;
       if (!novel.url) return;