Create multisrc

src/screens/Browse/BrowseSettings.js

@@ -23,7 +23,7 @@ const BrowseSettings = ({ navigation }) => {
         showNovelUpdates = true,
         showMyAnimeList = true,
     } = useSelector((state) => state.sourceReducer);
-    const languages = ["English", "Spanish"];
+    const languages = ["English", "Spanish", "Japanese"];
     const { searchAllSources = false } = useSettings();
 
     const renderItem = ({ item }) => {

src/services/Source/source.js

@@ -1,4 +1,3 @@
-import { showToast } from "../../hooks/showToast";
 import { getSource } from "../../sources/sources";
 
 export const fetchSources = async () => {
@@ -11,8 +10,6 @@ export const fetchSources = async () => {
 };
 
 export const fetchNovel = async (sourceId, novelUrl) => {
-    if (sourceId === 37) showToast("LNMTL might take around 20-30 seconds.");
-
     const source = getSource(sourceId);
 
     const res = await source.parseNovelAndChapters(novelUrl);

src/sources/en/lnmtl.js

@@ -1,4 +1,5 @@
 import cheerio from "react-native-cheerio";
+import { showToast } from "../../hooks/showToast";
 import { htmlToText } from "../helpers/htmlToText";
 
 const baseUrl = "https://lnmtl.com/";
@@ -35,6 +36,8 @@ const popularNovels = async (page) => {
 };
 
 const parseNovelAndChapters = async (novelUrl) => {
+    showToast("LNMTL might take around 20-30 seconds.");
+
     const url = baseUrl + "novel/" + novelUrl;
 
     const result = await fetch(url);
@@ -130,7 +133,10 @@ const parseChapter = async (novelUrl, chapterUrl) => {
     $(".original").remove();
 
     let chapterText = $(".chapter-body").html();
-    chapterText = chapterName + "\n\n" + htmlToText(chapterText);
+    chapterText =
+        chapterName +
+        "\n\n" +
+        htmlToText(chapterText, { removeLineBreaks: false });
 
     let nextChapter = null;
     let prevChapter = null;

src/sources/helpers/htmlToText.js

@@ -1,40 +1,41 @@
-const htmlToText = (html) => {
-    return (
-        html
-            // Remove line breaks
-            .replace(/(?:\n|\r\n|\r)/gi, "")
-            // Turn <br>'s into single line breaks.
-            .replace(/<\s*br[^>]*>/gi, "\n")
-            // Turn </li>'s into line breaks.
-            .replace(/<\s*\/li[^>]*>/gi, "\n")
-            // Turn <p>'s into double line breaks.
-            .replace(/<\s*p[^>]*>/gi, "\n\n")
-            // Remove content in script tags.
-            .replace(/<\s*script[^>]*>[\s\S]*?<\/script>/gim, "")
-            // Remove content in style tags.
-            .replace(/<\s*style[^>]*>[\s\S]*?<\/style>/gim, "")
-            // Remove content in comments.
-            .replace(/<!--.*?-->/gim, "")
-            // Format anchor tags properly.
-            .replace(
-                /<\s*a[^>]*href=['"](.*?)['"][^>]*>([\s\S]*?)<\/\s*a\s*>/gi,
-                "$2"
-            )
-            .replace(/<\s*img[^>]*src=['"](.*?)['"][^>]*>/gi, "[$1]")
-            // Remove all remaining tags.
-            .replace(/(<([^>]+)>)/gi, "")
-            // Make sure there are never more than two
-            // consecutive linebreaks.
-            .replace(/\n{2,}/g, "\n\n")
-            // Remove tabs.
-            .replace(/\t/g, "")
-            // Remove newlines at the beginning of the text.
-            .replace(/^\n+/m, "")
-            // Replace multiple spaces with a single space.
-            .replace(/ {2,}/g, " ")
-            // Decode HTML entities.
-            .replace(/&([^;]+);/g, decodeHtmlEntity)
-    );
+const htmlToText = (html, options = {}) => {
+    const { removeLineBreaks = true } = options;
+
+    text = removeLineBreaks && html.replace(/(?:\n|\r\n|\r)/gi, "");
+    text = html
+        // Turn <br>'s into single line breaks.
+        .replace(/<\s*br[^>]*>/gi, "\n")
+        // Turn </li>'s into line breaks.
+        .replace(/<\s*\/li[^>]*>/gi, "\n")
+        // Turn <p>'s into double line breaks.
+        .replace(/<\s*p[^>]*>/gi, "\n\n")
+        // Remove content in script tags.
+        .replace(/<\s*script[^>]*>[\s\S]*?<\/script>/gim, "")
+        // Remove content in style tags.
+        .replace(/<\s*style[^>]*>[\s\S]*?<\/style>/gim, "")
+        // Remove content in comments.
+        .replace(/<!--.*?-->/gim, "")
+        // Format anchor tags properly.
+        .replace(
+            /<\s*a[^>]*href=['"](.*?)['"][^>]*>([\s\S]*?)<\/\s*a\s*>/gi,
+            "$2"
+        )
+        .replace(/<\s*img[^>]*src=['"](.*?)['"][^>]*>/gi, "[$1]")
+        // Remove all remaining tags.
+        .replace(/(<([^>]+)>)/gi, "")
+        // Make sure there are never more than two
+        // consecutive linebreaks.
+        .replace(/\n{2,}/g, "\n\n")
+        // Remove tabs.
+        .replace(/\t/g, "")
+        // Remove newlines at the beginning of the text.
+        .replace(/^\n+/m, "")
+        // Replace multiple spaces with a single space.
+        .replace(/ {2,}/g, " ")
+        // Decode HTML entities.
+        .replace(/&([^;]+);/g, decodeHtmlEntity);
+
+    return text;
 };
 
 const decodeHtmlEntity = (m, n) => {

src/sources/multisrc/MadaraGenerator.js

@@ -0,0 +1,72 @@
+import MadaraScraper from "./madara/MadaraScraper";
+
+const getPath = (extensionId) => {
+    const path = {
+        38: { novels: "manga", novel: "manga", chapter: "manga" },
+        39: { novels: "series", novel: "series", chapter: "series" },
+        40: { novels: "novels", novel: "novel", chapter: "novel" },
+        41: { novels: "manga", novel: "manga", chapter: "manga" },
+        42: { novels: "all-novels", novel: "manga", chapter: "manga" },
+        43: { novels: "series", novel: "series", chapter: "series" },
+        44: { novels: "series", novel: "series", chapter: "series" },
+        45: { novels: "novel", novel: "novel", chapter: "novel" },
+    };
+
+    return path[extensionId];
+};
+
+export const SkyNovelScraper = new MadaraScraper(
+    38,
+    "https://skynovel.org/",
+    "SkyNovel",
+    getPath(38)
+);
+
+export const NovelCakeScraper = new MadaraScraper(
+    39,
+    "https://novelcake.com/",
+    "NovelCake",
+    getPath(39)
+);
+
+export const NovelsRockScraper = new MadaraScraper(
+    40,
+    "https://novelsrock.com/",
+    "NovelsRock",
+    getPath(40)
+);
+
+export const ZinnNovelScraper = new MadaraScraper(
+    41,
+    "https://zinnovel.com/",
+    "ZinnNovel",
+    getPath(41)
+);
+
+export const NovelTranslateScraper = new MadaraScraper(
+    42,
+    "https://noveltranslate.com/",
+    "NovelTranslate",
+    getPath(42)
+);
+
+export const LunarLettersScraper = new MadaraScraper(
+    43,
+    "https://www.lunarletters.com/",
+    "LunarLetters",
+    getPath(43)
+);
+
+export const SleepyTranslationsScraper = new MadaraScraper(
+    44,
+    "https://sleepytranslations.com/",
+    "SleepyTranslations",
+    getPath(44)
+);
+
+export const FreeNovelScraper = new MadaraScraper(
+    45,
+    "https://freenovel.me/",
+    "FreeNovelMe",
+    getPath(45)
+);

src/sources/multisrc/madara/MadaraScraper.js

@@ -0,0 +1,220 @@
+import cheerio from "react-native-cheerio";
+import { htmlToText } from "../../helpers/htmlToText";
+
+class MadaraScraper {
+    constructor(sourceId, baseUrl, sourceName, path) {
+        this.sourceId = sourceId;
+        this.baseUrl = baseUrl;
+        this.sourceName = sourceName;
+        this.path = path;
+
+        // this.novelsScraper = this.novelsScraper.bind(this);
+        // this.novelScraper = this.novelScraper.bind(this);
+        // this.chapterScraper = this.chapterScraper.bind(this);
+        // this.searchScraper = this.searchScraper.bind(this);
+    }
+
+    async popularNovels(page) {
+        const totalPages = 100;
+        let url =
+            this.baseUrl +
+            this.path.novels +
+            "/page/" +
+            page +
+            "/?m_orderby=rating";
+        let sourceId = this.sourceId;
+
+        const result = await fetch(url);
+        const body = await result.text();
+
+        const $ = cheerio.load(body);
+
+        let novels = [];
+
+        $(".manga-title-badges").remove();
+
+        $(".page-item-detail").each(function (result) {
+            const novelName = $(this).find(".post-title").text().trim();
+            let image = $(this).find("img");
+            const novelCover = image.attr("data-src") || image.attr("src");
+
+            let novelUrl = $(this)
+                .find(".post-title")
+                .find("a")
+                .attr("href")
+                .split("/")[4];
+
+            const novel = {
+                sourceId,
+                novelName,
+                novelCover,
+                novelUrl,
+            };
+
+            novels.push(novel);
+        });
+
+        return { totalPages, novels };
+    }
+
+    async parseNovelAndChapters(novelUrl) {
+        const url = `${this.baseUrl}${this.path.novel}/${novelUrl}/`;
+
+        const result = await fetch(url);
+        const body = await result.text();
+
+        let $ = cheerio.load(body);
+
+        let novel = {};
+
+        novel.sourceId = this.sourceId;
+
+        novel.sourceName = this.sourceName;
+
+        novel.url = url;
+
+        novel.novelUrl = novelUrl;
+
+        $(".manga-title-badges").remove();
+
+        novel.novelName = $(".post-title > h1").text().trim();
+
+        novel.novelCover =
+            $(".summary_image > a > img").attr("data-src") ||
+            $(".summary_image > a > img").attr("src");
+
+        $(".post-content_item").each(function (result) {
+            const detailName = $(this)
+                .find(".summary-heading > h5")
+                .text()
+                .trim();
+            const detail = $(this).find(".summary-content").text().trim();
+
+            switch (detailName) {
+                case "Genre(s)":
+                    novel.genre = detail.replace(/[\t\n]/g, ",");
+                    break;
+                case "Author(s)":
+                    novel.author = detail;
+                    break;
+                case "Status":
+                    novel.status = detail;
+                    break;
+            }
+        });
+
+        novel.summary = $("div.summary__content").text().trim();
+
+        let novelChapters = [];
+
+        const novelId =
+            $(".rating-post-id").attr("value") ||
+            $("#manga-chapters-holder").attr("data-id");
+
+        let formData = new FormData();
+        formData.append("action", "manga_get_chapters");
+        formData.append("manga", novelId);
+
+        const data = await fetch(this.baseUrl + "wp-admin/admin-ajax.php", {
+            method: "POST",
+            body: formData,
+        });
+        const text = await data.text();
+
+        $ = cheerio.load(text);
+
+        $(".wp-manga-chapter").each(function (result) {
+            const chapterName = $(this)
+                .find("a")
+                .text()
+                .replace(/[\t\n]/g, "")
+                .trim();
+
+            const releaseDate = $(this)
+                .find("span")
+                .text()
+                .replace("Free", "")
+                .trim();
+
+            const chapterUrl = $(this).find("a").attr("href").split("/")[5];
+
+            novelChapters.push({ chapterName, releaseDate, chapterUrl });
+        });
+
+        console.log(novel);
+
+        novel.chapters = novelChapters.reverse();
+
+        return novel;
+    }
+
+    async parseChapter(novelUrl, chapterUrl) {
+        let sourceId = this.sourceId;
+
+        const url = `${this.baseUrl}${this.path.chapter}/${novelUrl}/${chapterUrl}`;
+
+        const result = await fetch(url);
+        const body = await result.text();
+
+        const $ = cheerio.load(body);
+
+        let chapterName =
+            $(".text-center").text() || $("#chapter-heading").text();
+
+        let chapterText = $(".text-left").html();
+        chapterText = htmlToText(chapterText);
+
+        let nextChapter = null;
+        let prevChapter = null;
+
+        const chapter = {
+            sourceId,
+            novelUrl,
+            chapterUrl,
+            chapterName,
+            chapterText,
+            nextChapter,
+            prevChapter,
+        };
+
+        return chapter;
+    }
+
+    async searchNovels(searchTerm) {
+        const url = `${this.baseUrl}?s=${searchTerm}&post_type=wp-manga`;
+
+        const result = await fetch(url);
+        const body = await result.text();
+
+        const $ = cheerio.load(body);
+
+        let novels = [];
+        let sourceId = this.sourceId;
+
+        $(".c-tabs-item__content").each(function (result) {
+            const novelName = $(this).find(".post-title").text().trim();
+
+            let image = $(this).find("img");
+            const novelCover = image.attr("data-src") || image.attr("src");
+
+            let novelUrl = $(this)
+                .find(".post-title")
+                .find("a")
+                .attr("href")
+                .split("/")[4];
+
+            const novel = {
+                sourceId,
+                novelName,
+                novelCover,
+                novelUrl,
+            };
+
+            novels.push(novel);
+        });
+
+        return novels;
+    }
+}
+
+export default MadaraScraper;

src/sources/sources.js

@@ -34,6 +34,16 @@ import RoyalRoadScraper from "./en/royalroad";
 import ScribbleHubScraper from "./en/scribblehub";
 import SyosetuScraper from "./jp/syosetu";
 import LNMTLScraper from "./en/lnmtl";
+import {
+    FreeNovelScraper,
+    LunarLettersScraper,
+    NovelCakeScraper,
+    NovelsRockScraper,
+    NovelTranslateScraper,
+    SkyNovelScraper,
+    SleepyTranslationsScraper,
+    ZinnNovelScraper,
+} from "./multisrc/MadaraGenerator";
 
 export const sources = [
     /**
@@ -230,6 +240,62 @@ export const sources = [
         icon: "https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/lnmtl/icon.png?raw=true",
         lang: "English",
     },
+    {
+        sourceId: 38,
+        sourceUrl: "https://skynovel.org/",
+        sourceName: "SkyNovel",
+        icon: "https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/multisrc/madara/icons/skynovel.png?raw=true",
+        lang: "English",
+    },
+    {
+        sourceId: 39,
+        sourceUrl: "https://novelcake.com/",
+        sourceName: "NovelCake",
+        icon: "https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/multisrc/madara/icons/novelcake.png?raw=true",
+        lang: "English",
+    },
+    {
+        sourceId: 40,
+        sourceUrl: "https://novelsrock.com/",
+        sourceName: "NovelsRock",
+        icon: "https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/multisrc/madara/icons/novelsrock.png?raw=true",
+        lang: "English",
+    },
+    {
+        sourceId: 41,
+        sourceUrl: "https://zinnovel.com/",
+        sourceName: "ZinnNovel",
+        icon: "https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/multisrc/madara/icons/zinnovel.png?raw=true",
+        lang: "English",
+    },
+    {
+        sourceId: 42,
+        sourceUrl: "https://noveltranslate.com/",
+        sourceName: "NovelTranslate",
+        icon: "https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/multisrc/madara/icons/noveltranslate.png?raw=true",
+        lang: "English",
+    },
+    {
+        sourceId: 43,
+        sourceUrl: "https://www.lunarletters.com/",
+        sourceName: "LunarLetters",
+        icon: "https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/multisrc/madara/icons/lunarletters.png?raw=true",
+        lang: "English",
+    },
+    {
+        sourceId: 44,
+        sourceUrl: "https://sleepytranslations.com/",
+        sourceName: "SleepyTranslations",
+        icon: "https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/multisrc/madara/icons/sleepytranslations.png?raw=true",
+        lang: "English",
+    },
+    {
+        sourceId: 45,
+        sourceUrl: "https://freenovel.me/",
+        sourceName: "FreeNovelMe",
+        icon: "https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/multisrc/madara/icons/freenovel.png?raw=true",
+        lang: "English",
+    },
 
     /**
      * Spanish
@@ -414,5 +480,29 @@ export const getSource = (sourceId) => {
         case 37: {
             return LNMTLScraper;
         }
+        case 38: {
+            return SkyNovelScraper;
+        }
+        case 39: {
+            return NovelCakeScraper;
+        }
+        case 40: {
+            return NovelsRockScraper;
+        }
+        case 41: {
+            return ZinnNovelScraper;
+        }
+        case 42: {
+            return NovelTranslateScraper;
+        }
+        case 43: {
+            return LunarLettersScraper;
+        }
+        case 44: {
+            return SleepyTranslationsScraper;
+        }
+        case 45: {
+            return FreeNovelScraper;
+        }
     }
 };