Add new sources

2021-07-09 08:51:08 +05:30
parent 5931a0dc07
commit b0639e79a8
6 changed files with 418 additions and 6 deletions
@@ -29,7 +29,7 @@ const AboutScreen = ({ navigation }) => {
                        title="What's new"
                        onPress={() =>
                            Linking.openURL(
-                                "https://github.com/rajarsheechatterjee/lnreader/commits/main"
+                                "https://github.com/LNReader/lnreader/commits/main"
                            )
                        }
                        theme={theme}
@@ -45,20 +45,20 @@ const AboutScreen = ({ navigation }) => {
                    />
                    <ListItem
                        title="Github"
-                        description="https://github.com/rajarsheechatterjee/lnreader"
+                        description="https://github.com/LNReader/lnreader"
                        onPress={() =>
                            Linking.openURL(
-                                "https://github.com/rajarsheechatterjee/lnreader"
+                                "https://github.com/LNReader/lnreader"
                            )
                        }
                        theme={theme}
                    />
                    <ListItem
                        title="Sources"
-                        description="https://github.com/rajarsheechatterjee/lnreader-sources"
+                        description="https://github.com/LNReader/lnreader-sources"
                        onPress={() =>
                            Linking.openURL(
-                                "https://github.com/rajarsheechatterjee/lnreader-sources"
+                                "https://github.com/LNReader/lnreader-sources"
                            )
                        }
                        theme={theme}
@@ -133,6 +133,11 @@ const parseChapter = async (novelUrl, chapterUrl) => {
    $(".original").remove();

    let chapterText = $(".chapter-body").html();
+
+    if (!chapterText) {
+        chapterText = $(".alert.alert-warning").text();
+    }
+
    chapterText =
        chapterName +
        "\n\n" +
@@ -0,0 +1,183 @@
+import cheerio from "react-native-cheerio";
+import { htmlToText } from "../helpers/htmlToText";
+
+const sourceId = 48;
+
+const sourceName = "Wuxia.Blog";
+
+const baseUrl = "https://www.wuxia.blog/";
+
+const popularNovels = async (page) => {
+    const totalPages = 0;
+    let url = `${baseUrl}listNovels`;
+
+    const result = await fetch(url);
+    const body = await result.text();
+
+    $ = cheerio.load(body);
+
+    let novels = [];
+
+    $("div.panel-body > table")
+        .find("tr")
+        .each(function () {
+            const novelName = $(this).find("td.novel").text();
+
+            if (novelName) {
+                const novelId = $(this).find("td:nth-child(1)").text();
+
+                const novelCover = baseUrl + "/data/image/" + novelId + ".jpg";
+
+                let novelUrl = $(this)
+                    .find("td.novel > a")
+                    .attr("href")
+                    .replace("/novel/", "");
+
+                const novel = {
+                    sourceId,
+                    novelName,
+                    novelCover,
+                    novelUrl,
+                };
+
+                novels.push(novel);
+            }
+        });
+
+    return { totalPages, novels };
+};
+
+const parseNovelAndChapters = async (novelUrl) => {
+    const url = `${baseUrl}novel/${novelUrl}/`;
+
+    const result = await fetch(url);
+    const body = await result.text();
+
+    $ = cheerio.load(body);
+
+    let novel = {
+        sourceId,
+        sourceName,
+        url,
+        novelUrl,
+    };
+
+    novel.novelName = $(
+        "body > div.container-fluid.text-center > div.row.content > div.col-sm-8.text-left > div:nth-child(6) > div.panel-heading.clearfix > h4"
+    ).text();
+
+    novel.novelCover = $('img[itemprop="image"]').attr("src");
+
+    novel.author = $(
+        "div.col-md-6 > div > div:nth-child(2) > div > div:nth-child(2) > a"
+    ).text();
+
+    novel.genre = $("div.row > div.col-md-6 > div > a:nth-child(4)").text();
+
+    novel.summary = $('div[itemprop="description"]').find("p").text().trim();
+
+    let novelChapters = [];
+
+    $("table#chplist > #chapters > tr").each(function () {
+        const chapterName = $(this).find("a").text();
+
+        const releaseDate = $(this)
+            .find("td:nth-child(2)")
+            .text()
+            .replace(/-/g, "/");
+
+        const chapterUrl = $(this).find("a").attr("href").replace(url, "");
+
+        const chapter = {
+            chapterName,
+            releaseDate,
+            chapterUrl,
+        };
+
+        novelChapters.push(chapter);
+    });
+
+    novel.chapters = novelChapters.reverse();
+
+    return novel;
+};
+
+const parseChapter = async (novelUrl, chapterUrl) => {
+    const url = `${baseUrl}novel/${novelUrl}/${chapterUrl}`;
+
+    const result = await fetch(url);
+    const body = await result.text();
+
+    $ = cheerio.load(body);
+
+    let chapterName = $("div.panel-body.article").find("h4").text();
+
+    $("ul.pager").remove();
+
+    let chapterText = $("div.panel-body.article")
+        .html()
+        .replace(/<span(.*?)>(.*?)<\/span>/g, "");
+
+    chapterText = htmlToText(chapterText);
+
+    let nextChapter = null;
+    let prevChapter = null;
+
+    const chapter = {
+        sourceId,
+        novelUrl,
+        chapterUrl,
+        chapterName,
+        chapterText,
+        nextChapter,
+        prevChapter,
+    };
+
+    return chapter;
+};
+
+const searchNovels = async (searchTerm) => {
+    const url = `${baseUrl}?search=${searchTerm}`;
+
+    const result = await fetch(url);
+    const body = await result.text();
+
+    $ = cheerio.load(body);
+
+    let novels = [];
+
+    $("#table")
+        .find("tr")
+        .each(function () {
+            const novelName = $(this).find("a").text().trim();
+
+            if (novelName) {
+                const novelCover = $(this).find("img").attr("src");
+
+                let novelUrl = $(this)
+                    .find("a")
+                    .attr("href")
+                    .replace(baseUrl + "novel/", "");
+
+                const novel = {
+                    sourceId,
+                    novelName,
+                    novelCover,
+                    novelUrl,
+                };
+
+                novels.push(novel);
+            }
+        });
+
+    return novels;
+};
+
+const WuxiaBlogScraper = {
+    popularNovels,
+    parseNovelAndChapters,
+    parseChapter,
+    searchNovels,
+};
+
+export default WuxiaBlogScraper;
@@ -0,0 +1,180 @@
+import cheerio from "react-native-cheerio";
+import { htmlToText } from "../helpers/htmlToText";
+import moment from "moment";
+
+const sourceId = 49;
+
+const sourceName = "wuxia.city";
+
+const baseUrl = "https://wuxia.city/";
+
+const popularNovels = async (page) => {
+    const totalPages = 133;
+    let url = `${baseUrl}genre/all?page=` + page;
+
+    const result = await fetch(url);
+    const body = await result.text();
+
+    $ = cheerio.load(body);
+
+    let novels = [];
+
+    $(".section-item.card").each(function () {
+        const novelName = $(this).find(".book-name").text();
+        const novelCover = $(this).find("img").attr("src");
+        const novelUrl = $(this).find("a").attr("href").replace("/book/", "");
+
+        const novel = {
+            sourceId,
+            novelName,
+            novelCover,
+            novelUrl,
+        };
+
+        novels.push(novel);
+    });
+
+    return { totalPages, novels };
+};
+
+const parseNovelAndChapters = async (novelUrl) => {
+    const url = `${baseUrl}book/${novelUrl}`;
+
+    const result = await fetch(url);
+    const body = await result.text();
+
+    let $ = cheerio.load(body);
+
+    let novel = {
+        sourceId,
+        sourceName,
+        url,
+        novelUrl,
+    };
+
+    novel.novelName = $("h1.book-name").text();
+
+    novel.novelCover = $("div.book-img > img").attr("src");
+
+    novel.author = $("dl.author > dd").text();
+
+    novel.genre = $(".book-generes").text().trim();
+
+    novel.summary = $(".book-synopsis").text().replace("Synopsis", "").trim();
+
+    const chapterListUrl = `${baseUrl}book/${novelUrl}/chapter-list`;
+
+    const chaptersHtml = await fetch(chapterListUrl);
+    const chapterHtmlToString = await chaptersHtml.text();
+
+    $ = cheerio.load(chapterHtmlToString);
+
+    let novelChapters = [];
+
+    $("ul.chapters > li").each(function () {
+        const chapterName = $(this)
+            .find(".chapter-name")
+            .text()
+            .trim()
+            .replace(/\n/, " ");
+
+        let releaseDate = new Date();
+
+        let timeAgo = $(this).find(".chapter-time").text();
+
+        timeAgo = timeAgo.replace("one", 1);
+
+        timeAgo = timeAgo.match(/\d+/)[0];
+
+        if (timeAgo.includes("months ago") || timeAgo.includes("month ago")) {
+            releaseDate.setMonth(releaseDate.getMonth() - timeAgo);
+        }
+
+        if (timeAgo.includes("days ago")) {
+            releaseDate.setDate(releaseDate.getDate() - timeAgo);
+        }
+
+        releaseDate = moment(releaseDate).format("MM/DD/YY");
+
+        const chapterUrl = $(this).find("a").attr("href").split("/").pop();
+
+        const chapter = {
+            chapterName,
+            releaseDate,
+            chapterUrl,
+        };
+
+        novelChapters.push(chapter);
+    });
+
+    novel.chapters = novelChapters.reverse();
+
+    return novel;
+};
+
+const parseChapter = async (novelUrl, chapterUrl) => {
+    const url = `${baseUrl}book/${novelUrl}/${chapterUrl}`;
+
+    const result = await fetch(url);
+    const body = await result.text();
+
+    $ = cheerio.load(body);
+
+    let chapterName = $(".chapter-name").text();
+
+    let chapterText = $(".chapter-content").html();
+    chapterText = htmlToText(chapterText);
+
+    let nextChapter = null;
+    let prevChapter = null;
+
+    const chapter = {
+        sourceId,
+        novelUrl,
+        chapterUrl,
+        chapterName,
+        chapterText,
+        nextChapter,
+        prevChapter,
+    };
+
+    return chapter;
+};
+
+const searchNovels = async (searchTerm) => {
+    const url = `${baseUrl}search?q=${searchTerm}`;
+
+    const result = await fetch(url);
+    const body = await result.text();
+
+    $ = cheerio.load(body);
+
+    let novels = [];
+
+    $(".section-item").each(function () {
+        const novelName = $(this).find(".book-name").text().trim();
+        const novelCover = $(this).find("img").attr("src");
+
+        let novelUrl = $(this).find("a").attr("href").replace("/book/", "");
+
+        const novel = {
+            sourceId,
+            novelName,
+            novelCover,
+            novelUrl,
+        };
+
+        novels.push(novel);
+    });
+
+    return novels;
+};
+
+const WuxiaCityScraper = {
+    popularNovels,
+    parseNovelAndChapters,
+    parseChapter,
+    searchNovels,
+};
+
+export default WuxiaCityScraper;
@@ -80,7 +80,7 @@ export const FirstKissNovelScraper = new MadaraScraper(
    getPath(46)
 );

-export const DaoNovel = new MadaraScraper(
+export const DaoNovelScraper = new MadaraScraper(
    47,
    "https://daonovel.com/",
    "DaoNovel",
@@ -35,6 +35,8 @@ import ScribbleHubScraper from "./en/scribblehub";
 import SyosetuScraper from "./jp/syosetu";
 import LNMTLScraper from "./en/lnmtl";
 import {
+    DaoNovelScraper,
+    FirstKissNovelScraper,
    FreeNovelScraper,
    LunarLettersScraper,
    NovelCakeScraper,
@@ -44,6 +46,8 @@ import {
    SleepyTranslationsScraper,
    ZinnNovelScraper,
 } from "./multisrc/madara/MadaraGenerator";
+import WuxiaBlogScraper from "./en/wuxiablog";
+import WuxiaCityScraper from "./en/wuxiacity";

 export const sources = [
    /**
@@ -296,6 +300,34 @@ export const sources = [
        icon: "https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/multisrc/madara/icons/freenovel.png?raw=true",
        lang: "English",
    },
+    {
+        sourceId: 46,
+        sourceUrl: "https://1stkissnovel.love/",
+        sourceName: "1stKissNovel",
+        icon: "https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/multisrc/madara/icons/1stkissnovel.png?raw=true",
+        lang: "English",
+    },
+    {
+        sourceId: 47,
+        sourceUrl: "https://daonovel.com/",
+        sourceName: "DaoNovel",
+        icon: "https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/multisrc/madara/icons/daonovel.png?raw=true",
+        lang: "English",
+    },
+    {
+        sourceId: 48,
+        sourceUrl: "https://www.wuxia.blog/",
+        sourceName: "Wuxia.Blog",
+        icon: "https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/wuxiablog/icon.png?raw=true",
+        lang: "English",
+    },
+    {
+        sourceId: 49,
+        sourceUrl: "https://wuxia.city/",
+        sourceName: "Wuxia.City",
+        icon: "https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/wuxiacity/icon.png?raw=true",
+        lang: "English",
+    },

    /**
     * Spanish
@@ -504,5 +536,17 @@ export const getSource = (sourceId) => {
        case 45: {
            return FreeNovelScraper;
        }
+        case 46: {
+            return FirstKissNovelScraper;
+        }
+        case 47: {
+            return DaoNovelScraper;
+        }
+        case 48: {
+            return WuxiaBlogScraper;
+        }
+        case 49: {
+            return WuxiaCityScraper;
+        }
    }
 };