From b0639e79a8306e855a6a9cccfce3f7b27d23feee Mon Sep 17 00:00:00 2001 From: Rajarshee Chatterjee Date: Fri, 9 Jul 2021 08:51:08 +0530 Subject: [PATCH] Add new sources --- src/screens/More/About.js | 10 +- src/sources/en/lnmtl.js | 5 + src/sources/en/wuxiablog.js | 183 ++++++++++++++++++ src/sources/en/wuxiacity.js | 180 +++++++++++++++++ .../multisrc/madara/MadaraGenerator.js | 2 +- src/sources/sources.js | 44 +++++ 6 files changed, 418 insertions(+), 6 deletions(-) create mode 100644 src/sources/en/wuxiablog.js create mode 100644 src/sources/en/wuxiacity.js diff --git a/src/screens/More/About.js b/src/screens/More/About.js index 633ff424..0f920423 100644 --- a/src/screens/More/About.js +++ b/src/screens/More/About.js @@ -29,7 +29,7 @@ const AboutScreen = ({ navigation }) => { title="What's new" onPress={() => Linking.openURL( - "https://github.com/rajarsheechatterjee/lnreader/commits/main" + "https://github.com/LNReader/lnreader/commits/main" ) } theme={theme} @@ -45,20 +45,20 @@ const AboutScreen = ({ navigation }) => { /> Linking.openURL( - "https://github.com/rajarsheechatterjee/lnreader" + "https://github.com/LNReader/lnreader" ) } theme={theme} /> Linking.openURL( - "https://github.com/rajarsheechatterjee/lnreader-sources" + "https://github.com/LNReader/lnreader-sources" ) } theme={theme} diff --git a/src/sources/en/lnmtl.js b/src/sources/en/lnmtl.js index 61711746..877778ab 100644 --- a/src/sources/en/lnmtl.js +++ b/src/sources/en/lnmtl.js @@ -133,6 +133,11 @@ const parseChapter = async (novelUrl, chapterUrl) => { $(".original").remove(); let chapterText = $(".chapter-body").html(); + + if (!chapterText) { + chapterText = $(".alert.alert-warning").text(); + } + chapterText = chapterName + "\n\n" + diff --git a/src/sources/en/wuxiablog.js b/src/sources/en/wuxiablog.js new file mode 100644 index 00000000..a9a41e5b --- /dev/null +++ b/src/sources/en/wuxiablog.js @@ -0,0 +1,183 @@ +import cheerio from "react-native-cheerio"; +import { htmlToText } from "../helpers/htmlToText"; + +const sourceId = 48; + +const sourceName = "Wuxia.Blog"; + +const baseUrl = "https://www.wuxia.blog/"; + +const popularNovels = async (page) => { + const totalPages = 0; + let url = `${baseUrl}listNovels`; + + const result = await fetch(url); + const body = await result.text(); + + $ = cheerio.load(body); + + let novels = []; + + $("div.panel-body > table") + .find("tr") + .each(function () { + const novelName = $(this).find("td.novel").text(); + + if (novelName) { + const novelId = $(this).find("td:nth-child(1)").text(); + + const novelCover = baseUrl + "/data/image/" + novelId + ".jpg"; + + let novelUrl = $(this) + .find("td.novel > a") + .attr("href") + .replace("/novel/", ""); + + const novel = { + sourceId, + novelName, + novelCover, + novelUrl, + }; + + novels.push(novel); + } + }); + + return { totalPages, novels }; +}; + +const parseNovelAndChapters = async (novelUrl) => { + const url = `${baseUrl}novel/${novelUrl}/`; + + const result = await fetch(url); + const body = await result.text(); + + $ = cheerio.load(body); + + let novel = { + sourceId, + sourceName, + url, + novelUrl, + }; + + novel.novelName = $( + "body > div.container-fluid.text-center > div.row.content > div.col-sm-8.text-left > div:nth-child(6) > div.panel-heading.clearfix > h4" + ).text(); + + novel.novelCover = $('img[itemprop="image"]').attr("src"); + + novel.author = $( + "div.col-md-6 > div > div:nth-child(2) > div > div:nth-child(2) > a" + ).text(); + + novel.genre = $("div.row > div.col-md-6 > div > a:nth-child(4)").text(); + + novel.summary = $('div[itemprop="description"]').find("p").text().trim(); + + let novelChapters = []; + + $("table#chplist > #chapters > tr").each(function () { + const chapterName = $(this).find("a").text(); + + const releaseDate = $(this) + .find("td:nth-child(2)") + .text() + .replace(/-/g, "/"); + + const chapterUrl = $(this).find("a").attr("href").replace(url, ""); + + const chapter = { + chapterName, + releaseDate, + chapterUrl, + }; + + novelChapters.push(chapter); + }); + + novel.chapters = novelChapters.reverse(); + + return novel; +}; + +const parseChapter = async (novelUrl, chapterUrl) => { + const url = `${baseUrl}novel/${novelUrl}/${chapterUrl}`; + + const result = await fetch(url); + const body = await result.text(); + + $ = cheerio.load(body); + + let chapterName = $("div.panel-body.article").find("h4").text(); + + $("ul.pager").remove(); + + let chapterText = $("div.panel-body.article") + .html() + .replace(/(.*?)<\/span>/g, ""); + + chapterText = htmlToText(chapterText); + + let nextChapter = null; + let prevChapter = null; + + const chapter = { + sourceId, + novelUrl, + chapterUrl, + chapterName, + chapterText, + nextChapter, + prevChapter, + }; + + return chapter; +}; + +const searchNovels = async (searchTerm) => { + const url = `${baseUrl}?search=${searchTerm}`; + + const result = await fetch(url); + const body = await result.text(); + + $ = cheerio.load(body); + + let novels = []; + + $("#table") + .find("tr") + .each(function () { + const novelName = $(this).find("a").text().trim(); + + if (novelName) { + const novelCover = $(this).find("img").attr("src"); + + let novelUrl = $(this) + .find("a") + .attr("href") + .replace(baseUrl + "novel/", ""); + + const novel = { + sourceId, + novelName, + novelCover, + novelUrl, + }; + + novels.push(novel); + } + }); + + return novels; +}; + +const WuxiaBlogScraper = { + popularNovels, + parseNovelAndChapters, + parseChapter, + searchNovels, +}; + +export default WuxiaBlogScraper; diff --git a/src/sources/en/wuxiacity.js b/src/sources/en/wuxiacity.js new file mode 100644 index 00000000..340b4665 --- /dev/null +++ b/src/sources/en/wuxiacity.js @@ -0,0 +1,180 @@ +import cheerio from "react-native-cheerio"; +import { htmlToText } from "../helpers/htmlToText"; +import moment from "moment"; + +const sourceId = 49; + +const sourceName = "wuxia.city"; + +const baseUrl = "https://wuxia.city/"; + +const popularNovels = async (page) => { + const totalPages = 133; + let url = `${baseUrl}genre/all?page=` + page; + + const result = await fetch(url); + const body = await result.text(); + + $ = cheerio.load(body); + + let novels = []; + + $(".section-item.card").each(function () { + const novelName = $(this).find(".book-name").text(); + const novelCover = $(this).find("img").attr("src"); + const novelUrl = $(this).find("a").attr("href").replace("/book/", ""); + + const novel = { + sourceId, + novelName, + novelCover, + novelUrl, + }; + + novels.push(novel); + }); + + return { totalPages, novels }; +}; + +const parseNovelAndChapters = async (novelUrl) => { + const url = `${baseUrl}book/${novelUrl}`; + + const result = await fetch(url); + const body = await result.text(); + + let $ = cheerio.load(body); + + let novel = { + sourceId, + sourceName, + url, + novelUrl, + }; + + novel.novelName = $("h1.book-name").text(); + + novel.novelCover = $("div.book-img > img").attr("src"); + + novel.author = $("dl.author > dd").text(); + + novel.genre = $(".book-generes").text().trim(); + + novel.summary = $(".book-synopsis").text().replace("Synopsis", "").trim(); + + const chapterListUrl = `${baseUrl}book/${novelUrl}/chapter-list`; + + const chaptersHtml = await fetch(chapterListUrl); + const chapterHtmlToString = await chaptersHtml.text(); + + $ = cheerio.load(chapterHtmlToString); + + let novelChapters = []; + + $("ul.chapters > li").each(function () { + const chapterName = $(this) + .find(".chapter-name") + .text() + .trim() + .replace(/\n/, " "); + + let releaseDate = new Date(); + + let timeAgo = $(this).find(".chapter-time").text(); + + timeAgo = timeAgo.replace("one", 1); + + timeAgo = timeAgo.match(/\d+/)[0]; + + if (timeAgo.includes("months ago") || timeAgo.includes("month ago")) { + releaseDate.setMonth(releaseDate.getMonth() - timeAgo); + } + + if (timeAgo.includes("days ago")) { + releaseDate.setDate(releaseDate.getDate() - timeAgo); + } + + releaseDate = moment(releaseDate).format("MM/DD/YY"); + + const chapterUrl = $(this).find("a").attr("href").split("/").pop(); + + const chapter = { + chapterName, + releaseDate, + chapterUrl, + }; + + novelChapters.push(chapter); + }); + + novel.chapters = novelChapters.reverse(); + + return novel; +}; + +const parseChapter = async (novelUrl, chapterUrl) => { + const url = `${baseUrl}book/${novelUrl}/${chapterUrl}`; + + const result = await fetch(url); + const body = await result.text(); + + $ = cheerio.load(body); + + let chapterName = $(".chapter-name").text(); + + let chapterText = $(".chapter-content").html(); + chapterText = htmlToText(chapterText); + + let nextChapter = null; + let prevChapter = null; + + const chapter = { + sourceId, + novelUrl, + chapterUrl, + chapterName, + chapterText, + nextChapter, + prevChapter, + }; + + return chapter; +}; + +const searchNovels = async (searchTerm) => { + const url = `${baseUrl}search?q=${searchTerm}`; + + const result = await fetch(url); + const body = await result.text(); + + $ = cheerio.load(body); + + let novels = []; + + $(".section-item").each(function () { + const novelName = $(this).find(".book-name").text().trim(); + const novelCover = $(this).find("img").attr("src"); + + let novelUrl = $(this).find("a").attr("href").replace("/book/", ""); + + const novel = { + sourceId, + novelName, + novelCover, + novelUrl, + }; + + novels.push(novel); + }); + + return novels; +}; + +const WuxiaCityScraper = { + popularNovels, + parseNovelAndChapters, + parseChapter, + searchNovels, +}; + +export default WuxiaCityScraper; diff --git a/src/sources/multisrc/madara/MadaraGenerator.js b/src/sources/multisrc/madara/MadaraGenerator.js index 0582ec94..3f3c6a6d 100644 --- a/src/sources/multisrc/madara/MadaraGenerator.js +++ b/src/sources/multisrc/madara/MadaraGenerator.js @@ -80,7 +80,7 @@ export const FirstKissNovelScraper = new MadaraScraper( getPath(46) ); -export const DaoNovel = new MadaraScraper( +export const DaoNovelScraper = new MadaraScraper( 47, "https://daonovel.com/", "DaoNovel", diff --git a/src/sources/sources.js b/src/sources/sources.js index 0e3f9a6e..2cf36334 100644 --- a/src/sources/sources.js +++ b/src/sources/sources.js @@ -35,6 +35,8 @@ import ScribbleHubScraper from "./en/scribblehub"; import SyosetuScraper from "./jp/syosetu"; import LNMTLScraper from "./en/lnmtl"; import { + DaoNovelScraper, + FirstKissNovelScraper, FreeNovelScraper, LunarLettersScraper, NovelCakeScraper, @@ -44,6 +46,8 @@ import { SleepyTranslationsScraper, ZinnNovelScraper, } from "./multisrc/madara/MadaraGenerator"; +import WuxiaBlogScraper from "./en/wuxiablog"; +import WuxiaCityScraper from "./en/wuxiacity"; export const sources = [ /** @@ -296,6 +300,34 @@ export const sources = [ icon: "https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/multisrc/madara/icons/freenovel.png?raw=true", lang: "English", }, + { + sourceId: 46, + sourceUrl: "https://1stkissnovel.love/", + sourceName: "1stKissNovel", + icon: "https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/multisrc/madara/icons/1stkissnovel.png?raw=true", + lang: "English", + }, + { + sourceId: 47, + sourceUrl: "https://daonovel.com/", + sourceName: "DaoNovel", + icon: "https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/multisrc/madara/icons/daonovel.png?raw=true", + lang: "English", + }, + { + sourceId: 48, + sourceUrl: "https://www.wuxia.blog/", + sourceName: "Wuxia.Blog", + icon: "https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/wuxiablog/icon.png?raw=true", + lang: "English", + }, + { + sourceId: 49, + sourceUrl: "https://wuxia.city/", + sourceName: "Wuxia.City", + icon: "https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/wuxiacity/icon.png?raw=true", + lang: "English", + }, /** * Spanish @@ -504,5 +536,17 @@ export const getSource = (sourceId) => { case 45: { return FreeNovelScraper; } + case 46: { + return FirstKissNovelScraper; + } + case 47: { + return DaoNovelScraper; + } + case 48: { + return WuxiaBlogScraper; + } + case 49: { + return WuxiaCityScraper; + } } };