"); - chapterName = chapterText.shift(); - } else { - $("p").each((i, el) => { - if ($(el).css("display") !== "none") { - chapterText += $(el).text() + "\n"; - } - }); - chapterText = chapterText.split("\n"); - chapterName = chapterText.shift(); - } - - chapterText = chapterText.join("\n"); - - const prevChapter = $(".pre").attr("href") - ? $(".pre").attr("href").split("/").pop() - : null; - - const nextChapter = $(".next_sesction > a").attr("href") - ? $(".next_sesction > a").attr("href").split("/").pop() - : null; + let chapterName = $("h1.text-center.heading_read").text(); + let chapterText = $(".content-story").html(); novelUrl += "/"; chapterUrl += "/"; const chapter = { - extensionId: 12, + sourceId: 12, novelUrl, chapterUrl, chapterName, chapterText, - nextChapter, - prevChapter, }; - res.json(chapter); + return chapter; }; -const searchScraper = async (req, res) => { - const searchTerm = req.query.s; - +const searchNovels = async (searchTerm) => { const url = `${searchUrl}${searchTerm}`; const result = await fetch(url); @@ -185,7 +164,7 @@ const searchScraper = async (req, res) => { novelUrl = novelId[2] + "/"; const novel = { - extensionId: 12, + sourceId: 12, novelName, novelCover, novelUrl, @@ -194,12 +173,14 @@ const searchScraper = async (req, res) => { novels.push(novel); }); - res.json(novels); + return novels; }; -module.exports = WuxiaWorldSiteScraper = { - novelsScraper, - novelScraper, - chapterScraper, - searchScraper, +const WuxiaWorldSiteScraper = { + popularNovels, + parseNovelAndChapters, + parseChapter, + searchNovels, }; + +export default WuxiaWorldSiteScraper; diff --git a/src/en/wuxiaworldsite/WuxiaWorldSite.js b/src/en/wuxiaworldsite/WuxiaWorldSite.js deleted file mode 100644 index 0cf16e8..0000000 --- a/src/en/wuxiaworldsite/WuxiaWorldSite.js +++ /dev/null @@ -1,14 +0,0 @@ -const express = require("express"); -const WuxiaWorldScraper = require("./WuxiaWorldSiteScraper"); - -const router = express.Router(); - -router.get("/novels/", WuxiaWorldSiteScraper.novelsScraper); - -router.get("/novel/:novelUrl", WuxiaWorldSiteScraper.novelScraper); - -router.get("/novel/:novelUrl/:chapterUrl", WuxiaWorldSiteScraper.chapterScraper); - -router.get("/search/", WuxiaWorldSiteScraper.searchScraper); - -module.exports = router; diff --git a/src/en/wuxiaworldsite/icon.png b/src/en/wuxiaworldsite/icon.png deleted file mode 100644 index f94b242..0000000 Binary files a/src/en/wuxiaworldsite/icon.png and /dev/null differ diff --git a/src/es/einherjarproject/EinherjarProjectScraper.js b/src/es/einherjarproject.js similarity index 74% rename from src/es/einherjarproject/EinherjarProjectScraper.js rename to src/es/einherjarproject.js index 73dff33..b363fdb 100644 --- a/src/es/einherjarproject/EinherjarProjectScraper.js +++ b/src/es/einherjarproject.js @@ -1,11 +1,9 @@ -const cheerio = require("cheerio"); -const fetch = require("node-fetch"); -const { htmlToText } = require("html-to-text"); -const { parseHtml } = require("../../helper"); +import cheerio from "react-native-cheerio"; const baseUrl = "https://einherjarproject.net/"; -const novelsScraper = async (req, res) => { +const popularNovels = async (page) => { + let totalPages = 1; let url = baseUrl + "proyectos-activos/"; const result = await fetch(url); @@ -26,7 +24,7 @@ const novelsScraper = async (req, res) => { novelUrl = novelUrl.replace(baseUrl, ""); const novel = { - extensionId: 25, + sourceId: 25, novelName, novelCover, novelUrl, @@ -35,11 +33,10 @@ const novelsScraper = async (req, res) => { novels.push(novel); }); - res.json(novels); + return { totalPages, novels }; }; -const novelScraper = async (req, res) => { - const novelUrl = req.params.novelUrl; +const parseNovelAndChapters = async (novelUrl) => { const url = baseUrl + novelUrl; const result = await fetch(url); @@ -49,13 +46,13 @@ const novelScraper = async (req, res) => { let novel = {}; - novel.extensionId = 25; + novel.sourceId = 25; novel.sourceName = "Einherjar Project"; - novel.sourceUrl = url; + novel.url = url; - novel.novelUrl = novelUrl + "/"; + novel.novelUrl = novelUrl; novel.novelName = $("h1").text(); @@ -65,33 +62,33 @@ const novelScraper = async (req, res) => { .find("li") .each(function (result) { if ($(this).text().includes("Autor:")) { - novel["Author(s)"] = $(this) + novel.author = $(this) .text() .replace("Autor:", "") .slice(0, -1); } if ($(this).text().includes("Ilustrador: ")) { - novel["Artist(s)"] = $(this) + novel.artist = $(this) .text() .replace("Ilustrador: ", "") .slice(0, -1); } if ($(this).text().includes("Estado: ")) { - novel.Status = $(this) + novel.status = $(this) .text() .replace("Estado: ", "") .slice(0, -1); } }); - novel["Genre(s)"] = $(".post-content > h6") + novel.genre = $(".post-content > h6") .text() .replace(/GÉNEROS: /, "") .replace(/,\s/g, ","); let novelSummary = $(".post-content > .has-text-align-center").html(); - novel.novelSummary = htmlToText(novelSummary); + novel.summary = novelSummary.trim(); let novelChapters = []; @@ -114,17 +111,14 @@ const novelScraper = async (req, res) => { } }); - novel.novelChapters = novelChapters; + novel.chapters = novelChapters; - res.json(novel); + return novel; }; -const chapterScraper = async (req, res) => { - let novelUrl = req.params.novelUrl; - let chapterUrl = req.params.chapterUrl; - +const parseChapter = async (novelUrl, chapterUrl) => { const url = baseUrl + chapterUrl; - console.log(url); + const result = await fetch(url); const body = await result.text(); @@ -133,30 +127,21 @@ const chapterScraper = async (req, res) => { let chapterName = $("h1.post-title").text(); let chapterText = $(".post-content").html(); - chapterText = parseHtml(chapterText); - - let nextChapter = null; - let prevChapter = null; - novelUrl = novelUrl + "/"; chapterUrl = chapterUrl + "/"; const chapter = { - extensionId: 25, + sourceId: 25, novelUrl, chapterUrl, chapterName, chapterText, - nextChapter, - prevChapter, }; - res.json(chapter); + return chapter; }; -const searchScraper = async (req, res) => { - const searchTerm = req.query.s; - +const searchNovels = async (searchTerm) => { const url = baseUrl + "?s=" + searchTerm; const result = await fetch(url); @@ -186,7 +171,7 @@ const searchScraper = async (req, res) => { .replace(baseUrl, ""); const novel = { - extensionId: 25, + sourceId: 25, novelName, novelCover, novelUrl, @@ -196,12 +181,14 @@ const searchScraper = async (req, res) => { } }); - res.json(novels); + return novels; }; -module.exports = einharjarProjectScraper = { - novelsScraper, - novelScraper, - chapterScraper, - searchScraper, +const EinharjarProjectScraper = { + popularNovels, + parseNovelAndChapters, + parseChapter, + searchNovels, }; + +export default EinharjarProjectScraper; diff --git a/src/es/einherjarproject/EinherjarProject.js b/src/es/einherjarproject/EinherjarProject.js deleted file mode 100644 index f273888..0000000 --- a/src/es/einherjarproject/EinherjarProject.js +++ /dev/null @@ -1,17 +0,0 @@ -const express = require("express"); -const einherjarProjectScraper = require("./EinherjarProjectScraper"); - -const router = express.Router(); - -router.get("/novels/", einherjarProjectScraper.novelsScraper); - -router.get("/novel/:novelUrl", einherjarProjectScraper.novelScraper); - -router.get( - "/novel/:novelUrl/:chapterUrl", - einherjarProjectScraper.chapterScraper -); - -router.get("/search/", einherjarProjectScraper.searchScraper); - -module.exports = router; diff --git a/src/es/einherjarproject/icon.png b/src/es/einherjarproject/icon.png deleted file mode 100644 index b9ef6dc..0000000 Binary files a/src/es/einherjarproject/icon.png and /dev/null differ diff --git a/src/es/hasutl/HasutlScraper.js b/src/es/hasutl.js similarity index 65% rename from src/es/hasutl/HasutlScraper.js rename to src/es/hasutl.js index 904f549..5889d41 100644 --- a/src/es/hasutl/HasutlScraper.js +++ b/src/es/hasutl.js @@ -1,11 +1,9 @@ -const cheerio = require("cheerio"); -const fetch = require("node-fetch"); -const { htmlToText } = require("html-to-text"); -const { parseHtml } = require("../../helper"); +import cheerio from "react-native-cheerio"; const baseUrl = "https://hasutl.wordpress.com/"; -const novelsScraper = async (req, res) => { +const popularNovels = async (page) => { + let totalPages = 1; let url = baseUrl + "light-novels-activas/"; const result = await fetch(url); @@ -23,7 +21,7 @@ const novelsScraper = async (req, res) => { novelUrl = novelUrl.replace(baseUrl, ""); const novel = { - extensionId: 29, + sourceId: 29, novelName, novelCover, novelUrl, @@ -32,11 +30,10 @@ const novelsScraper = async (req, res) => { novels.push(novel); }); - res.json(novels); + return { totalPages, novels }; }; -const novelScraper = async (req, res) => { - const novelUrl = req.params.novelUrl; +const parseNovelAndChapters = async (novelUrl) => { const url = `${baseUrl}/${novelUrl}/`; const result = await fetch(url); @@ -46,13 +43,13 @@ const novelScraper = async (req, res) => { let novel = {}; - novel.extensionId = 29; + novel.sourceId = 29; novel.sourceName = "Hasu Translations"; - novel.sourceUrl = url; + novel.url = url; - novel.novelUrl = `${novelUrl}/`; + novel.novelUrl = novelUrl; novel.novelName = $(".post-header").text(); @@ -64,22 +61,22 @@ const novelScraper = async (req, res) => { details = details.match(/<\/strong>(.|\n)*?
/g); details = details.map((detail) => detail.replace(/<\/strong>|
/g, "")); - novel["Genre(s)"] = ""; + novel.genre = ""; detailName.map((detail, index) => { if (detail.includes("Autor")) { - novel["Author(s)"] = details[index]; + novel.author = details[index]; } if (detail.includes("Géneros")) { - novel["Genre(s)"] = details[index].replace(/\s/g, ""); + novel.genre = details[index].replace(/\s/g, ""); } if (detail.includes("Artista")) { - novel["Artist(s)"] = details[index]; + novel.artist = details[index]; } }); let novelSummary = $(".post-content").find("p").html(); - novel.novelSummary = htmlToText(novelSummary); + novel.summary = novelSummary; let novelChapters = []; @@ -98,15 +95,12 @@ const novelScraper = async (req, res) => { novelChapters.push(chapter); }); - novel.novelChapters = novelChapters; + novel.chapters = novelChapters; - res.json(novel); + return novel; }; -const chapterScraper = async (req, res) => { - let novelUrl = req.params.novelUrl; - let chapterUrl = req.params.chapterUrl; - +const parseChapter = async (novelUrl, chapterUrl) => { const url = `${baseUrl}/${novelUrl}/${chapterUrl}`; const result = await fetch(url); @@ -117,45 +111,21 @@ const chapterScraper = async (req, res) => { let chapterName = $(".post-title").text(); let chapterText = $(".post-content").html(); - chapterText = parseHtml(chapterText); - - let nextChapter = null; - - // if ($(".nav-next").length) { - // nextChapter = $(".nav-next") - // .find("a") - // .attr("href") - // .replace(baseUrl + "vipnovel/" + novelUrl + "/", ""); - // } - - let prevChapter = null; - - // if ($(".nav-previous").length) { - // prevChapter = $(".nav-previous") - // .find("a") - // .attr("href") - // .replace(baseUrl + "vipnovel/" + novelUrl + "/", ""); - // } - novelUrl = novelUrl + "/"; chapterUrl = chapterUrl; const chapter = { - extensionId: 29, + sourceId: 29, novelUrl, chapterUrl, chapterName, chapterText, - nextChapter, - prevChapter, }; - res.json(chapter); + return chapter; }; -const searchScraper = async (req, res) => { - const searchTerm = req.query.s; - +const searchNovels = async (searchTerm) => { const url = `${baseUrl}?s=${searchTerm}&post_type=wp-manga`; const result = await fetch(url); @@ -178,7 +148,7 @@ const searchScraper = async (req, res) => { novelUrl = novelUrl.replace(baseUrl, ""); const novel = { - extensionId: 29, + sourceId: 29, novelName, novelCover, novelUrl, @@ -188,12 +158,13 @@ const searchScraper = async (req, res) => { } }); - res.json(novels); + return novels; +}; +const HasuTlScraper = { + popularNovels, + parseNovelAndChapters, + parseChapter, + searchNovels, }; -module.exports = HasutlScraper = { - novelsScraper, - novelScraper, - chapterScraper, - searchScraper, -}; +export default HasuTlScraper; diff --git a/src/es/hasutl/Hasutl.js b/src/es/hasutl/Hasutl.js deleted file mode 100644 index 53112b7..0000000 --- a/src/es/hasutl/Hasutl.js +++ /dev/null @@ -1,14 +0,0 @@ -const express = require("express"); -const HasutlScraper = require("./HasutlScraper"); - -const router = express.Router(); - -router.get("/novels/", HasutlScraper.novelsScraper); - -router.get("/novel/:novelUrl", HasutlScraper.novelScraper); - -router.get("/novel/:novelUrl/:chapterUrl", HasutlScraper.chapterScraper); - -router.get("/search/", HasutlScraper.searchScraper); - -module.exports = router; diff --git a/src/es/hasutl/icon.png b/src/es/hasutl/icon.png deleted file mode 100644 index 9ca3582..0000000 Binary files a/src/es/hasutl/icon.png and /dev/null differ diff --git a/src/es/novelasligera/NovelasLigeraScraper.js b/src/es/novelasligera.js similarity index 60% rename from src/es/novelasligera/NovelasLigeraScraper.js rename to src/es/novelasligera.js index efba1c5..5e9c0fb 100644 --- a/src/es/novelasligera/NovelasLigeraScraper.js +++ b/src/es/novelasligera.js @@ -1,11 +1,10 @@ -const cheerio = require("cheerio"); -const fetch = require("node-fetch"); -const { htmlToText } = require("html-to-text"); -const { parseHtml } = require("../../helper"); +import cheerio from "react-native-cheerio"; +import { htmlToText } from "../helpers/htmlToText"; const baseUrl = "https://novelasligera.com/"; -const novelsScraper = async (req, res) => { +const popularNovels = async (page) => { + let totalPages = 1; let url = baseUrl; const result = await fetch(url); @@ -27,7 +26,7 @@ const novelsScraper = async (req, res) => { novelUrl = novelUrl.replace("novela/", ""); const novel = { - extensionId: 26, + sourceId: 26, novelName, novelCover, novelUrl, @@ -37,13 +36,14 @@ const novelsScraper = async (req, res) => { } }); - res.json(novels); + return { totalPages, novels }; }; -const novelScraper = async (req, res) => { - const novelUrl = req.params.novelUrl; +const parseNovelAndChapters = async (novelUrl) => { const url = baseUrl + "novela/" + novelUrl; + console.log(url); + const result = await fetch(url); const body = await result.text(); @@ -51,13 +51,13 @@ const novelScraper = async (req, res) => { let novel = {}; - novel.extensionId = 26; + novel.sourceId = 26; novel.sourceName = "Novelas Ligera"; - novel.sourceUrl = url; + novel.url = url; - novel.novelUrl = novelUrl + "/"; + novel.novelUrl = novelUrl; novel.novelName = $("h1").text(); @@ -67,25 +67,21 @@ const novelScraper = async (req, res) => { .find("p") .each(function (result) { if ($(this).text().includes("Autor:")) { - novel["Author(s)"] = $(this) - .text() - .replace("Autor:", "") - .trim(); + novel.author = $(this).text().replace("Autor:", "").trim(); } if ($(this).text().includes("Estado:")) { - novel.Status = $(this).text().replace("Estado: ", "").trim(); + novel.status = $(this).text().replace("Estado: ", "").trim(); } if ($(this).text().includes("Género:")) { $(this).find("span").remove(); - novel["Genre(s)"] = $(this).text().replace(/,\s/g, ","); + novel.genre = $(this).text().replace(/,\s/g, ","); } }); - novel["Artist(s)"] = null; + novel.artist = null; - let novelSummary = $(".elementor-text-editor.elementor-clearfix").html(); - novel.novelSummary = htmlToText(novelSummary); + novel.novelSummary = $(".elementor-text-editor.elementor-clearfix").text(); let novelChapters = []; @@ -106,25 +102,13 @@ const novelScraper = async (req, res) => { novelChapters.push(chapter); }); - novel.novelChapters = novelChapters; + novel.chapters = novelChapters; - res.json(novel); + return novel; }; -const chapterScraper = async (req, res) => { - let novelUrl = req.params.novelUrl; - let chapterUrl = req.params.chapterUrl; - const volumeUrl = req.params.volumeUrl; - - let optionalUrl; - - if (volumeUrl) { - optionalUrl = volumeUrl + "/"; - } else { - optionalUrl = novelUrl + "/"; - } - - const url = baseUrl + "novela/" + optionalUrl + chapterUrl; +const parseChapter = async (novelUrl, chapterUrl) => { + const url = baseUrl + "novela/" + chapterUrl; console.log(url); const result = await fetch(url); @@ -134,22 +118,6 @@ const chapterScraper = async (req, res) => { let chapterName = $("h1.entry-title").text(); - let nextChapter = null; - - // if ($(".wp-post-navigation-next").length > 0) { - // nextChapter = - // $(".wp-post-navigation-next").find("a").attr("href") || null; - // nextChapter && nextChapter.replace(baseUrl + "novela/", ""); - // } - - let prevChapter = null; - - // if ($(".wp-post-navigation-pre").length > 0) { - // prevChapter = - // $(".wp-post-navigation-pre").find("a").attr("href") || null; - // prevChapter && prevChapter.replace(baseUrl + "novela/", ""); - // } - $(".osny-nightmode.osny-nightmode--left").remove(); $(".code-block.code-block-1").remove(); $(".adsb30").remove(); @@ -159,25 +127,18 @@ const chapterScraper = async (req, res) => { let chapterText = $(".entry-content").html(); chapterText = htmlToText(chapterText, { preserveNewlines: true }); - novelUrl = novelUrl + "/"; - chapterUrl = chapterUrl + "/"; - const chapter = { - extensionId: 26, + sourceId: 26, novelUrl, chapterUrl, chapterName, chapterText, - nextChapter, - prevChapter, }; - res.json(chapter); + return chapter; }; -const searchScraper = async (req, res) => { - const searchTerm = req.query.s; - +const searchNovels = async (searchTerm) => { const url = baseUrl + "?s=" + searchTerm + "&post_type=wp-manga"; console.log(url); @@ -203,7 +164,7 @@ const searchScraper = async (req, res) => { novelUrl += "/"; const novel = { - extensionId: 26, + sourceId: 26, novelName, novelCover, novelUrl, @@ -214,12 +175,14 @@ const searchScraper = async (req, res) => { novels = [{ ...novels[1] }]; - res.json(novels); + return novels; }; -module.exports = einharjarProjectScraper = { - novelsScraper, - novelScraper, - chapterScraper, - searchScraper, +const NovelasLigeraScraper = { + popularNovels, + parseNovelAndChapters, + parseChapter, + searchNovels, }; + +export default NovelasLigeraScraper; diff --git a/src/es/novelasligera/NovelasLigera.js b/src/es/novelasligera/NovelasLigera.js deleted file mode 100644 index e92631e..0000000 --- a/src/es/novelasligera/NovelasLigera.js +++ /dev/null @@ -1,17 +0,0 @@ -const express = require("express"); -const NovelasLigeraScraper = require("./NovelasLigeraScraper"); - -const router = express.Router(); - -router.get("/novels/", NovelasLigeraScraper.novelsScraper); - -router.get("/novel/:novelUrl", NovelasLigeraScraper.novelScraper); - -router.get( - "/novel/:novelUrl/:volumeUrl?/:chapterUrl", - NovelasLigeraScraper.chapterScraper -); - -router.get("/search/", NovelasLigeraScraper.searchScraper); - -module.exports = router; diff --git a/src/es/novelasligera/icon.png b/src/es/novelasligera/icon.png deleted file mode 100644 index 09e1fdc..0000000 Binary files a/src/es/novelasligera/icon.png and /dev/null differ diff --git a/src/es/novelawuxia/NovelaWuxiaScraper.js b/src/es/novelawuxia.js similarity index 72% rename from src/es/novelawuxia/NovelaWuxiaScraper.js rename to src/es/novelawuxia.js index 5944530..9c12e96 100644 --- a/src/es/novelawuxia/NovelaWuxiaScraper.js +++ b/src/es/novelawuxia.js @@ -1,7 +1,4 @@ -const cheerio = require("cheerio"); -const fetch = require("node-fetch"); -const { htmlToText } = require("html-to-text"); -const { parseHtml } = require("../../helper"); +import cheerio from "react-native-cheerio"; const baseUrl = "http://www.novelawuxia.com/"; @@ -9,10 +6,19 @@ function getNovelName(y) { return y.replace(/-/g, " ").replace(/(?:^|\s)\S/g, (a) => a.toUpperCase()); } -const novelsScraper = async (req, res) => { +const popularNovels = async (page) => { + let totalPages = 1; let url = baseUrl + "p/todas-las-novelas.html"; - const result = await fetch(url); + let headers = new Headers({ + "User-Agent": + "'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36", + }); + + const result = await fetch(url, { + method: "GET", + headers: headers, + }); const body = await result.text(); $ = cheerio.load(body); @@ -34,7 +40,7 @@ const novelsScraper = async (req, res) => { novelUrl = novelUrl.replace(`${baseUrl}p/`, "") + "/"; const novel = { - extensionId: 31, + sourceId: 31, novelName, novelCover, novelUrl, @@ -43,12 +49,11 @@ const novelsScraper = async (req, res) => { novels.push(novel); }); - res.json(novels); + return { totalPages, novels }; }; -const novelScraper = async (req, res) => { - const novelUrl = req.params.novelUrl; - const url = `${baseUrl}p/${novelUrl}`; +const parseNovelAndChapters = async (novelUrl) => { + const url = `${baseUrl}p/${novelUrl.replace("/", "")}`; const result = await fetch(url); const body = await result.text(); @@ -57,20 +62,20 @@ const novelScraper = async (req, res) => { let novel = {}; - novel.extensionId = 31; + novel.sourceId = 31; novel.sourceName = "Novela Wuxia"; - novel.sourceUrl = url; + novel.url = url; - novel.novelUrl = `${novelUrl}/`; + novel.novelUrl = novelUrl; novel.novelName = $("h1.post-title").text().trim(); novel.novelCover = $("div.separator").find("a").attr("href"); - novel["Artist(s)"] = ""; - novel.Status = ""; + novel.artist = ""; + novel.status = ""; $("div > b").each(function (result) { const detailName = $(this).text(); @@ -80,14 +85,14 @@ const novelScraper = async (req, res) => { detail = detail.nodeValue; if (detailName.includes("Autor")) { - novel["Author(s)"] = detail.replace("Autor:", ""); + novel.author = detail.replace("Autor:", ""); } if (detailName.includes("Estatus")) { - novel.Status = detail.replace("Estatus: ", ""); + novel.status = detail.replace("Estatus: ", ""); } if (detailName.includes("Géneros:")) { - novel["Genre(s)"] = detail + novel.genre = detail .replace("Géneros: ", "") .replace(/,\s/g, ","); } @@ -98,13 +103,16 @@ const novelScraper = async (req, res) => { $("div").each(function (result) { const detailName = $(this).text(); - if (detailName.includes("Sinopsis")) { - novel.novelSummary = $(this) - .next() - .text() - .replace("Sinopsis", "") - .trim(); + novel.summary = + $(this).next().text() !== "" + ? $(this).next().text().replace("Sinopsis", "").trim() + : $(this) + .next() + .next() + .text() + .replace("Sinopsis", "") + .trim(); } if (detailName.includes("Lista de Capítulos")) { @@ -137,21 +145,14 @@ const novelScraper = async (req, res) => { } }); - novel.novelChapters = novelChapters; + novel.chapters = novelChapters; - res.json(novel); + return novel; }; -const chapterScraper = async (req, res) => { - let novelUrl = req.params.novelUrl; - let year = req.params.year; - let month = req.params.month; - let chapterUrl = req.params.chapterUrl; - +const parseChapter = async (novelUrl, chapterUrl) => { const url = `${baseUrl}${year}/${month}/${chapterUrl}`; - console.log(url); - const result = await fetch(url); const body = await result.text(); @@ -160,30 +161,21 @@ const chapterScraper = async (req, res) => { let chapterName = $("h1.post-title").text().trim(); let chapterText = $(".post-body.entry-content").html(); - chapterText = parseHtml(chapterText); - - let nextChapter = null; - let prevChapter = null; - novelUrl = novelUrl + "/"; chapterUrl = chapterUrl + "/"; const chapter = { - extensionId: 31, + sourceId: 31, novelUrl, chapterUrl, chapterName, chapterText, - nextChapter, - prevChapter, }; - res.json(chapter); + return chapter; }; -const searchScraper = async (req, res) => { - const searchTerm = req.query.s; - +const searchNovels = async (searchTerm) => { const url = `${baseUrl}search?q=${searchTerm}`; const result = await fetch(url); @@ -210,7 +202,7 @@ const searchScraper = async (req, res) => { if (!exists) { const novelCover = null; const novel = { - extensionId: 31, + sourceId: 31, novelName, novelCover, novelUrl, @@ -220,12 +212,14 @@ const searchScraper = async (req, res) => { } }); - res.json(novels); + return novels; }; -module.exports = vipNovelScraper = { - novelsScraper, - novelScraper, - chapterScraper, - searchScraper, +const NovelaWuxiaScraper = { + popularNovels, + parseNovelAndChapters, + parseChapter, + searchNovels, }; + +export default NovelaWuxiaScraper; diff --git a/src/es/novelawuxia/NovelaWuxia.js b/src/es/novelawuxia/NovelaWuxia.js deleted file mode 100644 index 511b9b1..0000000 --- a/src/es/novelawuxia/NovelaWuxia.js +++ /dev/null @@ -1,17 +0,0 @@ -const express = require("express"); -const NovelaWuxiaScraper = require("./NovelaWuxiaScraper"); - -const router = express.Router(); - -router.get("/novels/", NovelaWuxiaScraper.novelsScraper); - -router.get("/novel/:novelUrl", NovelaWuxiaScraper.novelScraper); - -router.get( - "/novel/:novelUrl/:year/:month/:chapterUrl", - NovelaWuxiaScraper.chapterScraper -); - -router.get("/search/", NovelaWuxiaScraper.searchScraper); - -module.exports = router; diff --git a/src/es/novelawuxia/icon.png b/src/es/novelawuxia/icon.png deleted file mode 100644 index 623c577..0000000 Binary files a/src/es/novelawuxia/icon.png and /dev/null differ diff --git a/src/es/oasistranslations/OasisTranslationsScraper.js b/src/es/oasistranslations.js similarity index 77% rename from src/es/oasistranslations/OasisTranslationsScraper.js rename to src/es/oasistranslations.js index 4dc7514..432925a 100644 --- a/src/es/oasistranslations/OasisTranslationsScraper.js +++ b/src/es/oasistranslations.js @@ -1,13 +1,12 @@ -const cheerio = require("cheerio"); -const fetch = require("node-fetch"); -const { htmlToText } = require("html-to-text"); -const { parseHtml } = require("../../helper"); +import cheerio from "react-native-cheerio"; const baseUrl = "https://oasistranslations.wordpress.com/"; -const novelsScraper = async (req, res) => { +const popularNovels = async (page) => { let url = baseUrl; + let totalPages = 1; + const result = await fetch(url); const body = await result.text(); @@ -27,7 +26,7 @@ const novelsScraper = async (req, res) => { novelUrl = novelUrl[novelUrl.length - 2] + "/"; const novel = { - extensionId: 30, + sourceId: 30, novelName, novelCover, novelUrl, @@ -37,11 +36,10 @@ const novelsScraper = async (req, res) => { } }); - res.json(novels); + return { totalPages, novels }; }; -const novelScraper = async (req, res) => { - const novelUrl = req.params.novelUrl; +const parseNovelAndChapters = async (novelUrl) => { const url = baseUrl + novelUrl + "/"; const result = await fetch(url); @@ -51,13 +49,13 @@ const novelScraper = async (req, res) => { let novel = {}; - novel.extensionId = 30; + novel.sourceId = 30; novel.sourceName = "Oasis Translations"; - novel.sourceUrl = url; + novel.url = url; - novel.novelUrl = novelUrl + "/"; + novel.novelUrl = novelUrl; novel.novelName = $("h1.entry-title") .text() @@ -74,16 +72,16 @@ const novelScraper = async (req, res) => { detail.replace(/|<\/strong>|
|:\s/g, "") ); - novel["Genre(s)"] = ""; + novel.genre = ""; - novel["Author(s)"] = details[2]; - novel["Genre(s)"] = details[4].replace(/\s| /g, ""); - novel["Artist(s)"] = details[3]; + novel.author = details[2]; + novel.genre = details[4].replace(/\s| /g, ""); + novel.artist = details[3]; } }); // let novelSummary = $(this).next().html(); - novel.novelSummary = ""; + novel.summary = ""; let novelChapters = []; @@ -125,15 +123,12 @@ const novelScraper = async (req, res) => { // }); // } - novel.novelChapters = novelChapters; + novel.chapters = novelChapters; - res.json(novel); + return novel; }; -const chapterScraper = async (req, res) => { - let novelUrl = req.params.novelUrl; - let chapterUrl = req.params.chapterUrl; - +const parseChapter = async (novelUrl, chapterUrl) => { const url = `${baseUrl}/${novelUrl}/${chapterUrl}`; const result = await fetch(url); @@ -146,29 +141,21 @@ const chapterScraper = async (req, res) => { $("div#jp-post-flair").remove(); let chapterText = $(".entry-content").html(); - chapterText = parseHtml(chapterText); - - let nextChapter = null; - let prevChapter = null; - novelUrl = novelUrl + "/"; chapterUrl = chapterUrl + "/"; const chapter = { - extensionId: 30, + sourceId: 30, novelUrl, chapterUrl, chapterName, chapterText, - nextChapter, - prevChapter, }; - res.json(chapter); + return chapter; }; -const searchScraper = async (req, res) => { - let searchTerm = req.query.s; +const searchNovels = async (searchTerm) => { searchTerm = searchTerm.toLowerCase(); let url = baseUrl; @@ -191,7 +178,7 @@ const searchScraper = async (req, res) => { novelUrl = novelUrl[novelUrl.length - 2] + "/"; const novel = { - extensionId: 30, + sourceId: 30, novelName, novelCover, novelUrl, @@ -205,12 +192,14 @@ const searchScraper = async (req, res) => { novel.novelName.toLowerCase().includes(searchTerm) ); - res.json(novels); + return novels; }; -module.exports = vipNovelScraper = { - novelsScraper, - novelScraper, - chapterScraper, - searchScraper, +const OasisTranslationsScraper = { + popularNovels, + parseNovelAndChapters, + parseChapter, + searchNovels, }; + +export default OasisTranslationsScraper; diff --git a/src/es/oasistranslations/OasisTranslations.js b/src/es/oasistranslations/OasisTranslations.js deleted file mode 100644 index a4568a4..0000000 --- a/src/es/oasistranslations/OasisTranslations.js +++ /dev/null @@ -1,17 +0,0 @@ -const express = require("express"); -const OasisTranslationsScraper = require("./OasisTranslationsScraper"); - -const router = express.Router(); - -router.get("/novels/", OasisTranslationsScraper.novelsScraper); - -router.get("/novel/:novelUrl", OasisTranslationsScraper.novelScraper); - -router.get( - "/novel/:novelUrl/:chapterUrl", - OasisTranslationsScraper.chapterScraper -); - -router.get("/search/", OasisTranslationsScraper.searchScraper); - -module.exports = router; diff --git a/src/es/oasistranslations/icon.png b/src/es/oasistranslations/icon.png deleted file mode 100644 index b529d0d..0000000 Binary files a/src/es/oasistranslations/icon.png and /dev/null differ diff --git a/src/es/skynovels/SkynovelsScraper.js b/src/es/skynovels.js similarity index 62% rename from src/es/skynovels/SkynovelsScraper.js rename to src/es/skynovels.js index 64f00ac..232f70f 100644 --- a/src/es/skynovels/SkynovelsScraper.js +++ b/src/es/skynovels.js @@ -1,11 +1,7 @@ -const cheerio = require("cheerio"); -const fetch = require("node-fetch"); -const { htmlToText } = require("html-to-text"); -const { parseHtml } = require("../../helper"); - const baseUrl = "https://www.skynovels.net/"; -const novelsScraper = async (req, res) => { +const popularNovels = async (page) => { + let totalPages = 1; const url = "https://api.skynovels.net/api/novels?&q"; const result = await fetch(url); @@ -21,17 +17,17 @@ const novelsScraper = async (req, res) => { "/novels/false"; const novelUrl = res.id + "/" + res.nvl_name + "/"; - const novel = { extensionId: 24, novelName, novelUrl, novelCover }; + const novel = { sourceId: 24, novelName, novelUrl, novelCover }; novels.push(novel); }); - res.json(novels); + return { totalPages, novels }; }; -const novelScraper = async (req, res) => { - const novelId = req.params.novelId; - const novelUrl = req.params.novelUrl; +const parseNovelAndChapters = async (novUrl) => { + let novelId = novUrl.split("/")[0]; + let novelUrl = novUrl[1]; const url = "https://api.skynovels.net/api/novel/" + novelId + "/reading?&q"; @@ -42,13 +38,13 @@ const novelScraper = async (req, res) => { let novel = {}; - novel.extensionId = 24; + novel.sourceId = 24; novel.sourceName = "SkyNovels"; - novel.sourceUrl = baseUrl + "novelas/" + novelId + "/" + novelUrl; + novel.url = baseUrl + "novelas/" + novelId + "/" + novelUrl; - novel.novelUrl = novelId + "/" + novelUrl + "/"; + novel.novelUrl = novUrl; novel.novelName = item.nvl_title; @@ -59,10 +55,10 @@ const novelScraper = async (req, res) => { let genres = []; item.genres.map((genre) => genres.push(genre.genre_name)); - novel["Genre(s)"] = genres.join(","); - novel["Author(s)"] = item.nvl_writer; - novel.novelSummary = item.nvl_content; - novel.Status = item.nvl_status; + novel.genre = genres.join(","); + novel.author = item.nvl_writer; + novel.summary = item.nvl_content; + novel.status = item.nvl_status; let novelChapters = []; @@ -78,21 +74,18 @@ const novelScraper = async (req, res) => { }); }); - novel.novelChapters = novelChapters; + novel.chapters = novelChapters; - res.json(novel); + return novel; }; -const chapterScraper = async (req, res) => { - let novelId = req.params.novelId; - let novelUrl = req.params.novelUrl; - let chapterId = req.params.chapterId; - let chapterUrl = req.params.chapterUrl; - +const parseChapter = async (novUrl, chapUrl) => { + let novelId = novUrl.split("/")[0]; + let novelUrl = novUrl[1]; + let chapterId = chapUrl.split("/")[0]; + let chapterUrl = chapUrl[1]; const url = `https://api.skynovels.net/api/novel-chapter/${chapterId}`; - console.log(url); - const result = await fetch(url); const body = await result.json(); @@ -100,29 +93,23 @@ const chapterScraper = async (req, res) => { let chapterName = item.chp_index_title; - let chapterText = parseHtml(item.chp_content); - - let nextChapter = null; - let prevChapter = null; + let chapterText = item.chp_content; novelUrl = novelId + "/" + novelUrl + "/"; chapterUrl = item.id + "/" + item.chp_name; const chapter = { - extensionId: 24, + sourceId: 24, novelUrl, chapterUrl, chapterName, chapterText, - nextChapter, - prevChapter, }; - res.json(chapter); + return chapter; }; -const searchScraper = async (req, res) => { - let searchTerm = req.query.s; +const searchNovels = async (searchTerm) => { searchTerm = searchTerm.toLowerCase(); const url = "https://api.skynovels.net/api/novels?&q"; @@ -143,17 +130,19 @@ const searchScraper = async (req, res) => { "/novels/false"; const novelUrl = res.id + "/" + res.nvl_name + "/"; - const novel = { extensionId: 24, novelName, novelUrl, novelCover }; + const novel = { sourceId: 24, novelName, novelUrl, novelCover }; novels.push(novel); }); - res.json(novels); + return novels; }; -module.exports = vipNovelScraper = { - novelsScraper, - novelScraper, - chapterScraper, - searchScraper, +const SkyNovelsScraper = { + popularNovels, + parseNovelAndChapters, + parseChapter, + searchNovels, }; + +export default SkyNovelsScraper; diff --git a/src/es/skynovels/Skynovels.js b/src/es/skynovels/Skynovels.js deleted file mode 100644 index 8187fe6..0000000 --- a/src/es/skynovels/Skynovels.js +++ /dev/null @@ -1,17 +0,0 @@ -const express = require("express"); -const skynovelsScraper = require("./SkynovelsScraper"); - -const router = express.Router(); - -router.get("/novels/", skynovelsScraper.novelsScraper); - -router.get("/novel/:novelId/:novelUrl/", skynovelsScraper.novelScraper); - -router.get( - "/novel/:novelId/:novelUrl/:chapterId/:chapterUrl", - skynovelsScraper.chapterScraper -); - -router.get("/search/", skynovelsScraper.searchScraper); - -module.exports = router; diff --git a/src/es/skynovels/icon.png b/src/es/skynovels/icon.png deleted file mode 100644 index e708d43..0000000 Binary files a/src/es/skynovels/icon.png and /dev/null differ diff --git a/src/es/tunovelaligera.js b/src/es/tunovelaligera.js new file mode 100644 index 0000000..282bddd --- /dev/null +++ b/src/es/tunovelaligera.js @@ -0,0 +1,190 @@ +import cheerio from "react-native-cheerio"; +import { defaultCoverUri, Status } from "../helpers/constants"; + +const sourceId = 23; +const sourceName = "TuNovelaLigera"; + +const baseUrl = "https://tunovelaligera.com/"; + +const popularNovels = async (page) => { + let totalPages = 62; + let url = baseUrl + "novelas/page/" + page + "/?m_orderby=rating"; + + const result = await fetch(url); + const body = await result.text(); + + $ = cheerio.load(body); + + let novels = []; + + $(".page-item-detail").each(function () { + const novelName = $(this).find(".h5 > a").text(); + const novelCover = $(this).find("img").attr("src"); + + let novelUrl = $(this).find(".h5 > a").attr("href").split("/")[4]; + novelUrl += "/"; + + const novel = { + sourceId, + novelName, + novelCover, + novelUrl, + }; + + novels.push(novel); + }); + + return { totalPages, novels }; +}; + +const parseNovelAndChapters = async (novelUrl) => { + const url = `${baseUrl}novelas/${novelUrl}`; + + const result = await fetch(url); + const body = await result.text(); + + $ = cheerio.load(body); + + let novel = { sourceId, sourceName, url }; + + novel.novelUrl = novelUrl; + + $(".manga-title-badges").remove(); + + novel.novelName = $(".post-title > h1").text().trim(); + + novelCover = $(".summary_image > a > img"); + + novel.novelCover = + novelCover.attr("data-src") || + novelCover.attr("src") || + defaultCoverUri; + + $(".post-content_item").each(function () { + const detailName = $(this).find(".summary-heading > h5").text().trim(); + const detail = $(this).find(".summary-content").text().trim(); + + switch (detailName) { + case "Generos": + novel.genre = detail.replace(/, /g, ","); + break; + case "Autores": + novel.author = detail; + break; + case "Estado": + novel.status = + detail.includes("OnGoing") || detail.includes("Updating") + ? Status.ONGOING + : Status.COMPLETED; + break; + } + }); + + novel.summary = $("div.summary__content > p").text().trim(); + + let novelChapters = []; + + const novelId = + $(".rating-post-id").attr("value") || + $("#manga-chapters-holder").attr("data-id"); + + let formData = new FormData(); + formData.append("action", "manga_get_chapters"); + formData.append("manga", novelId); + + const data = await fetch( + "https://tunovelaligera.com/wp-admin/admin-ajax.php", + { + method: "POST", + body: formData, + } + ); + const text = await data.text(); + + $ = cheerio.load(text); + + $(".wp-manga-chapter").each(function (result) { + chapterName = $(this) + .find("a") + .text() + .replace(/[\t\n]/g, "") + .trim(); + + releaseDate = $(this).find("span").text().trim(); + + chapterUrl = $(this).find("a").attr("href").split("/"); + + chapterUrl[6] + ? (chapterUrl = chapterUrl[5] + "/" + chapterUrl[6]) + : (chapterUrl = chapterUrl[5]); + + novelChapters.push({ chapterName, releaseDate, chapterUrl }); + }); + + novel.chapters = novelChapters.reverse(); + + return novel; +}; + +const parseChapter = async (novelUrl, chapterUrl) => { + const url = `${baseUrl}novelas/${novelUrl}/${chapterUrl}`; + + const result = await fetch(url); + const body = await result.text(); + + $ = cheerio.load(body); + + let chapterName = $("h1#chapter-heading").text(); + + let chapterText = $(".text-left").html(); + novelUrl = novelUrl + "/"; + + const chapter = { + sourceId, + novelUrl, + chapterUrl, + chapterName, + chapterText, + }; + + return chapter; +}; + +const searchNovels = async (searchTerm) => { + const url = `${baseUrl}?s=${searchTerm}&post_type=wp-manga`; + + const result = await fetch(url); + const body = await result.text(); + + $ = cheerio.load(body); + + let novels = []; + + $(".c-tabs-item__content").each(function (result) { + const novelName = $(this).find(".h4 > a").text(); + const novelCover = $(this).find("img").attr("src"); + + let novelUrl = $(this).find(".h4 > a").attr("href"); + novelUrl = novelUrl.replace(`${baseUrl}novelas/`, ""); + + const novel = { + sourceId, + novelName, + novelCover, + novelUrl, + }; + + novels.push(novel); + }); + + return novels; +}; + +const TuNovelaLigeraScraper = { + popularNovels, + parseNovelAndChapters, + parseChapter, + searchNovels, +}; + +export default TuNovelaLigeraScraper; diff --git a/src/es/tunovelaligera/Tunovelaligera.js b/src/es/tunovelaligera/Tunovelaligera.js deleted file mode 100644 index 24d41db..0000000 --- a/src/es/tunovelaligera/Tunovelaligera.js +++ /dev/null @@ -1,17 +0,0 @@ -const express = require("express"); -const tunovelaligeraScraper = require("./TunovelaligeraScraper"); - -const router = express.Router(); - -router.get("/novels/", tunovelaligeraScraper.novelsScraper); - -router.get("/novel/:novelUrl", tunovelaligeraScraper.novelScraper); - -router.get( - "/novel/:novelUrl/:volumeUrl?/:chapterUrl", - tunovelaligeraScraper.chapterScraper -); - -router.get("/search/", tunovelaligeraScraper.searchScraper); - -module.exports = router; diff --git a/src/es/tunovelaligera/TunovelaligeraScraper.js b/src/es/tunovelaligera/TunovelaligeraScraper.js deleted file mode 100644 index 0577bb3..0000000 --- a/src/es/tunovelaligera/TunovelaligeraScraper.js +++ /dev/null @@ -1,240 +0,0 @@ -const cheerio = require("cheerio"); -const fetch = require("node-fetch"); -const { htmlToText } = require("html-to-text"); -const { parseHtml } = require("../../helper"); -const FormData = require("form-data"); - -const baseUrl = "https://tunovelaligera.com/"; - -const novelsScraper = async (req, res) => { - let url = baseUrl + "mejores-novelas-originales/"; - - const result = await fetch(url); - const body = await result.text(); - - $ = cheerio.load(body); - - let novels = []; - - $(".page-item-detail").each(function (result) { - const novelName = $(this).find(".h5 > a").text(); - const novelCover = $(this).find("img").attr("src"); - - let novelUrl = $(this).find(".h5 > a").attr("href").split("/")[4]; - novelUrl += "/"; - - const novel = { - extensionId: 23, - novelName, - novelCover, - novelUrl, - }; - - novels.push(novel); - }); - - res.json(novels); -}; - -const novelScraper = async (req, res) => { - const novelUrl = req.params.novelUrl; - const url = `${baseUrl}novelas/${novelUrl}/`; - - const result = await fetch(url); - const body = await result.text(); - - $ = cheerio.load(body); - - let novel = {}; - - novel.extensionId = 23; - - novel.sourceName = "Tunovelaligera"; - - novel.sourceUrl = url; - - novel.novelUrl = novelUrl + "/"; - - novel.novelName = $("h1.titulo-novela") - .text() - .replace(/[\t\n]/g, "") - .trim(); - - novel.novelCover = $(".summary_image > a > img").attr("src"); - - $(".post-content_item").each(function (result) { - detailName = $(this) - .find(".summary-heading > h5") - .text() - .replace(/[\t\n]/g, "") - .trim(); - detail = $(this) - .find(".summary-content") - .text() - .replace(/[\t\n]/g, "") - .trim(); - - novel[detailName] = detail; - }); - - novel["Genre(s)"] = novel.Generos.replace(/, /g, ","); - novel["Author(s)"] = novel.Autores; - novel.Status = novel.Estado; - novel.Status = null; - - novel["Artist(s)"] = novel["Artista(s)"]; - - delete novel.Genre; - delete novel["Artista(s)"]; - delete novel.Autores; - delete novel.Estado; - - let novelSummary = ""; - - $(".summary__content") - .find("p") - .each(function () { - novelSummary += $(this).html(); - }); - - novel.novelSummary = htmlToText(novelSummary); - - let novelChapters = []; - - const novelId = $(".wp-manga-action-button").attr("data-post"); - - let formData = new FormData(); - formData.append("action", "manga_get_chapters"); - formData.append("manga", novelId); - - const data = await fetch( - "https://tunovelaligera.com/wp-admin/admin-ajax.php", - { - method: "POST", - body: formData, - } - ); - const text = await data.text(); - - $ = cheerio.load(text); - - $(".wp-manga-chapter").each(function (result) { - chapterName = $(this) - .find("a") - .text() - .replace(/[\t\n]/g, "") - .trim(); - - releaseDate = $(this).find("span").text().trim(); - - chapterUrl = $(this).find("a").attr("href").split("/"); - - chapterUrl[6] - ? (chapterUrl = chapterUrl[5] + "/" + chapterUrl[6]) - : (chapterUrl = chapterUrl[5]); - - novelChapters.push({ chapterName, releaseDate, chapterUrl }); - }); - - novel.novelChapters = novelChapters.reverse(); - - res.json(novel); -}; - -const chapterScraper = async (req, res) => { - let novelUrl = req.params.novelUrl; - let chapterUrl = req.params.chapterUrl; - let volumeUrl = req.params.volumeUrl; - - let optionalUrl; - - if (volumeUrl) { - optionalUrl = volumeUrl + "/"; - } else { - optionalUrl = ""; - } - - const url = `${baseUrl}novelas/${novelUrl}/${optionalUrl}${chapterUrl}/`; - - const result = await fetch(url); - const body = await result.text(); - - $ = cheerio.load(body); - - let chapterName = $("h1#chapter-heading").text(); - - let chapterText = $(".text-left").html(); - chapterText = parseHtml(chapterText); - - let nextChapter = null; - - if ($(".nav-next").length) { - nextChapter = $(".nav-next") - .find("a") - .attr("href") - .replace(baseUrl + "novelas/" + novelUrl + "/", ""); - } - - let prevChapter = null; - - if ($(".nav-previous").length) { - prevChapter = $(".nav-previous") - .find("a") - .attr("href") - .replace(baseUrl + "novelas/" + novelUrl + "/", ""); - } - - novelUrl = novelUrl + "/"; - chapterUrl = optionalUrl + chapterUrl; - - const chapter = { - extensionId: 23, - novelUrl, - chapterUrl, - chapterName, - chapterText, - nextChapter, - prevChapter, - }; - - res.json(chapter); -}; - -const searchScraper = async (req, res) => { - const searchTerm = req.query.s; - - const url = `${baseUrl}?s=${searchTerm}&post_type=wp-manga`; - - const result = await fetch(url); - const body = await result.text(); - - $ = cheerio.load(body); - - let novels = []; - - $(".c-tabs-item__content").each(function (result) { - const novelName = $(this).find(".h4 > a").text(); - const novelCover = $(this).find("img").attr("src"); - - let novelUrl = $(this).find(".h4 > a").attr("href"); - novelUrl = novelUrl.replace(`${baseUrl}novelas/`, ""); - - const novel = { - extensionId: 23, - novelName, - novelCover, - novelUrl, - }; - - novels.push(novel); - }); - - res.json(novels); -}; - -module.exports = tunovelaligeraScraper = { - novelsScraper, - novelScraper, - chapterScraper, - searchScraper, -}; diff --git a/src/es/tunovelaligera/icon.png b/src/es/tunovelaligera/icon.png deleted file mode 100644 index e42f5e0..0000000 Binary files a/src/es/tunovelaligera/icon.png and /dev/null differ diff --git a/src/es/yuukitls/YuukiTlsScraper.js b/src/es/yuukitls.js similarity index 76% rename from src/es/yuukitls/YuukiTlsScraper.js rename to src/es/yuukitls.js index dd9e179..1db606b 100644 --- a/src/es/yuukitls/YuukiTlsScraper.js +++ b/src/es/yuukitls.js @@ -1,11 +1,9 @@ -const cheerio = require("cheerio"); -const fetch = require("node-fetch"); -const { htmlToText } = require("html-to-text"); -const { parseHtml } = require("../../helper"); +import cheerio from "react-native-cheerio"; const baseUrl = "https://yuukitls.com/"; -const novelsScraper = async (req, res) => { +const popularNovels = async (page) => { + let totalPages = 1; let url = baseUrl; const result = await fetch(url); @@ -26,7 +24,7 @@ const novelsScraper = async (req, res) => { novelUrl = novelUrl[novelUrl.length - 2] + "/"; const novel = { - extensionId: 28, + sourceId: 28, novelName, novelCover, novelUrl, @@ -35,11 +33,10 @@ const novelsScraper = async (req, res) => { novels.push(novel); }); - res.json(novels); + return { totalPages, novels }; }; -const novelScraper = async (req, res) => { - const novelUrl = req.params.novelUrl; +const parseNovelAndChapters = async (novelUrl) => { const url = `${baseUrl}${novelUrl}/`; const result = await fetch(url); @@ -49,13 +46,13 @@ const novelScraper = async (req, res) => { let novel = {}; - novel.extensionId = 28; + novel.sourceId = 28; novel.sourceName = "YuukiTls"; - novel.sourceUrl = url; + novel.url = url; - novel.novelUrl = novelUrl + "/"; + novel.novelUrl = novelUrl; novel.novelName = $("h1.entry-title") .text() @@ -68,26 +65,23 @@ const novelScraper = async (req, res) => { .find("div") .each(function (result) { if ($(this).text().includes("Escritor:")) { - novel["Author(s)"] = $(this) - .text() - .replace("Escritor: ", "") - .trim(); + novel.author = $(this).text().replace("Escritor: ", "").trim(); } if ($(this).text().includes("Ilustrador:")) { - novel["Artist(s)"] = $(this) + novel.artist = $(this) .text() .replace("Ilustrador: ", "") .trim(); } if ($(this).text().includes("Género:")) { - novel["Genre(s)"] = $(this) + novel.genre = $(this) .text() .replace(/Género: |\s/g, ""); } if ($(this).text().includes("Sinopsis:")) { - novel.novelSummary = $(this).next().text(); + novel.summary = $(this).next().text(); } }); @@ -131,15 +125,12 @@ const novelScraper = async (req, res) => { }); } - novel.novelChapters = novelChapters; + novel.chapters = novelChapters; - res.json(novel); + return novel; }; -const chapterScraper = async (req, res) => { - let novelUrl = req.params.novelUrl; - let chapterUrl = req.params.chapterUrl; - +const parseChapter = async (novelUrl, chapterUrl) => { const url = `${baseUrl}/${novelUrl}/${chapterUrl}`; const result = await fetch(url); @@ -150,29 +141,21 @@ const chapterScraper = async (req, res) => { let chapterName = $("h3").text(); let chapterText = $(".entry-content").html(); - chapterText = parseHtml(chapterText); - - let nextChapter = null; - let prevChapter = null; - novelUrl = novelUrl + "/"; chapterUrl = chapterUrl + "/"; const chapter = { - extensionId: 28, + sourceId: 28, novelUrl, chapterUrl, chapterName, chapterText, - nextChapter, - prevChapter, }; - res.json(chapter); + return chapter; }; -const searchScraper = async (req, res) => { - let searchTerm = req.query.s; +const searchNovels = async (searchTerm) => { searchTerm = searchTerm.toLowerCase(); let url = baseUrl; @@ -195,7 +178,7 @@ const searchScraper = async (req, res) => { novelUrl = novelUrl[novelUrl.length - 2] + "/"; const novel = { - extensionId: 28, + sourceId: 28, novelName, novelCover, novelUrl, @@ -208,12 +191,14 @@ const searchScraper = async (req, res) => { novel.novelName.toLowerCase().includes(searchTerm) ); - res.json(novels); + return novels; }; -module.exports = vipNovelScraper = { - novelsScraper, - novelScraper, - chapterScraper, - searchScraper, +const YuukiTlsScraper = { + popularNovels, + parseNovelAndChapters, + parseChapter, + searchNovels, }; + +export default YuukiTlsScraper; diff --git a/src/es/yuukitls/YuukiTls.js b/src/es/yuukitls/YuukiTls.js deleted file mode 100644 index c70745b..0000000 --- a/src/es/yuukitls/YuukiTls.js +++ /dev/null @@ -1,14 +0,0 @@ -const express = require("express"); -const YuukiTlsScraper = require("./YuukiTlsScraper"); - -const router = express.Router(); - -router.get("/novels/", YuukiTlsScraper.novelsScraper); - -router.get("/novel/:novelUrl", YuukiTlsScraper.novelScraper); - -router.get("/novel/:novelUrl/:chapterUrl", YuukiTlsScraper.chapterScraper); - -router.get("/search/", YuukiTlsScraper.searchScraper); - -module.exports = router; diff --git a/src/es/yuukitls/icon.png b/src/es/yuukitls/icon.png deleted file mode 100644 index 301be42..0000000 Binary files a/src/es/yuukitls/icon.png and /dev/null differ diff --git a/src/fr/noveldeglace/NovelDeGlaceScraper.js b/src/fr/noveldeglace.js similarity index 63% rename from src/fr/noveldeglace/NovelDeGlaceScraper.js rename to src/fr/noveldeglace.js index a025eb1..a733fb2 100644 --- a/src/fr/noveldeglace/NovelDeGlaceScraper.js +++ b/src/fr/noveldeglace.js @@ -1,14 +1,14 @@ -const cheerio = require("cheerio"); -const fetch = require("node-fetch"); -const { htmlToText } = require("html-to-text"); -const { parseHtml } = require("../../helper"); +import cheerio from "react-native-cheerio"; + +const sourceId = 77; + +const sourceName = "NovelDeGlace"; const baseUrl = "https://noveldeglace.com/"; -const sourceName = "Novel De Glace"; -const sourceId = 76; +const popularNovels = async (page) => { + const totalPages = 1; -const novelsScraper = async (req, res) => { let url = baseUrl + "roman"; const result = await fetch(url); @@ -33,11 +33,10 @@ const novelsScraper = async (req, res) => { novels.push(novel); }); - res.json(novels); + return { novels, totalPages }; }; -const novelScraper = async (req, res) => { - const novelUrl = req.params.novelUrl; +const parseNovelAndChapters = async (novelUrl) => { const url = baseUrl + "roman/" + novelUrl; const result = await fetch(url); @@ -51,12 +50,15 @@ const novelScraper = async (req, res) => { "div.entry-content > div > strong" )[0].nextSibling.nodeValue.trim(); - novel.novelCover = $("#thumbnail > img").attr("src"); + novel.novelCover = $(".su-row > div > div > img").attr("src"); - novel.novelSummary = $("div[data-title=Synopsis]").text(); - novel.author = $( + novel.summary = $("div[data-title=Synopsis]").text(); + + author = $( "div.romans > div.project-large > div.su-row > div.su-column.su-column-size-3-4 > div > div:nth-child(3) > strong" - )[0].nextSibling.nodeValue.trim(); + )[0]; + + novel.author = author ? author.nextSibling.nodeValue.trim() : null; novel.genre = $(".genre") .text() @@ -68,7 +70,7 @@ const novelScraper = async (req, res) => { $(".chpt").each(function () { const chapterName = $(this).find("a").text().trim(); const releaseDate = null; - const chapterUrl = $(this).find("a").attr("href").split("/")[4]; + const chapterUrl = $(this).find("a").attr("href"); const novel = { sourceId, @@ -80,16 +82,13 @@ const novelScraper = async (req, res) => { novelChapters.push(novel); }); - novel.novelChapters = novelChapters; + novel.chapters = novelChapters; - res.json(novel); + return novel; }; -const chapterScraper = async (req, res) => { - let novelUrl = req.params.novelUrl; - let chapterUrl = req.params.chapterUrl; - - const url = `${baseUrl}chapitre/${chapterUrl}`; +const parseChapter = async (novelUrl, chapterUrl) => { + const url = chapterUrl; const result = await fetch(url); const body = await result.text(); @@ -97,16 +96,7 @@ const chapterScraper = async (req, res) => { $ = cheerio.load(body); let chapterName = $("h2.western").text(); - let chapterText = $(".chapter-content").html(); - chapterText = parseHtml(chapterText); - - let nextChapter = null; - - let prevChapter = null; - - novelUrl = novelUrl + "/"; - chapterUrl = chapterUrl + "/"; const chapter = { sourceId, @@ -114,16 +104,12 @@ const chapterScraper = async (req, res) => { chapterUrl, chapterName, chapterText, - nextChapter, - prevChapter, }; - res.json(chapter); + return chapter; }; -const searchScraper = async (req, res) => { - const searchTerm = req.query.s; - +const searchNovels = async (searchTerm) => { let url = baseUrl + "roman"; const result = await fetch(url); @@ -149,15 +135,17 @@ const searchScraper = async (req, res) => { }); novels = novels.filter((novel) => - novel.novelName.toLowerCase().includes(searchTerm) + novel.novelName.toLowerCase().includes(searchTerm.toLowerCase()) ); - res.json(novels); + return novels; }; -module.exports = NovelDeGlaceScraper = { - novelsScraper, - novelScraper, - chapterScraper, - searchScraper, +const NovelDeGlaceScraper = { + popularNovels, + parseNovelAndChapters, + parseChapter, + searchNovels, }; + +export default NovelDeGlaceScraper; diff --git a/src/fr/noveldeglace/NovelDeGlace.js b/src/fr/noveldeglace/NovelDeGlace.js deleted file mode 100644 index de5a2f3..0000000 --- a/src/fr/noveldeglace/NovelDeGlace.js +++ /dev/null @@ -1,14 +0,0 @@ -const express = require("express"); -const NovelDeGlaceScraper = require("./NovelDeGlaceScraper"); - -const router = express.Router(); - -router.get("/novels/", NovelDeGlaceScraper.novelsScraper); - -router.get("/novel/:novelUrl", NovelDeGlaceScraper.novelScraper); - -router.get("/novel/:novelUrl/:chapterUrl", NovelDeGlaceScraper.chapterScraper); - -router.get("/search/", NovelDeGlaceScraper.searchScraper); - -module.exports = router; diff --git a/src/fr/noveldeglace/icon.png b/src/fr/noveldeglace/icon.png deleted file mode 100644 index e69241a..0000000 Binary files a/src/fr/noveldeglace/icon.png and /dev/null differ diff --git a/src/helper.js b/src/helper.js deleted file mode 100644 index ae4e2e7..0000000 --- a/src/helper.js +++ /dev/null @@ -1,35 +0,0 @@ -const chromium = require("chrome-aws-lambda"); -const { htmlToText } = require("html-to-text"); -const puppeteer = require("puppeteer"); -const UserAgent = require("user-agents"); - -const userAgent = new UserAgent(); - -const scraper = async (url) => { - const browser = await puppeteer.launch({ - args: [ - ...chromium.args, - "--hide-scrollbars", - "--disable-web-security", - "--user-agent=" + userAgent + "", - ], - defaultViewport: chromium.defaultViewport, - executablePath: await chromium.executablePath, - headless: true, - ignoreHTTPSErrors: true, - }); - - const page = await browser.newPage(); - await page.goto(url); - const body = await page.content(); - - return body; -}; - -const parseHtml = (html) => { - let text = htmlToText(html); - text = text.replace(/\n{2,}/g, "\n\n"); - return text; -}; - -module.exports = { scraper, parseHtml }; diff --git a/src/helpers/cleanHtml.js b/src/helpers/cleanHtml.js new file mode 100644 index 0000000..84735e0 --- /dev/null +++ b/src/helpers/cleanHtml.js @@ -0,0 +1,36 @@ +export const cleanHtml = (html) => { + let text = html; + + if (!text) { + return "Chapter is empty.\n\nReport if it's available in webview."; + } + + text = text + /** + * Remove inline styles + */ + .replace( + /style="((?:font-family|line-height|font-size|text-align).*?)"/gi, + "" + ) + .replace(/class=".*?"/gi, "") + .replace(/id=".*?"/gi, "") + + /** + * Remove comments + */ + .replace(//gim, "") + .replace(/<\s*script[^>]*>[\s\S]*?<\/script>/gim, "") + .replace(/<\s*noscript[^>]*>[\s\S]*?<\/noscript>/gim, "") + .replace(/<\s*form[^>]*>[\s\S]*?<\/form>/gim, "") + /** + * Remove relative links + */ + .replace( + /<\s*a[^>]*href=['"](\/.*?)['"][^>]*>([\s\S]*?)<\/\s*a\s*>/gi, + "$2" + ) + .trim(); + + return text; +}; diff --git a/src/helpers/constants.js b/src/helpers/constants.js new file mode 100644 index 0000000..7397d40 --- /dev/null +++ b/src/helpers/constants.js @@ -0,0 +1,8 @@ +export const defaultCoverUri = + "https://github.com/LNReader/lnreader-sources/blob/main/src/coverNotAvailable.jpg?raw=true"; + +export const Status = { + ONGOING: "Ongoing", + COMPLETED: "Completed", + UNKNOWN: "Unknown", +}; diff --git a/src/helpers/htmlToText.js b/src/helpers/htmlToText.js new file mode 100644 index 0000000..ef1b56b --- /dev/null +++ b/src/helpers/htmlToText.js @@ -0,0 +1,263 @@ +const htmlToText = (html, options = {}) => { + const { removeLineBreaks = true } = options; + + if (!html) { + return "Chapter is empty.\n\nReport if it's available in webview."; + } + + html = html.trim(); + + text = removeLineBreaks && html.replace(/(?:\n|\r\n|\r)/gi, ""); + + text = html + /** + *
-> \n + */ + + .replace(/<\s*br[^>]*>/gi, "\n") + + /** + *
-> [hr]() + */ + + .replace(/<\s*hr[^>]*>/gi, "") + + /** + *
-> \n\n
+ */
+
+ .replace(/<\s*p[^>]*>/gi, "\n\n")
+
+ /**
+ * Remove script and noscript tags
+ */
+ .replace(/<\s*script[^>]*>[\s\S]*?<\/script>/gim, "")
+ .replace(/<\s*noscript[^>]*>[\s\S]*?<\/noscript>/gim, "")
+
+ /**
+ * Remove style tags
+ */
+
+ .replace(/<\s*style[^>]*>[\s\S]*?<\/style>/gim, "")
+
+ /**
+ * Remove comments
+ */
+
+ .replace(//gim, "")
+
+ /**
+ * Text -> [Text](link)
+ */
+
+ .replace(
+ /<\s*a[^>]*href=['"](http.*?)['"][^>]*>([\s\S]*?)<\/\s*a\s*>/gi,
+ "$2"
+ )
+ // .replace(
+ // /<\s*a[^>]*href=['"](http.*?)['"][^>]*>([\s\S]*?)<\/\s*a\s*>/gi,
+ // "[$2]($1)"
+ // )
+ /**
+ * -> [img](src)
+ */
+
+ // .replace(/<\s*img[^>]*src=['"](.*?)['"][^>]*>/gi, "[img]($1)")
+
+ /**
+ * Remove remaining tags
+ */
+
+ .replace(/(<([^>]+)>)/gi, "")
+
+ /**
+ * Remove tabs.
+ */
+
+ .replace(/\t/g, "")
+
+ /**
+ * Remove newlines at the beginning of the text.
+ */
+
+ .replace(/^\n+/m, "")
+
+ /**
+ * Replace multiple spaces with a single space.
+ */
+
+ .replace(/ {2,}/g, " ")
+
+ /**
+ * Make sure there are never more than two consecutive linebreaks.
+ */
+
+ .replace(/\n{2,}/g, "\n\n")
+
+ /**
+ * Decode HTML entities.
+ */
+
+ .replace(/&([^;]+);/g, decodeHtmlEntity)
+
+ .trim();
+
+ return text;
+};
+
+const decodeHtmlEntity = (m, n) => {
+ let code;
+
+ if (n.substr(0, 1) == "#") {
+ if (n.substr(1, 1) == "x") {
+ code = parseInt(n.substr(2), 16);
+ } else {
+ code = parseInt(n.substr(1), 10);
+ }
+ } else {
+ code = ENTITIES_MAP[n];
+ }
+
+ return code === undefined || code === NaN
+ ? "&" + n + ";"
+ : String.fromCharCode(code);
+};
+
+let ENTITIES_MAP = {
+ nbsp: 160,
+ iexcl: 161,
+ cent: 162,
+ pound: 163,
+ curren: 164,
+ yen: 165,
+ brvbar: 166,
+ sect: 167,
+ uml: 168,
+ copy: 169,
+ ordf: 170,
+ laquo: 171,
+ not: 172,
+ shy: 173,
+ reg: 174,
+ macr: 175,
+ deg: 176,
+ plusmn: 177,
+ sup2: 178,
+ sup3: 179,
+ acute: 180,
+ micro: 181,
+ para: 182,
+ middot: 183,
+ cedil: 184,
+ sup1: 185,
+ ordm: 186,
+ raquo: 187,
+ frac14: 188,
+ frac12: 189,
+ frac34: 190,
+ iquest: 191,
+ Agrave: 192,
+ Aacute: 193,
+ Acirc: 194,
+ Atilde: 195,
+ Auml: 196,
+ Aring: 197,
+ AElig: 198,
+ Ccedil: 199,
+ Egrave: 200,
+ Eacute: 201,
+ Ecirc: 202,
+ Euml: 203,
+ Igrave: 204,
+ Iacute: 205,
+ Icirc: 206,
+ Iuml: 207,
+ ETH: 208,
+ Ntilde: 209,
+ Ograve: 210,
+ Oacute: 211,
+ Ocirc: 212,
+ Otilde: 213,
+ Ouml: 214,
+ times: 215,
+ Oslash: 216,
+ Ugrave: 217,
+ Uacute: 218,
+ Ucirc: 219,
+ Uuml: 220,
+ Yacute: 221,
+ THORN: 222,
+ szlig: 223,
+ agrave: 224,
+ aacute: 225,
+ acirc: 226,
+ atilde: 227,
+ auml: 228,
+ aring: 229,
+ aelig: 230,
+ ccedil: 231,
+ egrave: 232,
+ eacute: 233,
+ ecirc: 234,
+ euml: 235,
+ igrave: 236,
+ iacute: 237,
+ icirc: 238,
+ iuml: 239,
+ eth: 240,
+ ntilde: 241,
+ ograve: 242,
+ oacute: 243,
+ ocirc: 244,
+ otilde: 245,
+ ouml: 246,
+ divide: 247,
+ oslash: 248,
+ ugrave: 249,
+ uacute: 250,
+ ucirc: 251,
+ uuml: 252,
+ yacute: 253,
+ thorn: 254,
+ yuml: 255,
+ quot: 34,
+ amp: 38,
+ lt: 60,
+ gt: 62,
+ OElig: 338,
+ oelig: 339,
+ Scaron: 352,
+ scaron: 353,
+ Yuml: 376,
+ circ: 710,
+ tilde: 732,
+ ensp: 8194,
+ emsp: 8195,
+ thinsp: 8201,
+ zwnj: 8204,
+ zwj: 8205,
+ lrm: 8206,
+ rlm: 8207,
+ ndash: 8211,
+ mdash: 8212,
+ lsquo: 8216,
+ rsquo: 8217,
+ sbquo: 8218,
+ ldquo: 8220,
+ rdquo: 8221,
+ bdquo: 8222,
+ dagger: 8224,
+ Dagger: 8225,
+ permil: 8240,
+ lsaquo: 8249,
+ rsaquo: 8250,
+ euro: 8364,
+ apos: 39,
+};
+
+export { htmlToText };
diff --git a/src/helpers/parseDate.js b/src/helpers/parseDate.js
new file mode 100644
index 0000000..b8bf19a
--- /dev/null
+++ b/src/helpers/parseDate.js
@@ -0,0 +1,28 @@
+import moment from "moment";
+
+export const parseMadaraDate = (date) => {
+ let releaseDate = date;
+
+ if (date.includes("ago")) {
+ let timeAgo = date;
+ releaseDate = new Date();
+
+ timeAgo = timeAgo.match(/\d+/)[0];
+
+ if (timeAgo.includes("hours ago") || timeAgo.includes("hour ago")) {
+ releaseDate.setHours(releaseDate.getHours() - timeAgo);
+ }
+
+ if (timeAgo.includes("days ago") || timeAgo.includes("day ago")) {
+ releaseDate.setDate(releaseDate.getDate() - timeAgo);
+ }
+
+ if (timeAgo.includes("months ago") || timeAgo.includes("month ago")) {
+ releaseDate.setMonth(releaseDate.getMonth() - timeAgo);
+ }
+
+ releaseDate = moment(releaseDate).format("MMMM DD, YYYY");
+ }
+
+ return releaseDate;
+};
diff --git a/src/jp/syosetu/SyosetuScrapper.js b/src/jp/syosetu.js
similarity index 63%
rename from src/jp/syosetu/SyosetuScrapper.js
rename to src/jp/syosetu.js
index 5d5aef9..f2b5fc1 100644
--- a/src/jp/syosetu/SyosetuScrapper.js
+++ b/src/jp/syosetu.js
@@ -1,6 +1,6 @@
-// scraper and cheerio for scrapping from site
-const { scraper } = require("../../helper");
-const cheerio = require("cheerio");
+import cheerio from "react-native-cheerio";
+
+import moment from "moment";
const baseUrl = "https://syosetu.com"; // base url for syosetu.com
@@ -25,10 +25,11 @@ const getChapterUrl = (id, chn) =>
`https://ncode.syosetu.com/${id}${chn === "oneshot" ? "" : `/${chn}`}`;
// Because syosetu does not have any covers, I did a replacement for them
-const novelCover = "https://raw.githubusercontent.com/skillgg/lnreader-sources/main/src/jp/syosetu/not_found.png";
+const novelCover =
+ "https://raw.githubusercontent.com/skillgg/lnreader-sources/main/src/jp/syosetu/not_found.png";
// ID of this extension
-const extensionId = 36;
+const sourceId = 36;
// Name of this extension
const sourceName = "Syosetu";
@@ -38,20 +39,18 @@ const sourceName = "Syosetu";
// TOOD: in-app dynamic loading on scroll
const maxPageLoad = 3;
-const novelsScraper = async (req, res) => {
+const popularNovels = async (page) => {
+ const totalPages = 100;
// array of all the novels
let novels = [];
// returns list of novels from given page
let getNovelsFromPage = async (pagenumber) => {
// load page
- console.log(searchUrl(pagenumber));
- const body = await scraper(searchUrl(pagenumber || null));
- console.log("Loaded");
+ const result = await fetch(searchUrl(pagenumber || null));
+ const body = await result.text();
// Cheerio it!
const cheerioQuery = cheerio.load(body, { decodeEntities: false });
- console.log("Parsed");
-
let pageNovels = [];
// find class=searchkekka_box
cheerioQuery(".searchkekka_box").each(function (i, e) {
@@ -63,58 +62,49 @@ const novelsScraper = async (req, res) => {
pageNovels.push({
novelName: novelDIV.text(), // get the name
novelUrl: getLastPartOfUrl(novelA.attribs.href), // get last part of the link
- extensionId,
+ sourceId,
novelCover, // TODO: IDK what to do about covers... On Syo they don't have them
});
});
- console.log("Read");
// return all novels from this page
return pageNovels;
};
- // counter of loaded pages
- let pagesLoaded = 0;
- do {
- // always load first one
- novels.push(...(await getNovelsFromPage(pagesLoaded + 1)));
- pagesLoaded++;
- console.log("Added");
- } while (pagesLoaded < maxPageLoad); // check if we should load more
-
+ novels = await getNovelsFromPage(page);
/** Use
* novels.push(...(await getNovelsFromPage(pageNumber)))
* if you want to load more
*/
// respond with novels!
- res.json(novels);
- console.log("Finished!");
+ return { totalPages, novels };
};
-const novelScraper = async (req, res) => {
- const novelUrl = getNovelUrl(req.params.novelUrl);
+const parseNovelAndChapters = async (novelUrl) => {
+ const url = getNovelUrl(novelUrl);
let chapters = [];
- const body = await scraper(novelUrl);
+ const result = await fetch(url);
+ const body = await result.text();
const cheerioQuery = cheerio.load(body, { decodeEntities: false });
// create novel object
let novel = {
- extensionId,
+ sourceId,
sourceName,
- sourceUrl: novelUrl,
- novelUrl: req.params.novelUrl,
+ url: url,
+ novelUrl,
novelName: cheerioQuery(".novel_title").text(),
+ author: cheerioQuery(".novel_writername").text().replace("作者：", ""),
novelCover,
};
// Get all the chapters
const cqGetChapters = cheerioQuery(".novel_sublist2");
- console.log(novelUrl, cqGetChapters.length);
if (cqGetChapters.length !== 0) {
// has more than 1 chapter
- novel.novelSummary = cheerioQuery("#novel_ex")
+ novel.summary = cheerioQuery("#novel_ex")
.text()
.replace(/<\s*br.*?>/g, "\n");
cqGetChapters.each(function (i, e) {
@@ -138,53 +128,51 @@ const novelScraper = async (req, res) => {
* but every oneshot is set as "there are no chapters" and all contents are thrown into the description!!
*/
// get summary for oneshot chapter
- novel.novelSummary = cheerio // because there is no summary anywhere on the novel page, we have to take it from search page for a one-shot manga
- .load(await scraper(searchUrl() + `&word=${novel.novelName}`), {
- decodeEntities: false,
- })(".searchkekka_box") // find the manga in search.php
+
+ const result = await fetch(searchUrl() + `&word=${novel.novelName}`);
+ const body = await result.text();
+ const summaryQuery = cheerio.load(body, { decodeEntities: false });
+ const foundText = summaryQuery(".searchkekka_box")
.first()
.find(".ex")
.text()
- .replace(/\s{2,}/g, "\n"); // get the description as text
+ .replace(/\s{2,}/g, "\n");
+ novel.summary = foundText;
+
// add single chapter
chapters.push({
- chapterName: novel.novelName,
+ chapterName: "Oneshot",
releaseDate: cheerioQuery("head")
.find("meta[name='WWWC']")
.attr("content"), // get date from metadata
chapterUrl: "oneshot", // set chapterUrl to oneshot so that chapterScraper knows it's a one-shot
});
}
+ novel.chapters = chapters;
- novel.novelChapters = chapters;
-
- res.json(novel);
- console.log("Finished!");
+ return novel;
};
-let chapterScraper = async (req, res) => {
- const novelUrl = req.params.novelUrl;
- const chapterUrl = req.params.chapterUrl;
-
+let parseChapter = async (novelUrl, chapterUrl) => {
const url = getChapterUrl(novelUrl, chapterUrl); // get Url
+ const result = await fetch(url);
+ const body = await result.text();
+
// create cheerioQuery
- const cheerioQuery = cheerio.load(await scraper(url), {
+ const cheerioQuery = cheerio.load(body, {
decodeEntities: false,
});
+ let chapterText = cheerioQuery("#novel_honbun") // get chapter text
+ .html();
// create chapter data structure
let chapter = {
- extensionId: 1,
+ sourceId: 36,
novelUrl,
chapterUrl,
chapterName: "",
- chapterText: cheerioQuery("#novel_honbun") // get chapter text
- .text()
- .replace(/<\s*br.*?>/g, "")
- .replace(/\n{2,}/g, "\n"), // remove double breaklines
- nextChapter: null,
- prevChapter: null,
+ chapterText,
};
if (chapterUrl === "oneshot")
@@ -200,26 +188,13 @@ let chapterScraper = async (req, res) => {
const chapterButtons = cheerioQuery(
"#novel_contents .novel_bn"
).first();
- if (chapterButtons.length === 1) {
- const button = chapterButtons.find("a");
- if (button.text().match(/次/))
- chapter.nextChapter = getLastPartOfUrl(button.attr("href"));
- else chapter.prevChapter = getLastPartOfUrl(button.attr("href"));
- } else {
- const firstButton = chapterButtons.find("a").first();
- const lastButton = chapterButtons.find("a").last();
- chapter.prevChapter = getLastPartOfUrl(firstButton.attribs.href);
- chapter.nextChapter = getLastPartOfUrl(lastButton.attribs.href);
- }
}
- res.json(chapter);
- console.log("Finished!");
+ return chapter;
};
-let searchScraper = async (req, res) => {
- const searchTerm = req.query.s;
- const orderBy = req.query.o;
+let searchNovels = async (searchTerm) => {
+ // const orderBy = req.query.o;
// array of all the novels
let novels = [];
@@ -229,10 +204,10 @@ let searchScraper = async (req, res) => {
// returns list of novels from given page
let getNovelsFromPage = async (pagenumber) => {
// load page
- const body = await scraper(
- searchUrl(pagenumber || null, req.query.o || null) +
- `&word=${searchTerm}`
+ const result = await fetch(
+ searchUrl(pagenumber || null, null) + `&word=${searchTerm}`
);
+ const body = await result.text();
// Cheerio it!
const cheerioQuery = cheerio.load(body, { decodeEntities: false });
@@ -249,7 +224,7 @@ let searchScraper = async (req, res) => {
pageNovels.push({
novelName: novelDIV.text(), // get the name
novelUrl: getLastPartOfUrl(novelA.attribs.href), // get last part of the link
- extensionId,
+ sourceId,
novelCover, // TODO: IDK what to do about covers... On Syo they don't have them
});
});
@@ -258,12 +233,14 @@ let searchScraper = async (req, res) => {
};
// counter of loaded pages
- let pagesLoaded = 0;
- do {
- // always load first one
- novels.push(...(await getNovelsFromPage(pagesLoaded + 1)));
- pagesLoaded++;
- } while (pagesLoaded < maxPageLoad && isNext); // check if we should load more
+ // let pagesLoaded = 0;
+ // do {
+ // // always load first one
+ // novels.push(...(await getNovelsFromPage(pagesLoaded + 1)));
+ // pagesLoaded++;
+ // } while (pagesLoaded < maxPageLoad && isNext); // check if we should load more
+
+ novels = await getNovelsFromPage(1);
/** Use
* novels.push(...(await getNovelsFromPage(pageNumber)))
@@ -271,13 +248,14 @@ let searchScraper = async (req, res) => {
*/
// respond with novels!
- res.json(novels);
- console.log("Finished!");
+ return novels;
};
-module.exports = mtlNovelScraper = {
- novelsScraper,
- novelScraper,
- chapterScraper,
- searchScraper,
+const SyosetuScraper = {
+ popularNovels,
+ parseNovelAndChapters,
+ parseChapter,
+ searchNovels,
};
+
+export default SyosetuScraper;
diff --git a/src/jp/syosetu/Syosetu.js b/src/jp/syosetu/Syosetu.js
deleted file mode 100644
index b970217..0000000
--- a/src/jp/syosetu/Syosetu.js
+++ /dev/null
@@ -1,11 +0,0 @@
-const express = require("express");
-const syosetuNovelScrapper = require("./SyosetuScrapper");
-
-const router = express.Router();
-
-router.get("/novels/", syosetuNovelScrapper.novelsScraper);
-router.get("/novel/:novelUrl", syosetuNovelScrapper.novelScraper);
-router.get("/novel/:novelUrl/:chapterUrl", syosetuNovelScrapper.chapterScraper);
-router.get("/search/", syosetuNovelScrapper.searchScraper);
-
-module.exports = router;
diff --git a/src/jp/syosetu/icon.png b/src/jp/syosetu/icon.png
deleted file mode 100644
index a516abf..0000000
Binary files a/src/jp/syosetu/icon.png and /dev/null differ
diff --git a/src/jp/syosetu/not_found.png b/src/jp/syosetu/not_found.png
deleted file mode 100644
index fd89a4a..0000000
Binary files a/src/jp/syosetu/not_found.png and /dev/null differ
diff --git a/src/multisrc/madara/MadaraGenerator.js b/src/multisrc/madara/MadaraGenerator.js
new file mode 100644
index 0000000..0aa7825
--- /dev/null
+++ b/src/multisrc/madara/MadaraGenerator.js
@@ -0,0 +1,188 @@
+import MadaraScraper from "./MadaraScraper";
+
+const getPath = (extensionId) => {
+ const path = {
+ 38: { novels: "manga", novel: "manga", chapter: "manga" },
+ 39: { novels: "series", novel: "series", chapter: "series" },
+ 40: { novels: "novels", novel: "novel", chapter: "novel" },
+ 41: { novels: "manga", novel: "manga", chapter: "manga" },
+ 42: { novels: "all-novels", novel: "manga", chapter: "manga" },
+ 43: { novels: "series", novel: "series", chapter: "series" },
+ 44: { novels: "series", novel: "series", chapter: "series" },
+ 45: { novels: "novel", novel: "novel", chapter: "novel" },
+ 46: { novels: "novel", novel: "novel", chapter: "novel" },
+ 47: { novels: "novel-list", novel: "novel", chapter: "novel" },
+ 55: { novels: "manga", novel: "manga", chapter: "manga" },
+ 56: { novels: "novel", novel: "novel", chapter: "novel" },
+ 57: { novels: "series", novel: "series", chapter: "series" },
+ 58: { novels: "novel", novel: "novel", chapter: "novel" },
+ 59: { novels: "novel", novel: "novels", chapter: "novel" },
+ 60: { novels: "novel", novel: "novel", chapter: "novel" },
+ 61: { novels: "novel", novel: "novel", chapter: "novel" },
+ 62: { novels: "novel", novel: "novel", chapter: "novel" },
+ 63: { novels: "novel", novel: "novel", chapter: "novel" },
+ 75: { novels: "series", novel: "series", chapter: "series" },
+ };
+
+ return (
+ path[extensionId] ?? {
+ novels: "novel",
+ novel: "novel",
+ chapter: "novel",
+ }
+ );
+};
+
+export const SkyNovelScraper = new MadaraScraper(
+ 38,
+ "https://skynovel.org/",
+ "SkyNovel",
+ getPath(38)
+);
+
+export const NovelCakeScraper = new MadaraScraper(
+ 39,
+ "https://novelcake.com/",
+ "NovelCake",
+ getPath(39)
+);
+
+export const NovelsRockScraper = new MadaraScraper(
+ 40,
+ "https://novelsrock.com/",
+ "NovelsRock",
+ getPath(40)
+);
+
+export const ZinnNovelScraper = new MadaraScraper(
+ 41,
+ "https://zinnovel.com/",
+ "ZinnNovel",
+ getPath(41)
+);
+
+export const NovelTranslateScraper = new MadaraScraper(
+ 42,
+ "https://noveltranslate.com/",
+ "NovelTranslate",
+ getPath(42)
+);
+
+export const LunarLettersScraper = new MadaraScraper(
+ 43,
+ "https://www.lunarletters.com/",
+ "LunarLetters",
+ getPath(43)
+);
+
+export const SleepyTranslationsScraper = new MadaraScraper(
+ 44,
+ "https://sleepytranslations.com/",
+ "SleepyTranslations",
+ getPath(44)
+);
+
+export const FreeNovelScraper = new MadaraScraper(
+ 45,
+ "https://freenovel.me/",
+ "FreeNovelMe",
+ getPath(45)
+);
+
+export const FirstKissNovelScraper = new MadaraScraper(
+ 46,
+ "https://1stkissnovel.love/",
+ "1stKissNovel",
+ getPath(46)
+);
+
+export const DaoNovelScraper = new MadaraScraper(
+ 47,
+ "https://daonovel.com/",
+ "DaoNovel",
+ getPath(47)
+);
+
+export const MostNovelScraper = new MadaraScraper(
+ 55,
+ "https://mostnovel.com/",
+ "MostNovel",
+ getPath(55)
+);
+
+export const NovelMultiverseScraper = new MadaraScraper(
+ 56,
+ "https://www.novelmultiverse.com/",
+ "NovelMultiverse",
+ getPath(56)
+);
+
+export const LightNovelHeavenScraper = new MadaraScraper(
+ 57,
+ "https://lightnovelheaven.com/",
+ "LightNovelHeaven",
+ getPath(57)
+);
+
+export const LightNovelsHubScraper = new MadaraScraper(
+ 58,
+ "https://lightnovelshub.com/",
+ "LightNovelsHub",
+ getPath(58)
+);
+
+export const MeionNovelScraper = new MadaraScraper(
+ 60,
+ "https://meionovel.id/",
+ "MeionNovel",
+ getPath(60)
+);
+
+export const WebNovelLoverScraper = new MadaraScraper(
+ 61,
+ "https://www.webnovelover.com/",
+ "WebNovelLover",
+ getPath(61)
+);
+
+export const BoxNovelOnlineScraper = new MadaraScraper(
+ 63,
+ "https://boxnovel.online/",
+ "BoxNovel.online",
+ getPath(63)
+);
+
+export const ClickNovelScraper = new MadaraScraper(
+ 64,
+ "https://clicknovel.net/",
+ "ClickNovel",
+ getPath(64)
+);
+
+export const ReadWebNovelsScraper = new MadaraScraper(
+ 65,
+ "https://readwebnovels.net/",
+ "ReadWebNovels",
+ getPath(65)
+);
+
+export const WBNovelScraper = new MadaraScraper(
+ 66,
+ "https://wbnovel.com/",
+ "WBNovel",
+ getPath(66)
+);
+
+export const WuxiaWorldDotSiteScraper = new MadaraScraper(
+ 74,
+ "https://wuxiaworld.site/",
+ "WuxiaWorld.Site",
+ getPath(74)
+);
+
+export const MysticalSeriesScraper = new MadaraScraper(
+ 75,
+ "https://mysticalmerries.com/",
+ "MysticalSeries",
+ getPath(75)
+);
diff --git a/multisrc/madara/MadaraScraper.js b/src/multisrc/madara/MadaraScraper.js
similarity index 64%
rename from multisrc/madara/MadaraScraper.js
rename to src/multisrc/madara/MadaraScraper.js
index 5bf8dac..6d9a67d 100644
--- a/multisrc/madara/MadaraScraper.js
+++ b/src/multisrc/madara/MadaraScraper.js
@@ -1,24 +1,25 @@
-const cheerio = require("cheerio");
-const fetch = require("node-fetch");
-const FormData = require("form-data");
-const { parseHtml } = require("../../src/helper");
+import moment from "moment";
+import cheerio from "react-native-cheerio";
+import { Status } from "../../helpers/constants";
+import { parseMadaraDate } from "../../helpers/parseDate";
class MadaraScraper {
- constructor(extensionId, baseUrl, sourceName, path) {
- this.extensionId = extensionId;
+ constructor(sourceId, baseUrl, sourceName, path) {
+ this.sourceId = sourceId;
this.baseUrl = baseUrl;
this.sourceName = sourceName;
this.path = path;
-
- this.novelsScraper = this.novelsScraper.bind(this);
- this.novelScraper = this.novelScraper.bind(this);
- this.chapterScraper = this.chapterScraper.bind(this);
- this.searchScraper = this.searchScraper.bind(this);
}
- async novelsScraper(req, res) {
- let url = this.baseUrl + this.path.novels + "/?m_orderby=rating";
- let extensionId = this.extensionId;
+ async popularNovels(page) {
+ const totalPages = 100;
+ let url =
+ this.baseUrl +
+ this.path.novels +
+ "/page/" +
+ page +
+ "/?m_orderby=rating";
+ let sourceId = this.sourceId;
const result = await fetch(url);
const body = await result.text();
@@ -29,7 +30,7 @@ class MadaraScraper {
$(".manga-title-badges").remove();
- $(".page-item-detail").each(function (result) {
+ $(".page-item-detail").each(function () {
const novelName = $(this).find(".post-title").text().trim();
let image = $(this).find("img");
const novelCover = image.attr("data-src") || image.attr("src");
@@ -41,7 +42,7 @@ class MadaraScraper {
.split("/")[4];
const novel = {
- extensionId,
+ sourceId,
novelName,
novelCover,
novelUrl,
@@ -50,11 +51,10 @@ class MadaraScraper {
novels.push(novel);
});
- res.json(novels);
+ return { totalPages, novels };
}
- async novelScraper(req, res) {
- const novelUrl = req.params.novelUrl;
+ async parseNovelAndChapters(novelUrl) {
const url = `${this.baseUrl}${this.path.novel}/${novelUrl}/`;
const result = await fetch(url);
@@ -64,11 +64,11 @@ class MadaraScraper {
let novel = {};
- novel.extensionId = this.extensionId;
+ novel.sourceId = this.sourceId;
novel.sourceName = this.sourceName;
- novel.sourceUrl = url;
+ novel.url = url;
novel.novelUrl = novelUrl;
@@ -78,9 +78,10 @@ class MadaraScraper {
novel.novelCover =
$(".summary_image > a > img").attr("data-src") ||
- $(".summary_image > a > img").attr("src");
+ $(".summary_image > a > img").attr("src") ||
+ "https://github.com/LNReader/lnreader-sources/blob/main/src/coverNotAvailable.jpg?raw=true";
- $(".post-content_item").each(function (result) {
+ $(".post-content_item").each(function () {
const detailName = $(this)
.find(".summary-heading > h5")
.text()
@@ -89,18 +90,20 @@ class MadaraScraper {
switch (detailName) {
case "Genre(s)":
- novel["Genre(s)"] = detail.replace(/[\t\n]/g, ",");
+ novel.genre = detail.replace(/[\t\n]/g, ",");
break;
case "Author(s)":
- novel["Author(s)"] = detail;
+ novel.author = detail;
break;
case "Status":
- novel.Status = detail;
+ novel.status = detail.includes("OnGoing")
+ ? Status.ONGOING
+ : Status.COMPLETED;
break;
}
});
- novel.novelSummary = $("div.summary__content").text().trim();
+ novel.summary = $("div.summary__content").text().trim();
let novelChapters = [];
@@ -118,36 +121,45 @@ class MadaraScraper {
});
const text = await data.text();
- $ = cheerio.load(text);
+ if (text !== "0") {
+ $ = cheerio.load(text);
+ }
- $(".wp-manga-chapter").each(function (result) {
+ $(".wp-manga-chapter").each(function () {
const chapterName = $(this)
.find("a")
.text()
.replace(/[\t\n]/g, "")
.trim();
- const releaseDate = $(this)
- .find("span")
+ let releaseDate = null;
+ releaseDate = $(this)
+ .find("span.chapter-release-date")
.text()
- .replace("Free", "")
.trim();
+ if (releaseDate) {
+ releaseDate = parseMadaraDate(releaseDate);
+ } else {
+ /**
+ * Insert current date
+ */
+
+ releaseDate = moment().format("MMMM DD, YYYY");
+ }
+
const chapterUrl = $(this).find("a").attr("href").split("/")[5];
novelChapters.push({ chapterName, releaseDate, chapterUrl });
});
- novel.novelChapters = novelChapters.reverse();
+ novel.chapters = novelChapters.reverse();
- res.json(novel);
+ return novel;
}
- async chapterScraper(req, res) {
- let novelUrl = req.params.novelUrl;
- let chapterUrl = req.params.chapterUrl;
-
- let extensionId = this.extensionId;
+ async parseChapter(novelUrl, chapterUrl) {
+ let sourceId = this.sourceId;
const url = `${this.baseUrl}${this.path.chapter}/${novelUrl}/${chapterUrl}`;
@@ -160,27 +172,19 @@ class MadaraScraper {
$(".text-center").text() || $("#chapter-heading").text();
let chapterText = $(".text-left").html();
- chapterText = parseHtml(chapterText);
-
- let nextChapter = null;
- let prevChapter = null;
const chapter = {
- extensionId,
+ sourceId,
novelUrl,
chapterUrl,
chapterName,
chapterText,
- nextChapter,
- prevChapter,
};
- res.json(chapter);
+ return chapter;
}
- async searchScraper(req, res) {
- const searchTerm = req.query.s;
-
+ async searchNovels(searchTerm) {
const url = `${this.baseUrl}?s=${searchTerm}&post_type=wp-manga`;
const result = await fetch(url);
@@ -189,9 +193,9 @@ class MadaraScraper {
const $ = cheerio.load(body);
let novels = [];
- let extensionId = this.extensionId;
+ let sourceId = this.sourceId;
- $(".c-tabs-item__content").each(function (result) {
+ $(".c-tabs-item__content").each(function () {
const novelName = $(this).find(".post-title").text().trim();
let image = $(this).find("img");
@@ -204,7 +208,7 @@ class MadaraScraper {
.split("/")[4];
const novel = {
- extensionId,
+ sourceId,
novelName,
novelCover,
novelUrl,
@@ -213,8 +217,8 @@ class MadaraScraper {
novels.push(novel);
});
- res.json(novels);
+ return novels;
}
}
-module.exports = MadaraScraper;
+export default MadaraScraper;
diff --git a/src/multisrc/wpmangastream/WPMangaStreamGenerator.js b/src/multisrc/wpmangastream/WPMangaStreamGenerator.js
new file mode 100644
index 0000000..8139632
--- /dev/null
+++ b/src/multisrc/wpmangastream/WPMangaStreamGenerator.js
@@ -0,0 +1,13 @@
+import WPMangaStreamScraper from "./WPMangaStreamScraper";
+
+export const KolNovelScraper = new WPMangaStreamScraper(
+ 53,
+ "https://kolnovel.com/",
+ "KolNovel"
+);
+
+export const RewayatArScraper = new WPMangaStreamScraper(
+ 54,
+ "https://rewayat-ar.site/",
+ "RewayatAr"
+);
diff --git a/multisrc/wpmangastream/WPMangaStreamScraper.js b/src/multisrc/wpmangastream/WPMangaStreamScraper.js
similarity index 57%
rename from multisrc/wpmangastream/WPMangaStreamScraper.js
rename to src/multisrc/wpmangastream/WPMangaStreamScraper.js
index 96baf61..7a67b55 100644
--- a/multisrc/wpmangastream/WPMangaStreamScraper.js
+++ b/src/multisrc/wpmangastream/WPMangaStreamScraper.js
@@ -1,22 +1,17 @@
-const cheerio = require("cheerio");
-const fetch = require("node-fetch");
-const { parseHtml } = require("../../src/helper");
+import cheerio from "react-native-cheerio";
-class MadaraScraper {
- constructor(extensionId, baseUrl, sourceName) {
- this.extensionId = extensionId;
+class WPMangaStreamScraper {
+ constructor(sourceId, baseUrl, sourceName) {
+ this.sourceId = sourceId;
this.baseUrl = baseUrl;
this.sourceName = sourceName;
-
- this.novelsScraper = this.novelsScraper.bind(this);
- this.novelScraper = this.novelScraper.bind(this);
- this.chapterScraper = this.chapterScraper.bind(this);
- this.searchScraper = this.searchScraper.bind(this);
}
- async novelsScraper(req, res) {
- let url = this.baseUrl + "series/?m_orderby=popular";
- let extensionId = this.extensionId;
+ async popularNovels(page) {
+ let totalPages = 100;
+ let url =
+ this.baseUrl + "series/?page=" + page + "&status=&order=popular";
+ let sourceId = this.sourceId;
const result = await fetch(url);
const body = await result.text();
@@ -25,7 +20,7 @@ class MadaraScraper {
let novels = [];
- $("article.bs").each(function (result) {
+ $("article.bs").each(function () {
const novelName = $(this).find(".ntitle").text().trim();
let image = $(this).find("img");
const novelCover = image.attr("src");
@@ -33,7 +28,7 @@ class MadaraScraper {
let novelUrl = $(this).find("a").attr("href").split("/")[4];
const novel = {
- extensionId,
+ sourceId,
novelName,
novelCover,
novelUrl,
@@ -42,11 +37,10 @@ class MadaraScraper {
novels.push(novel);
});
- res.json(novels);
+ return { totalPages, novels };
}
- async novelScraper(req, res) {
- const novelUrl = req.params.novelUrl;
+ async parseNovelAndChapters(novelUrl) {
const url = `${this.baseUrl}series/${novelUrl}/`;
const result = await fetch(url);
@@ -56,42 +50,42 @@ class MadaraScraper {
let novel = {};
- novel.extensionId = this.extensionId;
+ novel.sourceId = this.sourceId;
novel.sourceName = this.sourceName;
- novel.sourceUrl = url;
+ novel.url = url;
novel.novelUrl = novelUrl;
novel.novelName = $(".entry-title").text();
- novel.novelCover = $("img.thumb").attr("src");
+ novel.novelCover = $("img.wp-post-image").attr("src");
- $("div.spe > span").each(function (result) {
+ $("div.spe > span").each(function () {
const detailName = $(this).find("b").text().trim();
const detail = $(this).find("b").next().text().trim();
if ($(this).text().includes("الحالة:")) {
- novel.Status = $(this).text().replace("الحالة: ", "");
+ novel.status = $(this).text().replace("الحالة: ", "");
}
switch (detailName) {
case "المؤلف:":
- novel["Author(s)"] = detail;
+ novel.auhtor = detail;
break;
}
});
- novel["Genre(s)"] = $(".genxed").text().replace(/\s/g, ",");
+ novel.genre = $(".genxed").text().replace(/\s/g, ",");
- novel.novelSummary = $('div[itemprop="description"]').text().trim();
+ novel.summary = $('div[itemprop="description"]').text().trim();
let novelChapters = [];
$(".eplister")
.find("li")
- .each(function (result) {
+ .each(function () {
const chapterName =
$(this).find(".epl-num").text() +
" - " +
@@ -104,16 +98,13 @@ class MadaraScraper {
novelChapters.push({ chapterName, releaseDate, chapterUrl });
});
- novel.novelChapters = novelChapters.reverse();
+ novel.chapters = novelChapters.reverse();
- res.json(novel);
+ return novel;
}
- async chapterScraper(req, res) {
- let novelUrl = req.params.novelUrl;
- let chapterUrl = req.params.chapterUrl;
-
- let extensionId = this.extensionId;
+ async parseChapter(novelUrl, chapterUrl) {
+ let sourceId = this.sourceId;
const url = `${this.baseUrl}${chapterUrl}`;
@@ -123,29 +114,20 @@ class MadaraScraper {
const $ = cheerio.load(body);
let chapterName = $(".entry-title").text();
-
- let chapterText = $(".entry-content").html();
- chapterText = parseHtml(chapterText);
-
- let nextChapter = null;
- let prevChapter = null;
+ let chapterText = $("div.epcontent").html();
const chapter = {
- extensionId,
+ sourceId,
novelUrl,
chapterUrl,
chapterName,
chapterText,
- nextChapter,
- prevChapter,
};
- res.json(chapter);
+ return chapter;
}
- async searchScraper(req, res) {
- const searchTerm = req.query.s;
-
+ async searchNovels(searchTerm) {
const url = `${this.baseUrl}?s=${searchTerm}`;
const result = await fetch(url);
@@ -154,9 +136,9 @@ class MadaraScraper {
const $ = cheerio.load(body);
let novels = [];
- let extensionId = this.extensionId;
+ let sourceId = this.sourceId;
- $("article.bs").each(function (result) {
+ $("article.bs").each(function () {
const novelName = $(this).find(".ntitle").text().trim();
let image = $(this).find("img");
const novelCover = image.attr("src");
@@ -164,7 +146,7 @@ class MadaraScraper {
let novelUrl = $(this).find("a").attr("href").split("/")[4];
const novel = {
- extensionId,
+ sourceId,
novelName,
novelCover,
novelUrl,
@@ -173,8 +155,8 @@ class MadaraScraper {
novels.push(novel);
});
- res.json(novels);
+ return novels;
}
}
-module.exports = MadaraScraper;
+module.exports = WPMangaStreamScraper;
diff --git a/src/ru/ranobehub.js b/src/ru/ranobehub.js
new file mode 100644
index 0000000..da98731
--- /dev/null
+++ b/src/ru/ranobehub.js
@@ -0,0 +1,152 @@
+import cheerio from "react-native-cheerio";
+import { Status } from "../helpers/constants";
+
+const sourceId = 69;
+
+const sourceName = "RanobeHub";
+
+const baseUrl = "https://ranobehub.org/";
+
+const popularNovels = async (page) => {
+ let url =
+ baseUrl + "api/search?page=" + page + "&sort=computed_rating&take=40";
+ const totalPages = 69;
+
+ const result = await fetch(url);
+ const body = await result.json();
+
+ let novels = [];
+
+ body.resource.forEach((novel) => {
+ const novelName = novel.names.rus;
+ const novelCover = novel.poster.medium;
+ const novelUrl = novel.url.split("/").pop();
+
+ novels.push({
+ sourceId,
+ novelName,
+ novelCover,
+ novelUrl,
+ });
+ });
+
+ return { novels, totalPages };
+};
+
+const parseNovelAndChapters = async (novelUrl) => {
+ const url = baseUrl + "ranobe/" + novelUrl;
+
+ const result = await fetch(url);
+ const body = await result.text();
+
+ $ = cheerio.load(body);
+
+ let novel = { sourceId, sourceName, url, novelUrl };
+
+ novel.novelName = $("h1").text();
+
+ novel.novelCover = $("img.__posterbox ").attr("data-src");
+
+ novel.summary = $(".book-description__text").text().trim();
+
+ novel.genre = "";
+
+ novel.status = Status.UNKNOWN;
+
+ novel.author = $(".book-author").text().trim();
+
+ let novelChapters = [];
+
+ const novelId = novelUrl.split("-")[0];
+ const fetchChaptersUrl = `https://ranobehub.org/api/ranobe/${novelId}/contents`;
+
+ const chaptersRaw = await fetch(fetchChaptersUrl);
+ const chaptersJSON = await chaptersRaw.json();
+
+ chaptersJSON.volumes.map((volume) =>
+ volume.chapters.map((chapter) =>
+ novelChapters.push({
+ chapterName: chapter.name,
+ chapterUrl: chapter.url,
+ releaseDate: null,
+ })
+ )
+ );
+
+ novel.chapters = novelChapters;
+
+ return novel;
+};
+
+const parseChapter = async (novelUrl, chapterUrl) => {
+ const url = chapterUrl;
+ const chapterId = chapterUrl.split("/")[4];
+
+ const result = await fetch(url);
+ const body = await result.text();
+
+ $ = cheerio.load(body);
+
+ let chapterName = $(
+ "body > div.pusher.container_main > div:nth-child(5) > div:nth-child(1) > div.title-wrapper > h1"
+ ).text();
+ let chapterText = $(
+ "body > div.pusher.container_main > div:nth-child(5) > div:nth-child(1)"
+ ).html();
+
+ if (chapterText) {
+ /**
+ * Convert relative url to absolute
+ */
+
+ chapterText = chapterText
+ .replace(/<\s*script[^>]*>[\s\S]*?<\/script>/gim, "")
+ .replace(
+ /data-src="(.*)"/g,
+ `src="https://ranobehub.org/img/ranobe/content/${chapterId}/$1.jpg"`
+ );
+ }
+
+ console.log(chapterText);
+
+ const chapter = {
+ sourceId,
+ novelUrl,
+ chapterUrl,
+ chapterName,
+ chapterText,
+ };
+
+ return chapter;
+};
+
+const searchNovels = async (searchTerm) => {
+ const url = "https://ranobehub.org/api/fulltext/ranobe?query=" + searchTerm;
+
+ const result = await fetch(url);
+ const data = await result.json();
+
+ let novels = [];
+
+ data.data.map((novel) =>
+ novels.push({
+ sourceId,
+ novelName: novel.names.rus,
+ novelUrl: novel.url.match(
+ /https:\/\/ranobehub\.org\/ranobe\/(.*?)\?utm_source=search_name&utm_medium=search&utm_campaign=search_using/
+ )[1],
+ novelCover: novel.image,
+ })
+ );
+
+ return novels;
+};
+
+const RanobeHubScraper = {
+ popularNovels,
+ parseNovelAndChapters,
+ parseChapter,
+ searchNovels,
+};
+
+export default RanobeHubScraper;
diff --git a/src/ru/ranobehub/icon.png b/src/ru/ranobehub/icon.png
deleted file mode 100644
index 0591f7b..0000000
Binary files a/src/ru/ranobehub/icon.png and /dev/null differ
diff --git a/src/sources.js b/src/sources.js
new file mode 100644
index 0000000..1988c83
--- /dev/null
+++ b/src/sources.js
@@ -0,0 +1,165 @@
+import ComradeMaoScraper from "./en/comrademao";
+import ReadLightNovelScraper from "./en/readlightnovel";
+import BoxNovelScraper from "./en/boxnovel";
+import fastNovelScraper from "./en/fastnovel";
+import readNovelFullScraper from "./en/readnovelfull";
+import mtlNovelScraper from "./en/mtlnovel";
+import novelhallScraper from "./en/novelhall";
+import WuxiaWorldScraper from "./en/wuxiaworld";
+import novelFullScraper from "./en/novelfull";
+import novelTrenchScraper from "./en/noveltrench";
+import vipNovelScraper from "./en/vipnovel";
+import kissLightNovelScraper from "./en/kisslightnovels";
+import WuxiaWorldSiteScraper from "./en/wuxiaworldsite";
+import FreeWebNovelScraper from "./en/freewebnovel";
+import JPMTLScraper from "./en/jpmtl";
+import lightNovelPubScraper from "./en/lightnovelpub";
+import WuxiaWorldCoScraper from "./en/wuxiaworldco";
+import novelUpdatesCcScraper from "./en/novelupdatescc";
+import readLightNovelCcScraper from "./en/readlightnovelcc";
+import tapReadScraper from "./en/tapread";
+import WuxiaWorldCloudScraper from "./en/wuxiaworldcloud";
+import WoopReadScraper from "./en/woopread";
+import FoxaHolicScraper from "./en/foxaholic";
+import EinharjarProjectScraper from "./es/einherjarproject";
+import TuNovelaLigeraScraper from "./es/tunovelaligera";
+import SkyNovelsScraper from "./es/skynovels";
+import NovelasLigeraScraper from "./es/novelasligera";
+import YuukiTlsScraper from "./es/yuukitls";
+import NovelaWuxiaScraper from "./es/novelawuxia";
+import OasisTranslationsScraper from "./es/oasistranslations";
+import HasuTlScraper from "./es/hasutl";
+import NovelPassionScraper from "./en/novelpassion";
+import RoyalRoadScraper from "./en/royalroad";
+import ScribbleHubScraper from "./en/scribblehub";
+import SyosetuScraper from "./jp/syosetu";
+import LNMTLScraper from "./en/lnmtl";
+import {
+ BoxNovelOnlineScraper,
+ ClickNovelScraper,
+ DaoNovelScraper,
+ FirstKissNovelScraper,
+ FreeNovelScraper,
+ LightNovelHeavenScraper,
+ LightNovelsHubScraper,
+ LunarLettersScraper,
+ MeionNovelScraper,
+ MostNovelScraper,
+ MysticalSeriesScraper,
+ NovelCakeScraper,
+ NovelMultiverseScraper,
+ NovelsRockScraper,
+ NovelTranslateScraper,
+ ReadWebNovelsScraper,
+ SkyNovelScraper,
+ SleepyTranslationsScraper,
+ WBNovelScraper,
+ WebNovelLoverScraper,
+ WuxiaWorldDotSiteScraper,
+ ZinnNovelScraper,
+} from "./multisrc/madara/MadaraGenerator";
+import WuxiaBlogScraper from "./en/wuxiablog";
+import WuxiaCityScraper from "./en/wuxiacity";
+import NovelUpdatesScraper from "./en/novelupdates";
+import RanobesScraper from "./en/ranobes";
+import YushuboScraper from "./ch/yushubo";
+import {
+ KolNovelScraper,
+ RewayatArScraper,
+} from "./multisrc/wpmangastream/WPMangaStreamGenerator";
+import WLNUpdatesScraper from "./en/wlnupdates";
+import ReaperScansScraper from "./en/reaperscans";
+import ReadwnScraper from "./en/readwn";
+import RanobeHubScraper from "./ru/ranobehub";
+import ArNovelScraper from "./ar/arnovel";
+import DivineDaoLibraryScraper from "./en/divinedaolibrary";
+import NovelOnlineFullScraper from "./en/novelonlinefull";
+import LightNovelUpdatesScraper from "./en/lightnovelupdates";
+import EpikNovelScraper from "./tr/epiknovel";
+import TravisTranslationsScraper from "./en/travistranslations";
+import NovelDeGlaceScraper from "./fr/noveldeglace";
+import RainOfSnowScraper from "./en/rainofsnow";
+
+export const getSource = (sourceId) => {
+ const scrapers = {
+ 1: BoxNovelScraper,
+ 2: ReadLightNovelScraper,
+ 3: fastNovelScraper,
+ 4: readNovelFullScraper,
+ 5: mtlNovelScraper,
+ 6: novelhallScraper,
+ 7: WuxiaWorldScraper,
+ 8: novelFullScraper,
+ 9: novelTrenchScraper,
+ 10: vipNovelScraper,
+ 11: kissLightNovelScraper,
+ 12: WuxiaWorldSiteScraper,
+ 13: FreeWebNovelScraper,
+ 14: JPMTLScraper,
+ 15: lightNovelPubScraper,
+ 16: WuxiaWorldCoScraper,
+ 17: tapReadScraper,
+ 18: novelUpdatesCcScraper,
+ 19: readLightNovelCcScraper,
+ 20: WuxiaWorldCloudScraper,
+ 21: WoopReadScraper,
+ 22: FoxaHolicScraper,
+ 23: TuNovelaLigeraScraper,
+ 24: SkyNovelsScraper,
+ 25: EinharjarProjectScraper,
+ 26: NovelasLigeraScraper,
+ 27: ComradeMaoScraper,
+ 28: YuukiTlsScraper,
+ 29: HasuTlScraper,
+ 30: OasisTranslationsScraper,
+ 31: NovelaWuxiaScraper,
+ 33: NovelPassionScraper,
+ 34: RoyalRoadScraper,
+ 35: ScribbleHubScraper,
+ 36: SyosetuScraper,
+ 37: LNMTLScraper,
+ 38: SkyNovelScraper,
+ 39: NovelCakeScraper,
+ 40: NovelsRockScraper,
+ 41: ZinnNovelScraper,
+ 42: NovelTranslateScraper,
+ 43: LunarLettersScraper,
+ 44: SleepyTranslationsScraper,
+ 45: FreeNovelScraper,
+ 46: FirstKissNovelScraper,
+ 47: DaoNovelScraper,
+ 48: WuxiaBlogScraper,
+ 49: WuxiaCityScraper,
+ 50: NovelUpdatesScraper,
+ 51: RanobesScraper,
+ 52: YushuboScraper,
+ 53: KolNovelScraper,
+ 54: RewayatArScraper,
+ 55: MostNovelScraper,
+ 56: NovelMultiverseScraper,
+ 57: LightNovelHeavenScraper,
+ 58: LightNovelsHubScraper,
+ 59: ArNovelScraper,
+ 60: MeionNovelScraper,
+ 61: WebNovelLoverScraper,
+ 62: WLNUpdatesScraper,
+ 63: BoxNovelOnlineScraper,
+ 64: ClickNovelScraper,
+ 65: ReadWebNovelsScraper,
+ 66: WBNovelScraper,
+ 67: ReaperScansScraper,
+ 68: ReadwnScraper,
+ 69: RanobeHubScraper,
+ 70: DivineDaoLibraryScraper,
+ 71: NovelOnlineFullScraper,
+ 72: LightNovelUpdatesScraper,
+ 73: EpikNovelScraper,
+ 74: WuxiaWorldDotSiteScraper,
+ 75: MysticalSeriesScraper,
+ 76: TravisTranslationsScraper,
+ 77: NovelDeGlaceScraper,
+ 78: RainOfSnowScraper,
+ };
+
+ return scrapers[sourceId];
+};
diff --git a/src/sources.json b/src/sources.json
new file mode 100644
index 0000000..5fa85fa
--- /dev/null
+++ b/src/sources.json
@@ -0,0 +1,541 @@
+[
+ {
+ "sourceId": 1,
+ "url": "https://boxnovel.com/",
+ "sourceName": "BoxNovel",
+ "icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/en/boxnovel/icon.png?raw=true",
+ "lang": "English"
+ },
+ {
+ "sourceId": 2,
+ "sourceName": "ReadLightNovel",
+ "url": "https://www.readlightnovel.org",
+ "lang": "English",
+ "icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/en/readlightnovel/icon.png?raw=true"
+ },
+ {
+ "sourceId": 3,
+ "url": "https://fastnovel.net/",
+ "sourceName": "FastNovel",
+ "icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/en/fastnovel/icon.png?raw=true",
+ "lang": "English"
+ },
+ {
+ "sourceId": 4,
+ "url": "https://readnovelfull.com/",
+ "sourceName": "ReadNovelFull",
+ "icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/en/readnovelfull/icon.png?raw=true",
+ "lang": "English"
+ },
+ {
+ "sourceId": 5,
+ "url": "https://www.mtlnovel.com/",
+ "sourceName": "MTLNovel",
+ "icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/en/mtlnovel/icon.png?raw=true",
+ "lang": "English"
+ },
+ {
+ "sourceId": 6,
+ "url": "https://www.novelhall.com/",
+ "sourceName": "NovelHall",
+ "icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/en/novelhall/icon.png?raw=true",
+ "lang": "English"
+ },
+ {
+ "sourceId": 7,
+ "url": "https://www.wuxiaworld.com/",
+ "sourceName": "WuxiaWorld",
+ "icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/en/wuxiaworld/icon.png?raw=true",
+ "lang": "English"
+ },
+ {
+ "sourceId": 8,
+ "url": "https://novelfull.com/",
+ "sourceName": "NovelFull",
+ "icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/en/novelfull/icon.png?raw=true",
+ "lang": "English"
+ },
+ {
+ "sourceId": 9,
+ "url": "https://noveltrench.com/",
+ "sourceName": "NovelTrench",
+ "icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/en/noveltrench/icon.png?raw=true",
+ "lang": "English"
+ },
+ {
+ "sourceId": 10,
+ "url": "https://vipnovel.com/",
+ "sourceName": "VipNovel",
+ "icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/en/vipnovel/icon.png?raw=true",
+ "lang": "English"
+ },
+ {
+ "sourceId": 11,
+ "url": "https://kisslightnovels.info/",
+ "sourceName": "KissLightNovels",
+ "icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/en/kisslightnovels/icon.png?raw=true",
+ "lang": "English"
+ },
+ {
+ "sourceId": 12,
+ "url": "https://wuxiaworldsite.co/",
+ "sourceName": "WuxiaWorldSite",
+ "icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/en/wuxiaworldsite/icon.png?raw=true",
+ "lang": "English"
+ },
+ {
+ "sourceId": 13,
+ "url": "https://freewebnovel.com/",
+ "sourceName": "FreeWebNovel",
+ "icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/en/freewebnovel/icon.png?raw=true",
+ "lang": "English"
+ },
+ {
+ "sourceId": 14,
+ "url": "https://jpmtl.com/",
+ "sourceName": "JPMTL",
+ "icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/en/jpmtl/icon.png?raw=true",
+ "lang": "English"
+ },
+ {
+ "sourceId": 15,
+ "url": "https://www.lightnovelpub.com/",
+ "sourceName": "LightNovelPub",
+ "icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/en/lightnovelpub/icon.png?raw=true",
+ "lang": "English"
+ },
+ {
+ "sourceId": 16,
+ "url": "https://www.wuxiaworld.co/",
+ "sourceName": "WuxiaWorld.co",
+ "icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/en/wuxiaworldco/icon.png?raw=true",
+ "lang": "English"
+ },
+ {
+ "sourceId": 17,
+ "url": "http://www.tapread.com/",
+ "sourceName": "TapRead",
+ "icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/en/tapread/icon.png?raw=true",
+ "lang": "English"
+ },
+ {
+ "sourceId": 18,
+ "url": "https://www.novelupdates.cc/",
+ "sourceName": "NovelUpdates.cc",
+ "icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/en/novelupdatescc/icon.png?raw=true",
+ "lang": "English"
+ },
+ {
+ "sourceId": 19,
+ "url": "https://m.readlightnovel.cc/",
+ "sourceName": "ReadLightNovel.cc",
+ "icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/en/readlightnovelcc/icon.png?raw=true",
+ "lang": "English"
+ },
+ {
+ "sourceId": 20,
+ "url": "http://wuxiaworld.cloud/",
+ "sourceName": "WuxiaWorld.cloud",
+ "icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/en/wuxiaworldcloud/icon.png?raw=true",
+ "lang": "English"
+ },
+ {
+ "sourceId": 21,
+ "url": "https://woopread.com/",
+ "sourceName": "WoopRead",
+ "icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/en/woopread/icon.png?raw=true",
+ "lang": "English"
+ },
+ {
+ "sourceId": 22,
+ "url": "https://foxaholic.com/",
+ "sourceName": "Foxaholic",
+ "icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/en/foxaholic/icon.png?raw=true",
+ "lang": "English"
+ },
+ {
+ "sourceId": 23,
+ "url": "https://tunovelaligera.com/",
+ "sourceName": "TuNovelaLigera",
+ "icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/es/tunovelaligera/icon.png?raw=true",
+ "lang": "Spanish"
+ },
+ {
+ "sourceId": 24,
+ "url": "https://www.skynovels.net/",
+ "sourceName": "SkyNovels",
+ "icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/es/skynovels/icon.png?raw=true",
+ "lang": "Spanish"
+ },
+ {
+ "sourceId": 25,
+ "url": "https://einherjarproject.net/",
+ "sourceName": "EinherjarProject",
+ "icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/es/einherjarproject/icon.png?raw=true",
+ "lang": "Spanish"
+ },
+ {
+ "sourceId": 26,
+ "url": "https://novelasligera.com/",
+ "sourceName": "NovelasLigera",
+ "icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/es/novelasligera/icon.png?raw=true",
+ "lang": "Spanish"
+ },
+ {
+ "sourceId": 27,
+ "url": "https://comrademao.com/",
+ "sourceName": "ComradeMao",
+ "icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/en/comrademao/icon.png?raw=true",
+ "lang": "English"
+ },
+ {
+ "sourceId": 28,
+ "url": "https://yuukitls.com/",
+ "sourceName": "YuukiTls",
+ "icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/es/yuukitls/icon.png?raw=true",
+ "lang": "Spanish"
+ },
+ {
+ "sourceId": 29,
+ "url": "https://hasutl.wordpress.com/",
+ "sourceName": "Hasu Translations",
+ "icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/es/hasutl/icon.png?raw=true",
+ "lang": "Spanish"
+ },
+ {
+ "sourceId": 30,
+ "url": "https://oasistranslations.wordpress.com/",
+ "sourceName": "Oasis Translations",
+ "icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/es/oasistranslations/icon.png?raw=true",
+ "lang": "Spanish"
+ },
+ {
+ "sourceId": 31,
+ "url": "http://www.novelawuxia.com/",
+ "sourceName": "Novela Wuxia",
+ "icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/es/novelawuxia/icon.png?raw=true",
+ "lang": "Spanish"
+ },
+ {
+ "sourceId": 33,
+ "url": "https://www.novelpassion.com/",
+ "sourceName": "NovelPassion",
+ "icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/en/novelpassion/icon.png?raw=true",
+ "lang": "English"
+ },
+ {
+ "sourceId": 34,
+ "url": "https://www.royalroad.com/",
+ "sourceName": "Royal Road",
+ "icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/en/royalroad/icon.png?raw=true",
+ "lang": "English"
+ },
+ {
+ "sourceId": 35,
+ "url": "https://www.scribblehub.com/",
+ "sourceName": "Scribble Hub",
+ "icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/en/scribblehub/icon.png?raw=true",
+ "lang": "English"
+ },
+ {
+ "sourceId": 36,
+ "url": "https://syosetu.com",
+ "sourceName": "Syosetu",
+ "icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/jp/syosetu/icon.png?raw=true",
+ "lang": "Japanese"
+ },
+ {
+ "sourceId": 37,
+ "url": "https://lnmtl.com/",
+ "sourceName": "LNMTL",
+ "icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/en/lnmtl/icon.png?raw=true",
+ "lang": "English"
+ },
+ {
+ "sourceId": 38,
+ "url": "https://skynovel.org/",
+ "sourceName": "SkyNovel",
+ "icon": "https://github.com/LNReader/lnreader-sources/blob/main/multisrc/madara/icons/skynovel.png?raw=true",
+ "lang": "English"
+ },
+ {
+ "sourceId": 39,
+ "url": "https://novelcake.com/",
+ "sourceName": "NovelCake",
+ "icon": "https://github.com/LNReader/lnreader-sources/blob/main/multisrc/madara/icons/novelcake.png?raw=true",
+ "lang": "English"
+ },
+ {
+ "sourceId": 40,
+ "url": "https://novelsrock.com/",
+ "sourceName": "NovelsRock",
+ "icon": "https://github.com/LNReader/lnreader-sources/blob/main/multisrc/madara/icons/novelsrock.png?raw=true",
+ "lang": "English"
+ },
+ {
+ "sourceId": 41,
+ "url": "https://zinnovel.com/",
+ "sourceName": "ZinnNovel",
+ "icon": "https://github.com/LNReader/lnreader-sources/blob/main/multisrc/madara/icons/zinnovel.png?raw=true",
+ "lang": "English"
+ },
+ {
+ "sourceId": 42,
+ "url": "https://noveltranslate.com/",
+ "sourceName": "NovelTranslate",
+ "icon": "https://github.com/LNReader/lnreader-sources/blob/main/multisrc/madara/icons/noveltranslate.png?raw=true",
+ "lang": "English"
+ },
+ {
+ "sourceId": 43,
+ "url": "https://www.lunarletters.com/",
+ "sourceName": "LunarLetters",
+ "icon": "https://github.com/LNReader/lnreader-sources/blob/main/multisrc/madara/icons/lunarletters.png?raw=true",
+ "lang": "English"
+ },
+ {
+ "sourceId": 44,
+ "url": "https://sleepytranslations.com/",
+ "sourceName": "SleepyTranslations",
+ "icon": "https://github.com/LNReader/lnreader-sources/blob/main/multisrc/madara/icons/sleepytranslations.png?raw=true",
+ "lang": "English"
+ },
+ {
+ "sourceId": 45,
+ "url": "https://freenovel.me/",
+ "sourceName": "FreeNovelMe",
+ "icon": "https://github.com/LNReader/lnreader-sources/blob/main/multisrc/madara/icons/freenovel.png?raw=true",
+ "lang": "English"
+ },
+ {
+ "sourceId": 46,
+ "url": "https://1stkissnovel.love/",
+ "sourceName": "1stKissNovel",
+ "icon": "https://github.com/LNReader/lnreader-sources/blob/main/multisrc/madara/icons/1stkissnovel.png?raw=true",
+ "lang": "English"
+ },
+ {
+ "sourceId": 47,
+ "url": "https://daonovel.com/",
+ "sourceName": "DaoNovel",
+ "icon": "https://github.com/LNReader/lnreader-sources/blob/main/multisrc/madara/icons/daonovel.png?raw=true",
+ "lang": "English"
+ },
+ {
+ "sourceId": 48,
+ "url": "https://www.wuxia.blog/",
+ "sourceName": "Wuxia.Blog",
+ "icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/en/wuxiablog/icon.png?raw=true",
+ "lang": "English"
+ },
+ {
+ "sourceId": 49,
+ "url": "https://wuxia.city/",
+ "sourceName": "Wuxia.City",
+ "icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/en/wuxiacity/icon.png?raw=true",
+ "lang": "English"
+ },
+ {
+ "sourceId": 50,
+ "url": "https://www.novelupdates.com/",
+ "sourceName": "Novel Updates",
+ "icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/en/novelupdates/icon.png?raw=true",
+ "lang": "English"
+ },
+ {
+ "sourceId": 51,
+ "url": "https://ranobes.net/",
+ "sourceName": "Ranobes",
+ "icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/en/ranobes/icon.png?raw=true",
+ "lang": "English"
+ },
+ {
+ "sourceId": 52,
+ "url": "https://m.yushubo.com",
+ "sourceName": "Yushubo",
+ "icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/ch/yushubo/icon.png?raw=true",
+ "lang": "Chinese"
+ },
+ {
+ "sourceId": 53,
+ "url": "https://kolnovel.com/",
+ "sourceName": "KolNovel",
+ "icon": "https://github.com/LNReader/lnreader-sources/blob/main/multisrc/wpmangastream/icons/kolnovel.png?raw=true",
+ "lang": "Arabic"
+ },
+ {
+ "sourceId": 54,
+ "url": "https://rewayat-ar.site/",
+ "sourceName": "RewayatAr",
+ "icon": "https://github.com/LNReader/lnreader-sources/blob/main/multisrc/wpmangastream/icons/rewayatar.png?raw=true",
+ "lang": "Arabic"
+ },
+ {
+ "sourceId": 55,
+ "url": "https://mostnovel.com/",
+ "sourceName": "MostNovel",
+ "icon": "https://github.com/LNReader/lnreader-sources/blob/main/multisrc/madara/icons/mostnovel.png?raw=true",
+ "lang": "English"
+ },
+ {
+ "sourceId": 56,
+ "url": "https://www.novelmultiverse.com/",
+ "sourceName": "NovelMultiverse",
+ "icon": "https://github.com/LNReader/lnreader-sources/blob/main/multisrc/madara/icons/novelmultiverse.png?raw=true",
+ "lang": "English"
+ },
+ {
+ "sourceId": 57,
+ "url": "https://lightnovelheaven.com/",
+ "sourceName": "LightNovelHeaven",
+ "icon": "https://github.com/LNReader/lnreader-sources/blob/main/multisrc/madara/icons/lightnovelheaven.png?raw=true",
+ "lang": "English"
+ },
+ {
+ "sourceId": 58,
+ "url": "https://lightnovelshub.com/",
+ "sourceName": "LightNovelsHub",
+ "icon": "https://github.com/LNReader/lnreader-sources/blob/main/multisrc/madara/icons/lightnovelshub.png?raw=true",
+ "lang": "English"
+ },
+ {
+ "sourceId": 59,
+ "url": "https://arnovel.me/",
+ "sourceName": "ArNovel",
+ "icon": "https://github.com/LNReader/lnreader-sources/blob/main/multisrc/madara/icons/arnovel.png?raw=true",
+ "lang": "Arabic"
+ },
+ {
+ "sourceId": 60,
+ "url": "https://meionovel.id/",
+ "sourceName": "MeioNovel",
+ "icon": "https://github.com/LNReader/lnreader-sources/blob/main/multisrc/madara/icons/meionovel.png?raw=true",
+ "lang": "Indonesian"
+ },
+ {
+ "sourceId": 61,
+ "url": "https://www.webnovelover.com/",
+ "sourceName": "WebNovelLover",
+ "icon": "https://github.com/LNReader/lnreader-sources/blob/main/multisrc/madara/icons/webnovellover.png?raw=true",
+ "lang": "English"
+ },
+ {
+ "sourceId": 62,
+ "url": "https://www.wlnupdates.com/",
+ "sourceName": "WLNUpdates",
+ "icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/en/wlnupdates/icon.png?raw=true",
+ "lang": "English"
+ },
+ {
+ "sourceId": 63,
+ "url": "https://boxnovel.online/",
+ "sourceName": "BoxNovel.online",
+ "icon": "https://github.com/LNReader/lnreader-sources/blob/main/multisrc/madara/icons/boxnovel.online.png?raw=true",
+ "lang": "English"
+ },
+ {
+ "sourceId": 64,
+ "url": "https://clicknovel.net/",
+ "sourceName": "ClickNovel",
+ "icon": "https://github.com/LNReader/lnreader-sources/blob/main/multisrc/madara/icons/clicknovel.png?raw=true",
+ "lang": "English"
+ },
+ {
+ "sourceId": 65,
+ "url": "https://readwebnovels.net/",
+ "sourceName": "ReadWebNovels",
+ "icon": "https://github.com/LNReader/lnreader-sources/blob/main/multisrc/madara/icons/readwebnovels.png?raw=true",
+ "lang": "English"
+ },
+ {
+ "sourceId": 66,
+ "url": "https://wbnovel.com/",
+ "sourceName": "WBNovel",
+ "icon": "https://github.com/LNReader/lnreader-sources/blob/main/multisrc/madara/icons/wbnovel.png?raw=true",
+ "lang": "English"
+ },
+ {
+ "sourceId": 67,
+ "url": "https://reaperscans.com/all-series/novels/",
+ "sourceName": "ReaperScans",
+ "icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/en/reaperscans/icon.png?raw=true",
+ "lang": "English"
+ },
+ {
+ "sourceId": 68,
+ "url": "https://www.readwn.com/",
+ "sourceName": "Readwn.com",
+ "icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/en/readwn/icon.png?raw=true",
+ "lang": "English"
+ },
+ {
+ "sourceId": 69,
+ "url": "https://ranobehub.org/",
+ "sourceName": "Ranobehub",
+ "icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/ru/ranobehub/icon.png?raw=true",
+ "lang": "Russian"
+ },
+ {
+ "sourceId": 70,
+ "url": "https://divinedaolibrary.com/",
+ "sourceName": "Divine Dao Library",
+ "icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/en/divinedaolibrary/icon.png?raw=true",
+ "lang": "English"
+ },
+ {
+ "sourceId": 71,
+ "url": "https://novelonlinefull.com/",
+ "sourceName": "NovelOnlineFull.com",
+ "icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/en/novelonlinefull/icon.png?raw=true",
+ "lang": "English"
+ },
+ {
+ "sourceId": 72,
+ "url": "https://www.lightnovelupdates.com/",
+ "sourceName": "LightNovelUpdates",
+ "icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/en/lightnovelupdates/icon.png?raw=true",
+ "lang": "English"
+ },
+ {
+ "sourceId": 73,
+ "url": "https://www.epiknovel.com/",
+ "sourceName": "EpikNovel",
+ "icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/tr/epiknovel/icon.png?raw=true",
+ "lang": "Turkish"
+ },
+ {
+ "sourceId": 74,
+ "url": "https://wuxiaworld.site/",
+ "sourceName": "WuxiaWorld.Site",
+ "icon": "https://github.com/LNReader/lnreader-sources/blob/main/multisrc/madara/icons/wuxiaworldsite.png?raw=true",
+ "lang": "English"
+ },
+ {
+ "sourceId": 75,
+ "url": "https://mysticalmerries.com/",
+ "sourceName": "MysticalSeries",
+ "icon": "https://github.com/LNReader/lnreader-sources/blob/main/multisrc/madara/icons/mysticalseries.png?raw=true",
+ "lang": "English"
+ },
+ {
+ "sourceId": 76,
+ "url": "https://travistranslations.com/",
+ "sourceName": "TravisTranslations",
+ "icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/en/travistranslations/icon.png?raw=true",
+ "lang": "English"
+ },
+ {
+ "sourceId": 77,
+ "url": "https://noveldeglace.com/",
+ "sourceName": "NovelDeGlace",
+ "icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/fr/noveldeglace/icon.png?raw=true",
+ "lang": "French"
+ },
+ {
+ "sourceId": 78,
+ "url": "https://rainofsnow.com/",
+ "sourceName": "RainOfSnow",
+ "icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/en/rainofsnow/icon.png?raw=true",
+ "lang": "French"
+ }
+]
diff --git a/src/tr/epiknovel.js b/src/tr/epiknovel.js
new file mode 100644
index 0000000..2c22126
--- /dev/null
+++ b/src/tr/epiknovel.js
@@ -0,0 +1,170 @@
+import cheerio from "react-native-cheerio";
+
+const sourceId = 73;
+
+const sourceName = "EpikNovel";
+
+const baseUrl = "https://www.epiknovel.com/";
+
+const popularNovels = async (page) => {
+ let url = baseUrl + "seri-listesi?Sayfa=" + page;
+ const totalPages = 5;
+
+ const result = await fetch(url);
+ const body = await result.text();
+
+ $ = cheerio.load(body);
+
+ let novels = [];
+
+ $("div.col-lg-12.col-md-12").each(function () {
+ const novelName = $(this).find("h3").text();
+ const novelCover = $(this).find("img").attr("data-src");
+ const novelUrl = $(this).find("h3 > a").attr("href");
+
+ const novel = {
+ sourceId,
+ novelName,
+ novelCover,
+ novelUrl,
+ };
+
+ novels.push(novel);
+ });
+
+ console.log(novels);
+
+ return { novels, totalPages };
+};
+
+const parseNovelAndChapters = async (novelUrl) => {
+ const url = novelUrl;
+ console.log(url);
+
+ const result = await fetch(url);
+ const body = await result.text();
+
+ $ = cheerio.load(body);
+
+ let novel = { sourceId, sourceName, url, novelUrl };
+
+ novel.novelName = $("h1#tables").text().trim();
+
+ novel.novelCover = $("img.manga-cover").attr("src");
+
+ novel.summary = $(
+ "#wrapper > div.row > div.col-md-9 > div:nth-child(6) > p:nth-child(3)"
+ )
+ .text()
+ .trim();
+
+ novel.status = $(
+ "#wrapper > div.row > div.col-md-9 > div.row > div.col-md-9 > h4:nth-child(3) > a"
+ )
+ .text()
+ .trim();
+
+ novel.author = $("#NovelInfo > p:nth-child(4)")
+ .text()
+ .replace(/Publisher:|\s/g, "")
+ .trim();
+
+ let novelChapters = [];
+
+ $("table").find("tr").first().remove();
+
+ $("table")
+ .find("tr")
+ .each(function () {
+ const releaseDate = $(this).find("td:nth-child(3)").text();
+
+ let chapterName = $(this).find("td:nth-child(1) > a").text();
+
+ if ($(this).find("td:nth-child(1) > span").length > 0) {
+ chapterName = "🔒 " + chapterName;
+ }
+
+ const chapterUrl = $(this)
+ .find(" td:nth-child(1) > a")
+ .attr("href");
+
+ novelChapters.push({
+ chapterName,
+ chapterUrl,
+ releaseDate,
+ });
+ });
+
+ novel.chapters = novelChapters;
+ console.log(novel);
+
+ return novel;
+};
+
+const parseChapter = async (novelUrl, chapterUrl) => {
+ const url = chapterUrl;
+
+ console.log(url);
+
+ const result = await fetch(url);
+ const body = await result.text();
+
+ $ = cheerio.load(body);
+
+ let chapterName, chapterText;
+
+ if (result.url == "https://www.epiknovel.com/login") {
+ chapterName = "";
+ chapterText = "Premium Chapter";
+ } else {
+ chapterName = $("#icerik > center > h4 > b").text();
+ chapterText = $("div#icerik").html();
+ }
+
+ const chapter = {
+ sourceId,
+ novelUrl,
+ chapterUrl,
+ chapterName,
+ chapterText,
+ };
+
+ return chapter;
+};
+
+const searchNovels = async (searchTerm) => {
+ const url = baseUrl + "seri-listesi?q=" + searchTerm;
+
+ const result = await fetch(url);
+ const body = await result.text();
+
+ $ = cheerio.load(body);
+
+ let novels = [];
+
+ $("div.col-lg-12.col-md-12").each(function () {
+ const novelName = $(this).find("h3").text();
+ const novelCover = $(this).find("img").attr("data-src");
+ const novelUrl = $(this).find("h3 > a").attr("href");
+
+ const novel = {
+ sourceId,
+ novelName,
+ novelCover,
+ novelUrl,
+ };
+
+ novels.push(novel);
+ });
+
+ return novels;
+};
+
+const EpikNovelScraper = {
+ popularNovels,
+ parseNovelAndChapters,
+ parseChapter,
+ searchNovels,
+};
+
+export default EpikNovelScraper;
diff --git a/vercel.json b/vercel.json
deleted file mode 100644
index df47e78..0000000
--- a/vercel.json
+++ /dev/null
@@ -1,15 +0,0 @@
-{
- "version": 2,
- "builds": [
- {
- "src": "./index.js",
- "use": "@vercel/node"
- }
- ],
- "routes": [
- {
- "src": "/(.*)",
- "dest": "/"
- }
- ]
-}