diff --git a/index.js b/index.js index 00a89a6..d5b0fac 100644 --- a/index.js +++ b/index.js @@ -1,5 +1,5 @@ const express = require("express"); -const extensions = require("./src/extensions"); +const sources = require("./sources"); const app = express(); @@ -17,15 +17,15 @@ app.get("/", (req, res) => }) ); -app.get("/api/", (req, res) => res.json(extensions)); -app.use("/api/1/", require("./src/en/boxnovel/boxnovel")); -app.use("/api/2/", require("./src/en/readlightnovel/readlightnovel")); -app.use("/api/3/", require("./src/en/fastnovel/fastnovel")); -app.use("/api/4/", require("./src/en/readnovelfull/readnovelfull")); -app.use("/api/5/", require("./src/en/mtlnovel/mtlnovel")); -app.use("/api/6/", require("./src/en/novelhall/novelhall")); -app.use("/api/7/", require("./src/en/wuxiaworld/wuxiaworld")); -app.use("/api/8/", require("./src/en/novelfull/novelfull")); +app.get("/api/", (req, res) => res.json(sources)); +app.use("/api/1/", require("./src/en/boxnovel/BoxNovel")); +app.use("/api/2/", require("./src/en/readlightnovel/ReadLightNovel")); +app.use("/api/3/", require("./src/en/fastnovel/FastNovel")); +app.use("/api/4/", require("./src/en/readnovelfull/ReadNovelFull")); +app.use("/api/5/", require("./src/en/mtlnovel/MTLNovel")); +app.use("/api/6/", require("./src/en/novelhall/Novelhall")); +app.use("/api/7/", require("./src/en/wuxiaworld/WuxiaWorld")); +app.use("/api/8/", require("./src/en/novelfull/NovelFull")); const PORT = process.env.PORT || 5000; diff --git a/src/extensions.js b/sources.js similarity index 100% rename from src/extensions.js rename to sources.js diff --git a/src/en/boxnovel/BoxNovel.js b/src/en/boxnovel/BoxNovel.js new file mode 100644 index 0000000..c741661 --- /dev/null +++ b/src/en/boxnovel/BoxNovel.js @@ -0,0 +1,14 @@ +const express = require("express"); +const boxNovelScraper = require("./BoxNovelScraper"); + +const router = express.Router(); + +router.get("/novels/:pageNo", boxNovelScraper.novelsScraper); + +router.get("/novel/:novelUrl", boxNovelScraper.novelScraper); + +router.get("/novel/:novelUrl/:chapterUrl", boxNovelScraper.chapterScraper); + +router.get("/search/", boxNovelScraper.searchScraper); + +module.exports = router; diff --git a/src/en/boxnovel/BoxNovelScraper.js b/src/en/boxnovel/BoxNovelScraper.js new file mode 100644 index 0000000..e866e05 --- /dev/null +++ b/src/en/boxnovel/BoxNovelScraper.js @@ -0,0 +1,202 @@ +const cheerio = require("cheerio"); +const fetch = require("node-fetch"); + +const baseUrl = "https://boxnovel.com/novel"; +const searchUrl = "https://boxnovel.com/"; + +const novelsScraper = async (req, res) => { + const orderBy = req.params.o; + const pageNo = req.params.pageNo; + + let url = `${baseUrl}/page/1/?m_orderby=${orderBy}`; + + const result = await fetch(url); + const body = await result.text(); + + $ = cheerio.load(body); + + let novels = []; + + $(".page-item-detail").each(function (result) { + const novelName = $(this).find("h5 > a").text(); + const novelCover = $(this).find("img").attr("src"); + + let novelUrl = $(this).find("h5 > a").attr("href"); + novelUrl = novelUrl.replace(`${baseUrl}/`, ""); + + const novel = { + extensionId: 1, + novelName, + novelCover, + novelUrl, + }; + + novels.push(novel); + }); + + res.json(novels); +}; + +const novelScraper = async (req, res) => { + const novelUrl = req.params.novelUrl; + const url = `${baseUrl}/${novelUrl}/`; + + console.log(url); + + const result = await fetch(url); + const body = await result.text(); + + $ = cheerio.load(body); + + let novel = {}; + + $(".post-title > h3 > span").remove(); + + novel.extensionId = 1; + + novel.sourceName = "BoxNovel"; + + novel.sourceUrl = url; + + novel.novelUrl = `${novelUrl}/`; + + novel.novelName = $(".post-title > h3") + .text() + .replace(/[\t\n]/g, "") + .trim(); + + novel.novelCover = $(".summary_image > a > img").attr("src"); + + $(".post-content_item").each(function (result) { + detailName = $(this) + .find(".summary-heading > h5") + .text() + .replace(/[\t\n]/g, "") + .trim(); + detail = $(this) + .find(".summary-content") + .text() + .replace(/[\t\n]/g, "") + .trim(); + + novel[detailName] = detail; + }); + + $(".description-summary > div.summary__content").find("em").remove(); + + novel.novelSummary = $(".description-summary > div.summary__content > div") + .text() + .replace(/[\t\n]/g, ""); + + let novelChapters = []; + + $(".wp-manga-chapter").each(function (result) { + chapterName = $(this) + .find("a") + .text() + .replace(/[\t\n]/g, "") + .trim(); + + releaseDate = $(this) + .find("span") + .text() + .replace(/[\t\n]/g, "") + .trim(); + + chapterUrl = $(this).find("a").attr("href").replace(url, ""); + + novelChapters.push({ chapterName, releaseDate, chapterUrl }); + }); + + novel.novelChapters = novelChapters.reverse(); + + res.json(novel); +}; + +const chapterScraper = async (req, res) => { + let novelUrl = req.params.novelUrl; + let chapterUrl = req.params.chapterUrl; + + const url = `${baseUrl}/${novelUrl}/${chapterUrl}`; + + const result = await fetch(url); + const body = await result.text(); + + $ = cheerio.load(body); + + const chapterName = $("div.text-left > h3").text(); + const chapterText = $(".reading-content").text(); + + let nextChapter = null; + + if ($(".nav-next").length) { + nextChapter = $(".nav-next") + .find("a") + .attr("href") + .replace(baseUrl + "/" + novelUrl + "/", ""); + } + + let prevChapter = null; + + if ($(".nav-previous").length) { + prevChapter = $(".nav-previous") + .find("a") + .attr("href") + .replace(baseUrl + "/" + novelUrl + "/", ""); + } + + novelUrl = novelUrl + "/"; + chapterUrl = chapterUrl + "/"; + + const chapter = { + extensionId: 1, + novelUrl, + chapterUrl, + chapterName, + chapterText, + nextChapter, + prevChapter, + }; + + res.json(chapter); +}; + +const searchScraper = async (req, res) => { + const searchTerm = req.query.s; + const orderBy = req.query.o; + + const url = `${searchUrl}?s=${searchTerm}&post_type=wp-manga&m_orderby=${orderBy}`; + + const result = await fetch(url); + const body = await result.text(); + + $ = cheerio.load(body); + + let novels = []; + + $(".c-tabs-item__content").each(function (result) { + const novelName = $(this).find("h4 > a").text(); + const novelCover = $(this).find("img").attr("src"); + + let novelUrl = $(this).find("h4 > a").attr("href"); + novelUrl = novelUrlreplace(`${baseUrl}/`, ""); + + const novel = { + extensionId: 1, + novelName, + novelCover, + novelUrl, + }; + + novels.push(novel); + }); + + res.json(novels); +}; + +module.exports = boxNovelScraper = { + novelsScraper, + novelScraper, + chapterScraper, + searchScraper, +}; diff --git a/src/en/boxnovel/boxnovel.js b/src/en/boxnovel/boxnovel.js deleted file mode 100644 index 295605f..0000000 --- a/src/en/boxnovel/boxnovel.js +++ /dev/null @@ -1,247 +0,0 @@ -const express = require("express"); -const cheerio = require("cheerio"); -const request = require("request"); - -const router = express.Router(); - -const baseUrl = "https://boxnovel.com/novel"; - -const searchUrl = "https://boxnovel.com/"; - -// Top novels - -function apiCall(reqOps) { - return new Promise(function (resolve, reject) { - request(reqOps, (err, res, body) => { - if (!err && res.statusCode == 200) { - if (err) throw err; - - let novels = []; - - $ = cheerio.load(body); - - $(".page-item-detail").each(function (result) { - novelName = $(this).find("h5 > a").text(); - novelCover = $(this).find("img").attr("src"); - novelUrl = $(this) - .find("h5 > a") - .attr("href") - .replace(`${baseUrl}/`, ""); - novel = { - extensionId: 1, - novelName: novelName, - novelCover: novelCover, - novelUrl: novelUrl, - }; - - novels.push(novel); - }); - resolve(novels); - } - - reject(err); - }); - }); -} - -router.get("/novels/:pageNo/", (req, res) => { - orderBy = req.query.o; - pageNo = req.params.pageNo; - - let url = `${baseUrl}/page/1/?m_orderby=${orderBy}`; - - let page1, page2, page3, page4; - - apiCall(url) - .then((res) => { - page1 = res; - return apiCall(`${baseUrl}/page/2/?m_orderby=${orderBy}`); - }) - .then((res) => { - page2 = res; - return apiCall(`${baseUrl}/page/3/?m_orderby=${orderBy}`); - }) - .then((res) => { - page3 = res; - return apiCall(`${baseUrl}/page/4/?m_orderby=${orderBy}`); - }) - .then((result) => { - page4 = result; - - res.json([...page1, ...page2, ...page3, ...page4]); - }) - .catch((err) => { - console.log("Error occured in one of the API call: ", err); - }); -}); - -// Novel - -router.get("/novel/:novelUrl", (req, res) => { - novelUrl = req.params.novelUrl; - url = `${baseUrl}/${novelUrl}/`; - - request(url, (err, response, body) => { - if (err) throw err; - - $ = cheerio.load(body); - - let novel = {}; - - $(".post-title > h3 > span").remove(); - - novel.extensionId = 1; - - novel.sourceName = "BoxNovel"; - - novel.sourceUrl = url; - - novel.novelUrl = `${novelUrl}/`; - - novel.novelName = $(".post-title > h3") - .text() - .replace(/[\t\n]/g, "") - .trim(); - - novel.novelCover = $(".summary_image > a > img").attr("src"); - - $(".post-content_item").each(function (result) { - detailName = $(this) - .find(".summary-heading > h5") - .text() - .replace(/[\t\n]/g, "") - .trim(); - detail = $(this) - .find(".summary-content") - .text() - .replace(/[\t\n]/g, "") - .trim(); - - novel[detailName] = detail; - }); - - $(".description-summary > div.summary__content").find("em").remove(); - - novel.novelSummary = $( - ".description-summary > div.summary__content > div" - ) - .text() - .replace(/[\t\n]/g, ""); - - // if (novel.novelSummary === "") { - // novel.novelSummary = $(".c_000").text(); - // } - - let novelChapters = []; - - $(".wp-manga-chapter").each(function (result) { - chapterName = $(this) - .find("a") - .text() - .replace(/[\t\n]/g, "") - .trim(); - - releaseDate = $(this) - .find("span") - .text() - .replace(/[\t\n]/g, "") - .trim(); - - chapterUrl = $(this).find("a").attr("href").replace(url, ""); - - novelChapters.push({ chapterName, releaseDate, chapterUrl }); - }); - - novel.novelChapters = novelChapters.reverse(); - - res.json(novel); - }); -}); - -// Chapter - -router.get("/novel/:novelUrl/:chapterUrl", (req, res) => { - url = `${baseUrl}/${req.params.novelUrl}/${req.params.chapterUrl}`; - - request(url, (err, response, body) => { - if (err) throw err; - - $ = cheerio.load(body); - - chapterNameLower = req.params.chapterUrl.replace("-", " "); - - chapterName = - chapterNameLower.charAt(0).toUpperCase() + - chapterNameLower.slice(1); - - chapterText = $(".reading-content").text(); - - let nextChapter = null; - - if ($(".nav-next").length) { - nextChapter = $(".nav-next") - .find("a") - .attr("href") - .replace(baseUrl + "/" + req.params.novelUrl + "/", ""); - } - - let prevChapter = null; - - if ($(".nav-previous").length) { - prevChapter = $(".nav-previous") - .find("a") - .attr("href") - .replace(baseUrl + "/" + req.params.novelUrl + "/", ""); - } - - chapter = { - extensionId: 1, - novelUrl: `${req.params.novelUrl}/`, - chapterUrl: `${req.params.chapterUrl}/`, - chapterName, - chapterText, - nextChapter, - prevChapter, - }; - - res.json(chapter); - }); -}); - -// Search - -router.get("/search/", (req, res) => { - searchTerm = req.query.s; - orderBy = req.query.o; - - url = `${searchUrl}?s=${searchTerm}&post_type=wp-manga&m_orderby=${orderBy}`; - - request(url, (err, response, body) => { - if (err) throw err; - - $ = cheerio.load(body); - - let novels = []; - - $(".c-tabs-item__content").each(function (result) { - novelName = $(this).find("h4 > a").text(); - novelCover = $(this).find("img").attr("src"); - - novelUrl = $(this) - .find("h4 > a") - .attr("href") - .replace(`${baseUrl}/`, ""); - - novels.push({ - extensionId: 1, - novelName, - novelCover, - novelUrl, - }); - }); - - res.json(novels); - }); -}); - -module.exports = router; diff --git a/src/en/fastnovel/FastNovel.js b/src/en/fastnovel/FastNovel.js new file mode 100644 index 0000000..0d4c725 --- /dev/null +++ b/src/en/fastnovel/FastNovel.js @@ -0,0 +1,14 @@ +const express = require("express"); +const fastNovelScraper = require("./FastNovelScraper"); + +const router = express.Router(); + +router.get("/novels/", fastNovelScraper.novelsScraper); + +router.get("/novel/:novelUrl", fastNovelScraper.novelScraper); + +router.get("/novel/:novelUrl/:chapterUrl", fastNovelScraper.chapterScraper); + +router.get("/search/", fastNovelScraper.searchScraper); + +module.exports = router; diff --git a/src/en/fastnovel/FastNovelScraper.js b/src/en/fastnovel/FastNovelScraper.js new file mode 100644 index 0000000..7f28e4f --- /dev/null +++ b/src/en/fastnovel/FastNovelScraper.js @@ -0,0 +1,179 @@ +const cheerio = require("cheerio"); +const { scraper } = require("../../helper"); + +const baseUrl = "https://fastnovel.net"; +const searchUrl = `https://fastnovel.net/search/`; + +const novelsScraper = async (req, res) => { + const url = `${baseUrl}/list/most-popular.html`; + + const body = await scraper(url); + + $ = cheerio.load(body); + + let novels = []; + + $(".film-item").each(function (result) { + const novelName = $(this).find(".name").text(); + const novelCover = $(this).find(".img").attr("data-original"); + const novelUrl = $(this).find("a").attr("href").substring(1); + + const novel = { + extensionId: 3, + novelName, + novelCover, + novelUrl, + }; + + novels.push(novel); + }); + + res.json(novels); +}; + +const novelScraper = async (req, res) => { + const novelUrl = req.params.novelUrl; + const url = `${baseUrl}/${novelUrl}/`; + + const body = await scraper(url); + + $ = cheerio.load(body); + + let novel = {}; + + novel.extensionId = 3; + + novel.sourceName = "FastNovel"; + + novel.sourceUrl = url; + + novel.novelUrl = novelUrl + "/"; + + novel.novelName = $("h1").text(); + + novel.novelCover = $(".book-cover").attr("data-original"); + + novel.novelSummary = $("div.film-content > p").text(); + + novel["Author(s)"] = $("li > label") + .filter(function () { + return $(this).text().trim() === "Author:"; + }) + .next() + .text(); + + novel["Genre(s)"] = $("li") + .filter(function () { + return $(this).find("label").text().trim() === "Genre:"; + }) + .text() + .replace("Genre:", ""); + + novel["Artist(s)"] = null; + + novel.Status = null; + + let novelChapters = []; + + $(".chapter").each(function (result) { + const chapterName = $(this).text(); + const releaseDate = null; + let chapterUrl = $(this).attr("href"); + chapterUrl = chapterUrl.replace(`/${novelUrl}/`, ""); + + const novel = { + chapterName, + releaseDate, + chapterUrl, + }; + + novelChapters.push(novel); + }); + + novel.novelChapters = novelChapters; + + res.json(novel); +}; + +const chapterScraper = async (req, res) => { + let novelUrl = req.params.novelUrl; + let chapterUrl = req.params.chapterUrl; + + const url = `${baseUrl}/${novelUrl}/${chapterUrl}`; + + const body = await scraper(url); + + $ = cheerio.load(body); + + const chapterName = $(".episode-name").text(); + const chapterText = $("#chapter-body").text(); + + let nextChapter = null; + + nextChapter = $("div.btn-group > a") + .filter(function () { + return $(this).text().trim() === "Next Chapter"; + }) + .attr("href") + .replace("/", ""); + + let prevChapter = null; + + prevChapter = $("div.btn-group > a") + .filter(function () { + return $(this).text().trim() === "Prev Chapter"; + }) + .attr("href") + .replace("/", ""); + + novelUrl = novelUrl + "/"; + chapterUrl = chapterUrl + "/"; + + const chapter = { + extensionId: 3, + novelUrl, + chapterUrl, + chapterName, + chapterText, + nextChapter, + prevChapter, + }; + + res.json(chapter); +}; + +const searchScraper = async (req, res) => { + const searchTerm = req.query.s; + + const url = `${searchUrl}${searchTerm}`; + + const body = await scraper(url); + + $ = cheerio.load(body); + + let novels = []; + + $(".film-item").each(function (result) { + const novelName = $(this).find("div.title > p.name").text(); + const novelCover = $(this).find(".img").attr("data-original"); + const novelUrl = $(this).find("a").attr("href").substring(1); + + const novel = { + extensionId: 3, + novelName, + novelCover, + novelUrl, + }; + + novels.push(novel); + }); + + res.json(novels); +}; + +module.exports = fastNovelScraper = { + novelsScraper, + novelScraper, + chapterScraper, + searchScraper, +}; diff --git a/src/en/fastnovel/fastnovel.js b/src/en/fastnovel/fastnovel.js deleted file mode 100644 index dadbe66..0000000 --- a/src/en/fastnovel/fastnovel.js +++ /dev/null @@ -1,182 +0,0 @@ -const express = require("express"); -const cheerio = require("cheerio"); -const request = require("request"); - -const baseUrl = "https://fastnovel.net"; - -const router = express.Router(); -router.get("/novels/", (req, res) => { - url = `https://fastnovel.net/list/most-popular.html`; - - request(url, (err, response, body) => { - if (err) throw err; - - let novels = []; - - $ = cheerio.load(body); - - $(".film-item").each(function (result) { - let novelName = $(this).find("div.title > p.name").text(); - let novelCover = $(this).find(".img").attr("data-original"); - let novelUrl = $(this).find("a").attr("href").substring(1); - - novel = { - extensionId: 3, - novelName, - novelCover, - novelUrl, - }; - - novels.push(novel); - }); - - res.json(novels); - }); -}); - -// Novel - -router.get("/novel/:novelUrl", (req, res) => { - novelUrl = req.params.novelUrl; - url = `${baseUrl}/${novelUrl}/`; - - request(url, (err, response, body) => { - if (err) throw err; - - $ = cheerio.load(body); - - let novel = {}; - - novel.extensionId = 3; - - novel.sourceName = "FastNovel"; - - novel.sourceUrl = url; - - novel.novelUrl = `${novelUrl}/`; - - novel.novelName = $("h1").text(); - - novel.novelCover = $(".book-cover").attr("data-original"); - - novel.novelSummary = $("div.film-content > p").text(); - - novel["Author(s)"] = $("li > label") - .filter(function () { - return $(this).text().trim() === "Author:"; - }) - .next() - .text(); - - novel["Genre(s)"] = $("li") - .filter(function () { - return $(this).find("label").text().trim() === "Genre:"; - }) - .text() - .replace("Genre:", ""); - - novel["Artist(s)"] = null; - - novel.Status = null; - - let novelChapters = []; - - $(".chapter").each(function (result) { - let chapterName = $(this).text(); - - let releaseDate = null; - - let chapterUrl = $(this).attr("href"); - chapterUrl = chapterUrl.replace(`/${novelUrl}/`, ""); - - novelChapters.push({ - chapterName, - releaseDate, - chapterUrl, - }); - }); - - novel.novelChapters = novelChapters; - - res.json(novel); - }); -}); - -router.get("/novel/:novelUrl/:chapterUrl", (req, res) => { - url = `${baseUrl}/${req.params.novelUrl}/${req.params.chapterUrl}`; - - request(url, (err, response, body) => { - if (err) throw err; - - $ = cheerio.load(body); - - chapterName = $(".episode-name").text(); - - chapterText = $("#chapter-body").text(); - - let nextChapter = null; - - nextChapter = $("div.btn-group > a") - .filter(function () { - return $(this).text().trim() === "Next Chapter"; - }) - .attr("href") - .replace("/", ""); - - let prevChapter = null; - - prevChapter = $("div.btn-group > a") - .filter(function () { - return $(this).text().trim() === "Prev Chapter"; - }) - .attr("href") - .replace("/", ""); - - chapter = { - extensionId: 3, - novelUrl: `${req.params.novelUrl}/`, - chapterUrl: `${req.params.chapterUrl}/`, - chapterName, - chapterText, - nextChapter, - prevChapter, - }; - - res.json(chapter); - }); -}); - -router.get("/search/", (req, res) => { - let searchTerm = req.query.s; - searchTerm = encodeURI(searchTerm); - searchUrl = `https://fastnovel.net/search/`; - - url = `${searchUrl}${searchTerm}`; - - request(url, (err, response, body) => { - if (err) throw err; - - $ = cheerio.load(body); - - let novels = []; - - $(".film-item").each(function (result) { - let novelName = $(this).find("div.title > p.name").text(); - let novelCover = $(this).find(".img").attr("data-original"); - let novelUrl = $(this).find("a").attr("href").substring(1); - - novel = { - extensionId: 3, - novelName, - novelCover, - novelUrl, - }; - - novels.push(novel); - }); - - res.json(novels); - }); -}); - -module.exports = router; diff --git a/src/en/mtlnovel/MTLNovel.js b/src/en/mtlnovel/MTLNovel.js new file mode 100644 index 0000000..241823c --- /dev/null +++ b/src/en/mtlnovel/MTLNovel.js @@ -0,0 +1,14 @@ +const express = require("express"); +const mtlNovelScraper = require("./MTLNovelScraper"); + +const router = express.Router(); + +router.get("/novels/", mtlNovelScraper.novelsScraper); + +router.get("/novel/:novelUrl", mtlNovelScraper.novelScraper); + +router.get("/novel/:novelUrl/:chapterUrl", mtlNovelScraper.chapterScraper); + +router.get("/search/", mtlNovelScraper.searchScraper); + +module.exports = router; diff --git a/src/en/mtlnovel/MTLNovelScraper.js b/src/en/mtlnovel/MTLNovelScraper.js new file mode 100644 index 0000000..6672fb9 --- /dev/null +++ b/src/en/mtlnovel/MTLNovelScraper.js @@ -0,0 +1,196 @@ +const cheerio = require("cheerio"); +const fetch = require("node-fetch"); +const { scraper } = require("../../helper"); + +const baseUrl = "https://mtlnovel.com"; + +const novelsScraper = async (req, res) => { + const url = `${baseUrl}/alltime-rank/`; + + const body = await scraper(url); + + $ = cheerio.load(body); + + let novels = []; + + $("div.box.wide").each(function (result) { + const novelName = $(this).find("a.list-title").text().slice(4); + const novelCover = $(this).find("amp-img").attr("src"); + + let novelUrl = $(this).find("a.list-title").attr("href"); + novelUrl = novelUrl.replace("https://www.mtlnovel.com/", ""); + + const novel = { + extensionId: 5, + novelUrl, + novelName, + novelCover, + }; + + novels.push(novel); + }); + + res.json(novels); +}; + +const novelScraper = async (req, res) => { + const novelUrl = req.params.novelUrl; + const url = `${baseUrl}/${novelUrl}`; + + const body = await scraper(url); + + $ = cheerio.load(body); + + let novel = {}; + + novel.extensionId = 5; + + novel.sourceName = "MTLNovel"; + + novel.sourceUrl = url; + + novel.novelUrl = novelUrl + "/"; + + novel.novelName = $("h1.entry-title").text(); + + novel.novelCover = $("amp-img").attr("src"); + + novel.novelSummary = $("div.desc > h2").next().text(); + + novel["Author(s)"] = $("tr > td") + .filter(function () { + return $(this).prev().text().trim() === "Author"; + }) + .next() + .text() + .replace("Auhtor:", ""); + + novel.Status = $("tr > td") + .filter(function () { + return $(this).prev().text().trim() === "Status"; + }) + .next() + .text() + .replace("Status:", ""); + + novel["Genre(s)"] = $("td") + .filter(function () { + return $(this).prev().text().trim() === "Genre"; + }) + .next() + .text() + .replace("Genre:", ""); + + novel["Artist(s)"] = null; + + const chapterListUrl = url + "/chapter-list/"; + + const getChapters = async () => { + const body = await scraper(chapterListUrl); + + $ = cheerio.load(body); + + let novelChapters = []; + + $("div.ch-list") + .find("a.ch-link") + .each(function (result) { + const chapterName = $(this).text().replace("~ ", ""); + const releaseDate = null; + + let chapterUrl = $(this).attr("href"); + chapterUrl = chapterUrl.replace( + `https://www.mtlnovel.com/${novelUrl}/`, + "" + ); + + novelChapters.push({ + chapterUrl, + chapterName, + releaseDate, + }); + }); + return novelChapters; + }; + + novel.novelChapters = await getChapters(); + + res.json(novel); +}; + +const chapterScraper = async (req, res) => { + let novelUrl = req.params.novelUrl; + let chapterUrl = req.params.chapterUrl; + + const url = `${baseUrl}/${novelUrl}/${chapterUrl}/`; + + const result = await fetch(url); + const body = await result.text(); + + $ = cheerio.load(body); + + const chapterName = $("h1.main-title").text(); + const chapterText = $("div.par").text(); + + let nextChapter = null; + if ($("a.next").attr("href")) { + nextChapter = $("a.next").attr("href").replace(`/${novelUrl}/`, ""); + } + + let prevChapter = null; + if ($("a.prev").attr("href")) { + prevChapter = $("a.prev").attr("href").replace(`/${novelUrl}/`, ""); + } + + chapter = { + extensionId: 5, + novelUrl, + chapterUrl, + chapterName, + chapterText, + nextChapter, + prevChapter, + }; + + res.json(chapter); +}; + +const searchScraper = async (req, res) => { + const searchTerm = req.query.s; + + const result = await fetch(baseUrl, { + method: "POST", + body: { s: searchTerm }, + }); + const body = await result.text(); + + $ = cheerio.load(body); + + let novels = []; + + $("div.box").each(function (result) { + const novelName = $(this).find("a.list-title").text(); + const novelCover = $(this).find("amp-img").attr("src"); + + let novelUrl = $(this).find("a").attr("href"); + novelUrl = novelUrl.replace("https://www.mtlnovel.com/", ""); + + const novel = { + extensionId: 5, + novelUrl, + novelName, + novelCover, + }; + + novels.push(novel); + }); + + res.json(novels); +}; + +module.exports = mtlNovelScraper = { + novelsScraper, + novelScraper, + chapterScraper, + searchScraper, +}; diff --git a/src/en/mtlnovel/mtlnovel.js b/src/en/mtlnovel/mtlnovel.js deleted file mode 100644 index eb3ec78..0000000 --- a/src/en/mtlnovel/mtlnovel.js +++ /dev/null @@ -1,202 +0,0 @@ -const express = require("express"); -const cheerio = require("cheerio"); -const request = require("request"); - -const router = express.Router(); - -const baseUrl = "https://mtlnovel.com"; - -// Top novels - -router.get("/novels/", (req, res) => { - const url = `${baseUrl}/alltime-rank/`; - - request(url, (err, response, body) => { - if (err) throw err; - - let novels = []; - - $ = cheerio.load(body); - - $("div.box.wide").each(function (result) { - let novelUrl = $(this).find("a.list-title").attr("href"); - novelUrl = novelUrl.replace("https://www.mtlnovel.com/", ""); - - let novelName = $(this).find("a.list-title").text().slice(4); - let novelCover = $(this).find("amp-img").attr("src"); - - novel = { - extensionId: 5, - novelUrl, - novelName, - novelCover, - }; - - novels.push(novel); - }); - - res.json(novels); - }); -}); - -router.get("/novel/:novelUrl", (req, res) => { - let novelUrl = req.params.novelUrl; - const url = `${baseUrl}/${novelUrl}`; - console.log(url); - - request(url, (err, response, body) => { - $ = cheerio.load(body); - - let novel = {}; - - novel.extensionId = 5; - - novel.sourceName = "MTLNovel"; - - novel.sourceUrl = url; - - novel.novelUrl = `${novelUrl}/`; - - novel.novelName = $("h1.entry-title").text(); - - novel.novelCover = $("amp-img").attr("src"); - - novel.novelSummary = $("div.desc > h2").next().text(); - - novel["Author(s)"] = $("tr > td") - .filter(function () { - return $(this).prev().text().trim() === "Author"; - }) - .next() - .text() - .replace("Auhtor:", ""); - - novel.Status = $("tr > td") - .filter(function () { - return $(this).prev().text().trim() === "Status"; - }) - .next() - .text() - .replace("Status:", ""); - - novel["Genre(s)"] = $("td") - .filter(function () { - return $(this).prev().text().trim() === "Genre"; - }) - .next() - .text() - .replace("Genre:", ""); - - novel["Artist(s)"] = null; - - function callback(res) { - console.log(res); - novel.novelChapters = res; - } - - request(`${url}/chapter-list/`, (err, response, body) => { - $ = cheerio.load(body); - console.log(`${url}/chapter-list/`); - // console.log($("div.ch-list").find("a.ch-link").next().html()); - let novelChapters = []; - - $("div.ch-list") - .find("a.ch-link") - .each(function (result) { - let chapterName = $(this).text().replace("~ ", ""); - - let releaseDate = null; - - let chapterUrl = $(this).attr("href"); - chapterUrl = chapterUrl.replace( - `https://www.mtlnovel.com/${novelUrl}/`, - "" - ); - - novelChapters.push({ - chapterUrl, - chapterName, - releaseDate, - }); - }); - - callback(novelChapters); - - return res.json(novel); - }); - }); -}); - -router.get("/novel/:novelUrl/:chapterUrl", (req, res) => { - let novelUrl = req.params.novelUrl; - const url = `https://www.mtlnovel.com/${novelUrl}/${req.params.chapterUrl}/`; - console.log(url); - request(url, (err, response, body) => { - if (err) throw err; - - $ = cheerio.load(body); - - let chapterName = $("h1.main-title").text(); - - let chapterText = $("div.par").text(); - - let nextChapter = null; - if ($("a.next").attr("href")) { - nextChapter = $("a.next").attr("href").replace(`/${novelUrl}/`, ""); - } - - let prevChapter = null; - if ($("a.prev").attr("href")) { - prevChapter = $("a.prev").attr("href").replace(`/${novelUrl}/`, ""); - } - - chapter = { - extensionId: 5, - novelUrl: `${req.params.novelUrl}`, - chapterUrl: `${req.params.chapterUrl}`, - chapterName, - chapterText, - nextChapter, - prevChapter, - }; - - res.json(chapter); - }); -}); - -router.get("/search/", (req, res) => { - searchTerm = req.query.s; - - const searchUrl = "https://www.mtlnovel.com"; - - request.post( - { url: searchUrl, form: { s: searchTerm } }, - (err, response, body) => { - if (err) throw err; - - $ = cheerio.load(body); - console.log(body); - let novels = []; - - $("div.box").each(function (result) { - let novelName = $(this).find("a.list-title").text(); - let novelCover = $(this).find("amp-img").attr("src"); - let novelUrl = $(this).find("a").attr("href"); - novelUrl = novelUrl.replace("https://www.mtlnovel.com/", ""); - - novel = { - extensionId: 5, - novelUrl, - novelName, - novelCover, - }; - - novels.push(novel); - }); - - res.json(novels); - } - ); -}); - -module.exports = router; diff --git a/src/en/novelfull/NovelFull.js b/src/en/novelfull/NovelFull.js new file mode 100644 index 0000000..949ab32 --- /dev/null +++ b/src/en/novelfull/NovelFull.js @@ -0,0 +1,13 @@ +const express = require("express"); +const novelFullScraper = require("./NovelFullScraper"); +const router = express.Router(); + +router.get("/novels/", novelFullScraper.novelsScraper); + +router.get("/novel/:novelUrl", novelFullScraper.novelScraper); + +router.get("/novel/:novelUrl/:chapterUrl", novelFullScraper.chapterScraper); + +router.get("/search/", novelFullScraper.searchScraper); + +module.exports = router; diff --git a/src/en/novelfull/NovelFullScraper.js b/src/en/novelfull/NovelFullScraper.js new file mode 100644 index 0000000..3178fcc --- /dev/null +++ b/src/en/novelfull/NovelFullScraper.js @@ -0,0 +1,206 @@ +const cheerio = require("cheerio"); +const fetch = require("node-fetch"); + +const baseUrl = "https://novelfull.com"; + +const novelsScraper = async (req, res) => { + const url = `${baseUrl}/most-popular`; + + const result = await fetch(url); + const body = await result.text(); + + $ = cheerio.load(body); + + let novels = []; + + $("div.col-truyen-main > div.list-truyen > .row").each(function (result) { + const novelName = $(this).find("h3.truyen-title > a").text(); + + let novelCover = $(this).find("img").attr("src"); + novelCover = baseUrl + novelCover; + + let novelUrl = $(this) + .find("h3.truyen-title > a") + .attr("href") + .replace(".html", "") + .substring(1); + novelUrl = `${novelUrl}/`; + + const novel = { + extensionId: 8, + novelUrl, + novelName, + novelCover, + }; + + novels.push(novel); + }); + + res.json(novels); +}; + +const novelScraper = async (req, res) => { + const novelUrl = req.params.novelUrl; + const url = `${baseUrl}/${novelUrl}.html`; + + const result = await fetch(url); + const body = await result.text(); + + $ = cheerio.load(body); + + let novel = {}; + + novel.extensionId = 8; + + novel.sourceName = "NovelFull"; + + novel.sourceUrl = url; + + novel.novelUrl = `${novelUrl}/`; + + novel.novelName = $("div.book > img").attr("alt"); + + novel.novelCover = baseUrl + $("div.book > img").attr("src"); + + novel.novelSummary = $("div.desc-text").text(); + + novel["Author(s)"] = $("div.info > div > h3") + .filter(function () { + return $(this).text().trim() === "Author:"; + }) + .siblings() + .text(); + + novel["Genre(s)"] = $("div.info > div") + .filter(function () { + return $(this).find("h3").text().trim() === "Genre:"; + }) + .text() + .replace("Genre:", ""); + + novel["Artist(s)"] = null; + + novel.Status = $("div.info > div > h3") + .filter(function () { + return $(this).text().trim() === "Status:"; + }) + .next() + .text(); + + const novelId = $("#rating").attr("data-novel-id"); + + const getChapters = async (novelId) => { + const chapterListUrl = + baseUrl + "/ajax/chapter-option?novelId=" + novelId; + + const data = await fetch(chapterListUrl); + const chapters = await data.text(); + + $ = cheerio.load(chapters); + + let novelChapters = []; + + $("select > option").each(function (result) { + let chapterName = $(this).text(); + let releaseDate = null; + let chapterUrl = $(this).attr("value"); + chapterUrl = chapterUrl.replace(`/${novelUrl}/`, ""); + + novelChapters.push({ + chapterName, + releaseDate, + chapterUrl, + }); + }); + return novelChapters; + }; + + novel.novelChapters = await getChapters(novelId); + + res.json(novel); +}; + +const chapterScraper = async (req, res) => { + const novelUrl = req.params.novelUrl.replace(".html/", ""); + const chapterUrl = req.params.chapterUrl; + + const url = `${baseUrl}/${novelUrl}/${chapterUrl}`; + + const result = await fetch(url); + const body = await result.text(); + + $ = cheerio.load(body); + + const chapterName = $(".chapter-title").attr("title"); + const chapterText = $("#chapter-content").text(); + + let nextChapter = null; + if ($("a#next_chap").attr("href")) { + nextChapter = $("a#next_chap") + .attr("href") + .replace(`/${novelUrl}/`, ""); + } + + let prevChapter = null; + if ($("a#prev_chap").attr("href")) { + prevChapter = $("a#prev_chap") + .attr("href") + .replace(`/${novelUrl}/`, ""); + } + + const chapter = { + extensionId: 8, + novelUrl, + chapterUrl, + chapterName, + chapterText, + nextChapter, + prevChapter, + }; + + res.json(chapter); +}; + +const searchScraper = async (req, res) => { + const searchTerm = req.query.s; + const searchUrl = `https://novelfull.com/search?keyword=`; + + const url = `${searchUrl}${searchTerm}`; + + const result = await fetch(url); + const body = await result.text(); + + $ = cheerio.load(body); + + let novels = []; + + $("div.col-truyen-main > div.list-truyen > .row").each(function (result) { + let novelUrl = $(this) + .find("h3.truyen-title > a") + .attr("href") + .replace(".html", "") + .substring(1); + novelUrl = `${novelUrl}/`; + + const novelName = $(this).find("h3.truyen-title > a").text(); + const novelCover = $(this).find("img").attr("src"); + + const novel = { + extensionId: 8, + novelUrl, + novelName, + novelCover, + }; + + novels.push(novel); + }); + + res.json(novels); +}; + +module.exports = novelFullScraper = { + novelsScraper, + novelScraper, + chapterScraper, + searchScraper, +}; diff --git a/src/en/novelfull/novelfull.js b/src/en/novelfull/novelfull.js deleted file mode 100644 index c211831..0000000 --- a/src/en/novelfull/novelfull.js +++ /dev/null @@ -1,210 +0,0 @@ -const express = require("express"); -const cheerio = require("cheerio"); -const request = require("request"); - -const router = express.Router(); - -const baseUrl = "https://novelfull.com"; - -// Top novels - -router.get("/novels/", (req, res) => { - let url = `${baseUrl}/most-popular`; - - console.log(url); - - request(url, (err, response, body) => { - if (err) throw err; - - let novels = []; - - $ = cheerio.load(body); - - $("div.col-truyen-main > div.list-truyen > .row").each(function ( - result - ) { - let novelUrl = $(this) - .find("h3.truyen-title > a") - .attr("href") - .replace(".html", "") - .substring(1); - novelUrl = `${novelUrl}/`; - let novelName = $(this).find("h3.truyen-title > a").text(); - let novelCover = $(this).find("img").attr("src"); - novelCover = baseUrl + novelCover; - - novel = { - extensionId: 8, - novelUrl, - novelName, - novelCover, - }; - - novels.push(novel); - }); - - res.json(novels); - }); -}); - -router.get("/novel/:novelUrl", (req, res) => { - let novelUrl = req.params.novelUrl; - url = `${baseUrl}/${novelUrl}.html`; - - request(url, (err, response, body) => { - $ = cheerio.load(body); - - let novel = {}; - - novel.extensionId = 8; - - novel.sourceName = "NovelFull"; - - novel.sourceUrl = url; - - novel.novelUrl = `${novelUrl}/`; - - novel.novelName = $("div.book > img").attr("alt"); - - novel.novelCover = baseUrl + $("div.book > img").attr("src"); - - novel.novelSummary = $("div.desc-text").text(); - - novel["Author(s)"] = $("div.info > div > h3") - .filter(function () { - return $(this).text().trim() === "Author:"; - }) - .siblings() - .text(); - - novel["Genre(s)"] = $("div.info > div") - .filter(function () { - return $(this).find("h3").text().trim() === "Genre:"; - }) - .text() - .replace("Genre:", ""); - - novel["Artist(s)"] = null; - - novel.Status = $("div.info > div > h3") - .filter(function () { - return $(this).text().trim() === "Status:"; - }) - .next() - .text(); - - const novelId = $("#rating").attr("data-novel-id"); - console.log(novelId); - - const chapterListUrl = - baseUrl + "/ajax/chapter-option?novelId=" + novelId; - - function callback(res) { - console.log(res); - novel.novelChapters = res; - } - - request(chapterListUrl, (err, response, body) => { - $ = cheerio.load(body); - let novelChapters = []; - - $("select > option").each(function (result) { - let chapterName = $(this).text(); - let releaseDate = null; - let chapterUrl = $(this).attr("value"); - chapterUrl = chapterUrl.replace(`/${novelUrl}/`, ""); - - novelChapters.push({ - chapterName, - releaseDate, - chapterUrl, - }); - }); - callback(novelChapters); - - res.json(novel); - }); - }); -}); - -router.get("/novel/:novelUrl/:chapterUrl", (req, res) => { - let novelUrl = req.params.novelUrl.replace(".html/", ""); - url = `${baseUrl}/${novelUrl}/${req.params.chapterUrl}`; - - request(url, (err, response, body) => { - if (err) throw err; - - $ = cheerio.load(body); - - let chapterName = $(".chapter-title").attr("title"); - - let chapterText = $("#chapter-content").text(); - - let nextChapter = null; - if ($("a#next_chap").attr("href")) { - nextChapter = $("a#next_chap") - .attr("href") - .replace(`/${novelUrl}/`, ""); - } - - let prevChapter = null; - if ($("a#prev_chap").attr("href")) { - prevChapter = $("a#prev_chap") - .attr("href") - .replace(`/${novelUrl}/`, ""); - } - - chapter = { - extensionId: 8, - novelUrl: `${req.params.novelUrl}`, - chapterUrl: `${req.params.chapterUrl}`, - chapterName, - chapterText, - nextChapter, - prevChapter, - }; - - res.json(chapter); - }); -}); - -router.get("/search/", (req, res) => { - let searchTerm = req.query.s; - searchUrl = `https://novelfull.com/search?keyword=`; - - url = `${searchUrl}${searchTerm}`; - - request(url, (err, response, body) => { - if (err) throw err; - - $ = cheerio.load(body); - - let novels = []; - - $("div.col-truyen-main > div.list-truyen > .row").each(function ( - result - ) { - let novelUrl = $(this) - .find("h3.truyen-title > a") - .attr("href") - .replace(".html", "") - .substring(1); - novelUrl = `${novelUrl}/`; - let novelName = $(this).find("h3.truyen-title > a").text(); - let novelCover = $(this).find("img").attr("src"); - - novel = { - extensionId: 8, - novelUrl, - novelName, - novelCover, - }; - - novels.push(novel); - }); - - res.json(novels); - }); -}); - -module.exports = router; diff --git a/src/en/novelhall/Novelhall.js b/src/en/novelhall/Novelhall.js new file mode 100644 index 0000000..717ec36 --- /dev/null +++ b/src/en/novelhall/Novelhall.js @@ -0,0 +1,14 @@ +const express = require("express"); +const novelhallScraper = require("./NovelhallScraper"); + +const router = express.Router(); + +router.get("/novels/", novelhallScraper.novelsScraper); + +router.get("/novel/:novelUrl", novelhallScraper.novelScraper); + +router.get("/novel/:novelUrl/:chapterUrl", novelhallScraper.chapterScraper); + +router.get("/search/", novelhallScraper.searchScraper); + +module.exports = router; diff --git a/src/en/novelhall/NovelhallScraper.js b/src/en/novelhall/NovelhallScraper.js new file mode 100644 index 0000000..072caf8 --- /dev/null +++ b/src/en/novelhall/NovelhallScraper.js @@ -0,0 +1,182 @@ +const cheerio = require("cheerio"); +const fetch = require("node-fetch"); + +const baseUrl = "https://www.novelhall.com/"; + +const novelsScraper = async (req, res) => { + const result = await fetch(baseUrl); + const body = await result.text(); + + $ = cheerio.load(body); + + let novels = []; + + $("div.section1") + .find("li") + .each(function (result) { + const novelName = $(this).find(".book-info > h2 > a").text(); + const novelCover = $(this).find("img").attr("src"); + const novelUrl = $(this).find("a").attr("href").substring(1); + + const novel = { + extensionId: 6, + novelName, + novelCover, + novelUrl, + }; + + novels.push(novel); + }); + + res.json(novels); +}; + +const novelScraper = async (req, res) => { + const novelUrl = req.params.novelUrl; + const url = `${baseUrl}${novelUrl}/`; + + const result = await fetch(url); + const body = await result.text(); + + $ = cheerio.load(body); + + let novel = {}; + + novel.extensionId = 6; + + novel.sourceName = "NovelHall"; + + novel.sourceUrl = url; + + novel.novelUrl = `${novelUrl}/`; + + novel.novelName = $("h1").text(); + + novel.novelCover = $("div.book-img > img").attr("src"); + + novel.novelSummary = $("div.intro") + .text() + .replace(/[\t\n]/g, ""); + + novel["Author(s)"] = $("span.blue").first().text().replace("Author:", ""); + + novel["Genre(s)"] = $("a.red").text(); + + novel["Artist(s)"] = null; + + novel.Status = $("span.blue").first().next().text().replace("Status:", ""); + + let novelChapters = []; + + $("div.book-catalog.hidden-xs#morelist") + .find("li.post-11") + .each(function (result) { + let chapterName = $(this).find("a").text(); + + let releaseDate = null; + + let chapterUrl = $(this).find("a").attr("href"); + + if (chapterUrl) { + chapterUrl = chapterUrl.replace(`/${novelUrl}/`, ""); + } + + const chapter = { + chapterName, + releaseDate, + chapterUrl, + }; + + novelChapters.push(chapter); + }); + + novel.novelChapters = novelChapters; + + res.json(novel); +}; + +const chapterScraper = async (req, res) => { + let novelUrl = req.params.novelUrl.replace(".html/", ""); + let chapterUrl = req.params.chapterUrl; + + const url = `${baseUrl}${novelUrl}/${chapterUrl}`; + + const result = await fetch(url); + const body = await result.text(); + + $ = cheerio.load(body); + + const chapterName = $("h1").text(); + const chapterText = $("div.entry-content").text(); + + let nextChapter = null; + nextChapter = $('a[rel="next"]').attr("href"); + if (nextChapter) { + nextChapter = nextChapter.replace(`/${req.params.novelUrl}/`, ""); + } + + let prevChapter = null; + prevChapter = $('a[rel="prev"]').attr("href"); + if (prevChapter) { + prevChapter = prevChapter.replace(`/${req.params.novelUrl}/`, ""); + } + + novelUrl = novelUrl + "/"; + chapterUrl = chapterUrl + "/"; + + chapter = { + extensionId: 6, + novelUrl, + chapterUrl, + chapterName, + chapterText, + nextChapter, + prevChapter, + }; + + res.json(chapter); +}; + +const searchScraper = async (req, res) => { + const searchTerm = req.query.s; + const searchUrl = `${baseUrl}index.php?s=so&module=book&keyword=`; + + const url = `${searchUrl}${searchTerm}`; + + const result = await fetch(url); + const body = await result.text(); + + $ = cheerio.load(body); + + let novels = []; + + $("tr").each(function (result) { + let novelName = $(this).find("td:nth-child(2)").text(); + novelName = novelName.replace(/[\t\n]/g, ""); + + const novelCover = "https://cdn.novelupdates.com/imgmid/noimagemid.jpg"; + + let novelUrl = $(this).find("td:nth-child(2) >").attr("href"); + if (novelUrl) { + novelUrl = novelUrl.slice(1); + } + + novel = { + extensionId: 6, + novelName, + novelCover, + novelUrl, + }; + + novels.push(novel); + }); + + res.json(novels); +}; + +module.exports = novelhallScraper = { + novelsScraper, + novelScraper, + chapterScraper, + searchScraper, +}; diff --git a/src/en/novelhall/novelhall.js b/src/en/novelhall/novelhall.js deleted file mode 100644 index a7f659b..0000000 --- a/src/en/novelhall/novelhall.js +++ /dev/null @@ -1,191 +0,0 @@ -const express = require("express"); -const cheerio = require("cheerio"); -const request = require("request"); - -const baseUrl = "https://www.novelhall.com/"; - -const router = express.Router(); -router.get("/novels/", (req, res) => { - request(baseUrl, (err, response, body) => { - if (err) throw err; - - let novels = []; - - $ = cheerio.load(body); - - $("div.section1") - .find("li") - .each(function (result) { - let novelName = $(this).find(".book-info > h2 > a").text(); - let novelCover = $(this).find("img").attr("src"); - let novelUrl = $(this).find("a").attr("href").substring(1); - - novel = { - extensionId: 6, - novelName, - novelCover, - novelUrl, - }; - - novels.push(novel); - }); - - res.json(novels); - }); -}); - -// Novel - -router.get("/novel/:novelUrl", (req, res) => { - let novelUrl = req.params.novelUrl; - const url = `${baseUrl}${novelUrl}/`; - - request(url, (err, response, body) => { - if (err) throw err; - - $ = cheerio.load(body); - - let novel = {}; - - novel.extensionId = 6; - - novel.sourceName = "NovelHall"; - - novel.sourceUrl = url; - - novel.novelUrl = `${novelUrl}/`; - - novel.novelName = $("h1").text(); - - novel.novelCover = $("div.book-img > img").attr("src"); - - novel.novelSummary = $("div.intro") - .text() - .replace(/[\t\n]/g, ""); - - novel["Author(s)"] = $("span.blue") - .first() - .text() - .replace("Author:", ""); - - novel["Genre(s)"] = $("a.red").text(); - - novel["Artist(s)"] = null; - - novel.Status = $("span.blue") - .first() - .next() - .text() - .replace("Status:", ""); - - let novelChapters = []; - - $("div.book-catalog.hidden-xs#morelist") - .find("li.post-11") - .each(function (result) { - let chapterName = $(this).find("a").text(); - - let releaseDate = null; - - let chapterUrl = $(this).find("a").attr("href"); - - if (chapterUrl) { - chapterUrl = chapterUrl.replace(`/${novelUrl}/`, ""); - } - - novelChapters.push({ - chapterName, - releaseDate, - chapterUrl, - }); - }); - - novel.novelChapters = novelChapters; - - res.json(novel); - }); -}); - -router.get("/novel/:novelUrl/:chapterUrl", (req, res) => { - url = `${baseUrl}${req.params.novelUrl}/${req.params.chapterUrl}`; - - request(url, (err, response, body) => { - if (err) throw err; - - $ = cheerio.load(body); - - let chapterName = $("h1").text(); - - let chapterText = $("div.entry-content").text(); - - let nextChapter = null; - - nextChapter = $('a[rel="next"]').attr("href"); - - if (nextChapter) { - nextChapter = nextChapter.replace(`/${req.params.novelUrl}/`, ""); - } - - let prevChapter = null; - - prevChapter = $('a[rel="prev"]').attr("href"); - - if (prevChapter) { - prevChapter = prevChapter.replace(`/${req.params.novelUrl}/`, ""); - } - - chapter = { - extensionId: 6, - novelUrl: `${req.params.novelUrl}/`, - chapterUrl: `${req.params.chapterUrl}/`, - chapterName, - chapterText, - nextChapter, - prevChapter, - }; - - res.json(chapter); - }); -}); - -router.get("/search/", (req, res) => { - let searchTerm = req.query.s; - searchTerm = encodeURI(searchTerm); - const searchUrl = `https://www.novelhall.com/index.php?s=so&module=book&keyword=${searchTerm}`; - - request(searchUrl, (err, response, body) => { - if (err) throw err; - - $ = cheerio.load(body); - - let novels = []; - - $("tr").each(function (result) { - let novelName = $(this) - .find("td:nth-child(2)") - .text() - .replace(/[\t\n]/g, ""); - - let novelCover = - "https://cdn.novelupdates.com/imgmid/noimagemid.jpg"; - let novelUrl = $(this).find("td:nth-child(2) >").attr("href"); - - if (novelUrl) { - novelUrl = novelUrl.slice(1); - } - - novel = { - extensionId: 6, - novelName, - novelCover, - novelUrl, - }; - - novels.push(novel); - }); - - res.json(novels); - }); -}); - -module.exports = router; diff --git a/src/en/readlightnovel/ReadLightNovel.js b/src/en/readlightnovel/ReadLightNovel.js new file mode 100644 index 0000000..bb5e049 --- /dev/null +++ b/src/en/readlightnovel/ReadLightNovel.js @@ -0,0 +1,17 @@ +const express = require("express"); +const readLightNovelScraper = require("./ReadLightNovelScraper.js"); + +const router = express.Router(); + +router.get("/novels/", readLightNovelScraper.novelsScraper); + +router.get("/novel/:novelUrl", readLightNovelScraper.novelScraper); + +router.get( + "/novel/:novelUrl/:chapterUrl/:volumeUrl?", + readLightNovelScraper.chapterScraper +); + +router.get("/search/", readLightNovelScraper.searchScraper); + +module.exports = router; diff --git a/src/en/readlightnovel/ReadLightNovelScraper.js b/src/en/readlightnovel/ReadLightNovelScraper.js new file mode 100644 index 0000000..691e3ef --- /dev/null +++ b/src/en/readlightnovel/ReadLightNovelScraper.js @@ -0,0 +1,222 @@ +const cheerio = require("cheerio"); +const fetch = require("node-fetch"); +const request = require("request"); + +const baseUrl = "https://www.readlightnovel.org"; +const searchUrl = "https://www.readlightnovel.org/detailed-search"; + +const novelsScraper = async (req, res) => { + const url = `${baseUrl}/top-novel?change_type=top_rated`; + + const result = await fetch(url); + const body = await result.text(); + + $ = cheerio.load(body); + + let novels = []; + + $(".top-novel-block").each(function (result) { + const novelName = $(this).find(".top-novel-header > h2 > a").text(); + const novelCover = $(this).find("img").attr("src"); + + let novelUrl = $(this) + .find(".top-novel-header > h2 > a") + .attr("href") + .replace(`${baseUrl}/`, ""); + novelUrl = novelUrl + "/"; + + const novel = { + extensionId: 2, + novelName, + novelCover, + novelUrl, + }; + + novels.push(novel); + }); + + res.json(novels); +}; + +const novelScraper = async (req, res) => { + const novelUrl = req.params.novelUrl; + const url = `${baseUrl}/${novelUrl}`; + + const result = await fetch(url); + const body = await result.text(); + + $ = cheerio.load(body); + + let novel = {}; + + novel.extensionId = 2; + + novel.sourceName = "ReadLightNovel"; + + novel.sourceUrl = url; + + novel.novelUrl = novelUrl + "/"; + + novel.novelName = $(".block-title > h1").text(); + + novel.novelCover = $(".novel-cover > a > img").attr("src"); + + $(".novel-detail-item").each(function (result) { + const detailName = $(this) + .find(".novel-detail-header > h6") + .text() + .replace(/[\t\n]/g, "") + .trim(); + const detail = $(this).find(".novel-detail-body").text().trim(); + + novel[detailName] = detail; + }); + + novel.Alternative = novel["Alternative Names"]; + novel.novelSummary = novel.Description; + novel["Genre(s)"] = novel.Genre.replace(/[\t\n]/g, ", "); + novel.Release = novel.Year; + + delete novel["Alternative Names"]; + delete novel.Description; + delete novel.Genre; + delete novel.Year; + + let novelChapters = []; + + $(".panel").each(function (res) { + let volumeName = $(this).find("h4.panel-title").text(); + + $(this) + .find("ul.chapter-chs > li") + .each(function (result) { + chapterName = $(this) + .find("a") + .text() + .replace(/[\t\n]/g, "") + .trim(); + + releaseDate = null; + + chapterUrl = $(this).find("a").attr("href"); + + chapterUrl = chapterUrl.replace(`${baseUrl}/${novelUrl}/`, ""); + + if (volumeName.includes("Volume")) { + chapterName = volumeName + " " + chapterName; + } + + const chapter = { + chapterName, + releaseDate, + chapterUrl, + }; + + novelChapters.push(chapter); + }); + }); + + novel.novelChapters = novelChapters; + + res.json(novel); +}; + +const chapterScraper = async (req, res) => { + const novelUrl = req.params.novelUrl; + let chapterUrl = req.params.chapterUrl; + const volumeUrl = req.params.volumeUrl; + + let optionalUrl; + + if (volumeUrl) { + optionalUrl = volumeUrl; + } else { + optionalUrl = ""; + } + + const url = `${baseUrl}/${novelUrl}/${chapterUrl}/${optionalUrl}`; + + const result = await fetch(url); + const body = await result.text(); + + $ = cheerio.load(body); + + $(".block-title > h1").find("a").remove(); + + const chapterName = $(".block-title > h1").text().replace(" - ", ""); + const chapterText = $(".desc").text(); + + let nextChapter = null; + + if ($("a.next.next-link").length) { + nextChapter = $("a.next.next-link") + .attr("href") + .replace(`${baseUrl}/${novelUrl}/`, ""); + } + + let prevChapter = null; + + if ($("a.prev.prev-link").length) { + prevChapter = $("a.prev.prev-link") + .attr("href") + .replace(`${baseUrl}/${novelUrl}/`, ""); + } + + chapterUrl = chapterUrl + "/" + optionalUrl; + + const chapter = { + extensionId: 2, + novelUrl, + chapterUrl, + chapterName, + chapterText, + nextChapter, + prevChapter, + }; + + res.json(chapter); +}; + +const searchScraper = async (req, res) => { + const searchTerm = req.query.s; + + request.post( + { url: searchUrl, form: { keyword: searchTerm, search: 1 } }, + (err, response, body) => { + $ = cheerio.load(body); + + let novels = []; + + $(".top-novel-block").each(function (result) { + const novelName = $(this) + .find(".top-novel-header > h2 > a") + .text(); + const novelCover = $(this).find("img").attr("src"); + + let novelUrl = $(this) + .find(".top-novel-header > h2 > a") + .attr("href") + .replace(`${baseUrl}/`, ""); + novelUrl = novelUrl + "/"; + + const novel = { + extensionId: 2, + novelName, + novelCover, + novelUrl, + }; + + novels.push(novel); + }); + + res.json(novels); + } + ); +}; + +module.exports = readLightNovelScraper = { + novelsScraper, + novelScraper, + chapterScraper, + searchScraper, +}; diff --git a/src/en/readlightnovel/readlightnovel.js b/src/en/readlightnovel/readlightnovel.js deleted file mode 100644 index d515740..0000000 --- a/src/en/readlightnovel/readlightnovel.js +++ /dev/null @@ -1,223 +0,0 @@ -const express = require("express"); -const cheerio = require("cheerio"); -const request = require("request"); - -const router = express.Router(); - -const baseUrl = "https://www.readlightnovel.org"; - -const searchUrl = "https://www.readlightnovel.org/detailed-search"; - -// Top Novels - -router.get("/novels/", (req, res) => { - // orderBy = req.query.o; - - url = `${baseUrl}/top-novel?change_type=top_rated`; - - request(url, (err, response, body) => { - if (err) throw err; - - $ = cheerio.load(body); - - let novels = []; - - $(".top-novel-block").each(function (result) { - novelName = $(this).find(".top-novel-header > h2 > a").text(); - novelCover = $(this).find("img").attr("src"); - novelUrl = $(this) - .find(".top-novel-header > h2 > a") - .attr("href") - .replace(`${baseUrl}/`, ""); - novel = { - extensionId: 2, - novelName: novelName, - novelCover: novelCover, - novelUrl: `${novelUrl}/`, - }; - - novels.push(novel); - }); - - res.json(novels); - }); -}); - -// Novel - -router.get("/novel/:novelUrl", (req, res) => { - novelUrl = req.params.novelUrl; - url = `${baseUrl}/${novelUrl}`; - - request(url, (err, response, body) => { - if (err) throw err; - - $ = cheerio.load(body); - - let novel = {}; - - novel.extensionId = 2; - - novel.sourceName = "ReadLightNovel"; - - novel.sourceUrl = url; - - novel.novelUrl = `${novelUrl}/`; - - novel.novelName = $(".block-title > h1").text(); - - novel.novelCover = $(".novel-cover > a > img").attr("src"); - - $(".novel-detail-item").each(function (result) { - detailName = $(this) - .find(".novel-detail-header > h6") - .text() - .replace(/[\t\n]/g, "") - .trim(); - detail = $(this) - .find(".novel-detail-body") - .text() - // .replace(/[\t\n]/g, " ") - .trim(); - - novel[detailName] = detail; - }); - - novel.Alternative = novel["Alternative Names"]; - novel.novelSummary = novel.Description; - novel["Genre(s)"] = novel.Genre.replace(/[\t\n]/g, ", "); - novel.Release = novel.Year; - - delete novel["Alternative Names"]; - delete novel.Description; - delete novel.Genre; - delete novel.Year; - - let novelChapters = []; - - $(".panel").each(function (res) { - let volumeName = $(this).find("h4.panel-title").text(); - - $(this) - .find("ul.chapter-chs > li") - .each(function (result) { - chapterName = $(this) - .find("a") - .text() - .replace(/[\t\n]/g, "") - .trim(); - - releaseDate = null; - - chapterUrl = $(this).find("a").attr("href"); - - chapterUrl = chapterUrl.replace( - `${baseUrl}/${novelUrl}/`, - "" - ); - - if (volumeName.includes("Volume")) { - chapterName = volumeName + " " + chapterName; - } - - novelChapters.push({ - chapterName, - releaseDate, - chapterUrl, - }); - }); - }); - - novel.novelChapters = novelChapters; - - res.json(novel); - }); -}); - -// Chapter - -router.get("/novel/:novelUrl/:chapterUrl/:volumeUrl?", (req, res) => { - req.params.volumeUrl - ? (optionalUrl = req.params.volumeUrl) - : (optionalUrl = ""); - - url = `${baseUrl}/${req.params.novelUrl}/${req.params.chapterUrl}/${optionalUrl}`; - - request(url, (err, response, body) => { - if (err) throw err; - - $ = cheerio.load(body); - - $(".block-title > h1").find("a").remove(); - - chapterName = $(".block-title > h1").text().replace(" - ", ""); - - // $(".desc").find(".trinity-player-iframe-wrapper").remove(); - - chapterText = $(".desc").text(); - - let nextChapter = null; - - if ($("a.next.next-link").length) { - nextChapter = $("a.next.next-link") - .attr("href") - .replace(`${baseUrl}/${req.params.novelUrl}/`, ""); - } - - let prevChapter = null; - - if ($("a.prev.prev-link").length) { - prevChapter = $("a.prev.prev-link") - .attr("href") - .replace(`${baseUrl}/${req.params.novelUrl}/`, ""); - } - - chapter = { - extensionId: 2, - novelUrl: req.params.novelUrl, - chapterUrl: `${req.params.chapterUrl}/${optionalUrl}`, - chapterName, - chapterText, - nextChapter, - prevChapter, - }; - - res.json(chapter); - }); -}); - -router.get("/search/", (req, res) => { - searchTerm = req.query.s; - - request.post( - { url: searchUrl, form: { keyword: searchTerm, search: 1 } }, - (err, response, body) => { - if (err) throw err; - - $ = cheerio.load(body); - - let novels = []; - - $(".top-novel-block").each(function (result) { - novelName = $(this).find(".top-novel-header > h2 > a").text(); - novelCover = $(this).find("img").attr("src"); - novelUrl = $(this) - .find(".top-novel-header > h2 > a") - .attr("href") - .replace(`${baseUrl}/`, ""); - novel = { - extensionId: 2, - novelName: novelName, - novelCover: novelCover, - novelUrl: `${novelUrl}/`, - }; - - novels.push(novel); - }); - - res.json(novels); - } - ); -}); - -module.exports = router; diff --git a/src/en/readnovelfull/ReadNovelFull.js b/src/en/readnovelfull/ReadNovelFull.js new file mode 100644 index 0000000..44e6dfa --- /dev/null +++ b/src/en/readnovelfull/ReadNovelFull.js @@ -0,0 +1,14 @@ +const express = require("express"); +const readNovelFullScraper = require("./ReadNovelFullScraper"); + +const router = express.Router(); + +router.get("/novels/", readNovelFullScraper.novelsScraper); + +router.get("/novel/:novelUrl", readNovelFullScraper.novelScraper); + +router.get("/novel/:novelUrl/:chapterUrl", readNovelFullScraper.chapterScraper); + +router.get("/search/", readNovelFullScraper.searchScraper); + +module.exports = router; diff --git a/src/en/readnovelfull/ReadNovelFullScraper.js b/src/en/readnovelfull/ReadNovelFullScraper.js new file mode 100644 index 0000000..c715410 --- /dev/null +++ b/src/en/readnovelfull/ReadNovelFullScraper.js @@ -0,0 +1,206 @@ +const cheerio = require("cheerio"); +const fetch = require("node-fetch"); + +const baseUrl = "https://readnovelfull.com"; + +const novelsScraper = async (req, res) => { + const url = `${baseUrl}/most-popular-novel`; + + const result = await fetch(url); + const body = await result.text(); + + $ = cheerio.load(body); + + let novels = []; + + $("div.col-novel-main > div.list-novel > .row").each(function (result) { + let novelUrl = $(this) + .find("h3.novel-title > a") + .attr("href") + .replace(".html", "") + .substring(1); + novelUrl = `${novelUrl}/`; + + const novelName = $(this).find("h3.novel-title > a").text(); + const novelCover = $(this).find("img").attr("src"); + + const novel = { + extensionId: 4, + novelUrl, + novelName, + novelCover, + }; + + novels.push(novel); + }); + + res.json(novels); +}; + +const novelScraper = async (req, res) => { + const novelUrl = req.params.novelUrl; + const url = `${baseUrl}/${novelUrl}.html`; + + const result = await fetch(url); + const body = await result.text(); + + $ = cheerio.load(body); + + let novel = {}; + + novel.extensionId = 4; + + novel.sourceName = "ReadNovelFull"; + + novel.sourceUrl = url; + + novel.novelUrl = novelUrl + "/"; + + novel.novelName = $("div.book > img").attr("alt"); + + novel.novelCover = $("div.book > img").attr("src"); + + novel.novelSummary = $("div.desc-text").text(); + + novel["Author(s)"] = $("li > h3") + .filter(function () { + return $(this).text().trim() === "Author:"; + }) + .siblings() + .text(); + + novel["Genre(s)"] = $("li") + .filter(function () { + return $(this).find("h3").text().trim() === "Genre:"; + }) + .text() + .replace("Genre:", ""); + + novel["Artist(s)"] = null; + + novel.Status = $("li > h3") + .filter(function () { + return $(this).text().trim() === "Status:"; + }) + .siblings() + .text(); + + const novelId = $("#rating").attr("data-novel-id"); + + const getChapters = async (novelId) => { + const chapterListUrl = + "https://readnovelfull.com/ajax/chapter-archive?novelId=" + novelId; + + const data = await fetch(chapterListUrl); + const chapters = await data.text(); + + $ = cheerio.load(chapters); + + let novelChapters = []; + + $(".panel-body") + .find("li") + .each(function (result) { + let chapterName = $(this).find("a").attr("title"); + let releaseDate = null; + let chapterUrl = $(this).find("a").attr("href"); + chapterUrl = chapterUrl.replace(`/${novelUrl}/`, ""); + + novelChapters.push({ + chapterName, + releaseDate, + chapterUrl, + }); + }); + return novelChapters; + }; + + novel.novelChapters = await getChapters(novelId); + + res.json(novel); +}; + +const chapterScraper = async (req, res) => { + let novelUrl = req.params.novelUrl.replace(".html/", ""); + let chapterUrl = req.params.chapterUrl; + + const url = `${baseUrl}/${novelUrl}/${chapterUrl}`; + + const result = await fetch(url); + const body = await result.text(); + + $ = cheerio.load(body); + + const chapterName = $(".chr-title").attr("title"); + const chapterText = $("#chr-content").text(); + + let nextChapter = null; + if ($("a#next_chap").attr("href")) { + nextChapter = $("a#next_chap") + .attr("href") + .replace(`/${novelUrl}/`, ""); + } + + let prevChapter = null; + if ($("a#prev_chap").attr("href")) { + prevChapter = $("a#prev_chap") + .attr("href") + .replace(`/${novelUrl}/`, ""); + } + + const chapter = { + extensionId: 4, + novelUrl, + chapterUrl, + chapterName, + chapterText, + nextChapter, + prevChapter, + }; + + res.json(chapter); +}; + +const searchScraper = async (req, res) => { + const searchTerm = req.query.s; + const searchUrl = `https://readnovelfull.com/search?keyword=`; + + const url = `${searchUrl}${searchTerm}`; + + const result = await fetch(url); + const body = await result.text(); + + $ = cheerio.load(body); + + let novels = []; + + $("div.col-novel-main > div.list-novel > .row").each(function (result) { + let novelUrl = $(this) + .find("h3.novel-title > a") + .attr("href") + .replace(".html", "") + .substring(1); + novelUrl = `${novelUrl}/`; + + const novelName = $(this).find("h3.novel-title > a").text(); + const novelCover = $(this).find("img").attr("src"); + + const novel = { + extensionId: 4, + novelUrl, + novelName, + novelCover, + }; + + novels.push(novel); + }); + + res.json(novels); +}; + +module.exports = readNovelFullScraper = { + novelsScraper, + novelScraper, + chapterScraper, + searchScraper, +}; diff --git a/src/en/readnovelfull/readnovelfull.js b/src/en/readnovelfull/readnovelfull.js deleted file mode 100644 index 613f4e4..0000000 --- a/src/en/readnovelfull/readnovelfull.js +++ /dev/null @@ -1,204 +0,0 @@ -const express = require("express"); -const cheerio = require("cheerio"); -const request = require("request"); - -const router = express.Router(); - -const baseUrl = "https://readnovelfull.com"; - -// Top novels - -router.get("/novels/", (req, res) => { - url = `${baseUrl}/most-popular-novel`; - - request(url, (err, response, body) => { - if (err) throw err; - - let novels = []; - - $ = cheerio.load(body); - - $("div.col-novel-main > div.list-novel > .row").each(function (result) { - let novelUrl = $(this) - .find("h3.novel-title > a") - .attr("href") - .replace(".html", "") - .substring(1); - novelUrl = `${novelUrl}/`; - let novelName = $(this).find("h3.novel-title > a").text(); - let novelCover = $(this).find("img").attr("src"); - - novel = { - extensionId: 4, - novelUrl, - novelName, - novelCover, - }; - - novels.push(novel); - }); - - res.json(novels); - }); -}); - -router.get("/novel/:novelUrl", (req, res) => { - let novelUrl = req.params.novelUrl; - url = `${baseUrl}/${novelUrl}.html`; - - request(url, (err, response, body) => { - $ = cheerio.load(body); - - let novel = {}; - - novel.extensionId = 4; - - novel.sourceName = "ReadNovelFull"; - - novel.sourceUrl = url; - - novel.novelUrl = `${novelUrl}/`; - - novel.novelName = $("div.book > img").attr("alt"); - - novel.novelCover = $("div.book > img").attr("src"); - - novel.novelSummary = $("div.desc-text").text(); - - novel["Author(s)"] = $("li > h3") - .filter(function () { - return $(this).text().trim() === "Author:"; - }) - .siblings() - .text(); - - novel["Genre(s)"] = $("li") - .filter(function () { - return $(this).find("h3").text().trim() === "Genre:"; - }) - .text() - .replace("Genre:", ""); - - novel["Artist(s)"] = null; - - novel.Status = $("li > h3") - .filter(function () { - return $(this).text().trim() === "Status:"; - }) - .siblings() - .text(); - - const novelId = $("#rating").attr("data-novel-id"); - - const chapterListUrl = - "https://readnovelfull.com/ajax/chapter-archive?novelId=" + novelId; - - function callback(res) { - console.log(res); - novel.novelChapters = res; - } - - request(chapterListUrl, (err, response, body) => { - $ = cheerio.load(body); - let novelChapters = []; - - $(".panel-body") - .find("li") - .each(function (result) { - let chapterName = $(this).find("a").attr("title"); - let releaseDate = null; - let chapterUrl = $(this).find("a").attr("href"); - chapterUrl = chapterUrl.replace(`/${novelUrl}/`, ""); - - novelChapters.push({ - chapterName, - releaseDate, - chapterUrl, - }); - }); - callback(novelChapters); - - res.json(novel); - }); - }); -}); - -router.get("/novel/:novelUrl/:chapterUrl", (req, res) => { - let novelUrl = req.params.novelUrl.replace(".html/", ""); - url = `${baseUrl}/${novelUrl}/${req.params.chapterUrl}`; - - request(url, (err, response, body) => { - if (err) throw err; - - $ = cheerio.load(body); - - let chapterName = $(".chr-title").attr("title"); - - chapterText = $("#chr-content").text(); - - let nextChapter = null; - if ($("a#next_chap").attr("href")) { - nextChapter = $("a#next_chap") - .attr("href") - .replace(`/${novelUrl}/`, ""); - } - - let prevChapter = null; - if ($("a#prev_chap").attr("href")) { - prevChapter = $("a#prev_chap") - .attr("href") - .replace(`/${novelUrl}/`, ""); - } - - chapter = { - extensionId: 4, - novelUrl: `${req.params.novelUrl}`, - chapterUrl: `${req.params.chapterUrl}`, - chapterName, - chapterText, - nextChapter, - prevChapter, - }; - - res.json(chapter); - }); -}); - -router.get("/search/", (req, res) => { - let searchTerm = req.query.s; - searchUrl = `https://readnovelfull.com/search?keyword=`; - - url = `${searchUrl}${searchTerm}`; - - request(url, (err, response, body) => { - if (err) throw err; - - $ = cheerio.load(body); - - let novels = []; - - $("div.col-novel-main > div.list-novel > .row").each(function (result) { - let novelUrl = $(this) - .find("h3.novel-title > a") - .attr("href") - .replace(".html", "") - .substring(1); - novelUrl = `${novelUrl}/`; - let novelName = $(this).find("h3.novel-title > a").text(); - let novelCover = $(this).find("img").attr("src"); - - novel = { - extensionId: 4, - novelUrl, - novelName, - novelCover, - }; - - novels.push(novel); - }); - - res.json(novels); - }); -}); - -module.exports = router; diff --git a/src/en/wuxiaworld/WuxiaWorld.js b/src/en/wuxiaworld/WuxiaWorld.js new file mode 100644 index 0000000..746264b --- /dev/null +++ b/src/en/wuxiaworld/WuxiaWorld.js @@ -0,0 +1,14 @@ +const express = require("express"); +const wuxiaWorldScraper = require("./WuxiaWorldScraper"); + +const router = express.Router(); + +router.get("/novels/", wuxiaWorldScraper.novelsScraper); + +router.get("/novel/:novelUrl", wuxiaWorldScraper.novelScraper); + +router.get("/novel/:novelUrl/:chapterUrl", wuxiaWorldScraper.chapterScraper); + +router.get("/search/", wuxiaWorldScraper.searchScraper); + +module.exports = router; diff --git a/src/en/wuxiaworld/WuxiaWorldScraper.js b/src/en/wuxiaworld/WuxiaWorldScraper.js new file mode 100644 index 0000000..7afc049 --- /dev/null +++ b/src/en/wuxiaworld/WuxiaWorldScraper.js @@ -0,0 +1,180 @@ +const cheerio = require("cheerio"); +const fetch = require("node-fetch"); + +const baseUrl = "https://www.wuxiaworld.com/"; + +const novelsScraper = async (req, res) => { + const url = `${baseUrl}api/novels`; + + const result = await fetch(url); + const data = await result.json(); + + let novels = []; + + data.items.map((novel) => { + let novelName = novel.name; + let novelCover = novel.coverUrl; + let novelUrl = novel.slug; + novelUrl = novelUrl + "/"; + + novels.push({ + extensionId: 7, + novelUrl, + novelName, + novelCover, + }); + }); + + res.json(novels); +}; + +const novelScraper = async (req, res) => { + const novelUrl = req.params.novelUrl; + const url = `${baseUrl}novel/${novelUrl}/`; + + const result = await fetch(url); + const body = await result.text(); + + $ = cheerio.load(body); + + let novel = {}; + + novel.extensionId = 7; + + novel.sourceName = "WuxiaWorld"; + + novel.sourceUrl = url; + + novel.novelUrl = `${novelUrl}/`; + + novel.novelName = $("h2").text(); + + novel.novelCover = $("img.img-thumbnail").attr("src"); + + novel.novelSummary = $(".fr-view > p").text(); + + novel["Author(s)"] = $("div > dt") + .filter(function () { + return $(this).text().trim() === "Author:"; + }) + .next() + .text(); + + let genres = []; + + $(".genres") + .find("div") + .each(function (res) { + genres.push($(this).find("a").text()); + }); + + novel["Genre(s)"] = genres.join(","); + + novel["Artist(s)"] = null; + + novel.Status = null; + + let novelChapters = []; + + $(".chapter-item").each(function (result) { + let chapterName = $(this).text(); + chapterName = chapterName.replace(/[\t\n]/g, ""); + + const releaseDate = null; + + let chapterUrl = $(this).find("a").attr("href"); + chapterUrl = chapterUrl.replace(`/novel/${novelUrl}/`, ""); + + const chapter = { + chapterName, + releaseDate, + chapterUrl, + }; + + novelChapters.push(chapter); + }); + + novel.novelChapters = novelChapters; + + res.json(novel); +}; + +const chapterScraper = async (req, res) => { + let novelUrl = req.params.novelUrl; + let chapterUrl = req.params.chapterUrl; + + let url = `${baseUrl}novel/${novelUrl}/${chapterUrl}`; + + const result = await fetch(url); + const body = await result.text(); + + $ = cheerio.load(body); + + let chapterName = $("#sidebar-toggler-container").next().text(); + chapterName = chapterName.replace(/[\t\n]/g, ""); + + const chapterText = $("#chapter-content").text(); + + let nextChapter = null; + + if ($("li.next > a").attr("href") !== "#") { + nextChapter = $("li.next > a") + .attr("href") + .replace(`/novel/${novelUrl}`, ""); + } + + let prevChapter = null; + + if ($("li.prev > a").attr("href") !== "#") { + prevChapter = $("li.prev > a") + .attr("href") + .replace(`/novel/${novelUrl}`, ""); + } + + const chapter = { + extensionId: 7, + novelUrl, + chapterUrl, + chapterName, + chapterText, + nextChapter, + prevChapter, + }; + + res.json(chapter); +}; + +const searchScraper = async (req, res) => { + const searchTerm = req.query.s; + const searchUrl = `https://www.wuxiaworld.com/api/novels/search?query=`; + + const url = `${searchUrl}${searchTerm}`; + + const result = await fetch(url); + const data = await result.json(); + + let novels = []; + + data.items.map((novel) => { + let novelName = novel.name; + let novelCover = novel.coverUrl; + let novelUrl = novel.slug; + novelUrl = novelUrl + "/"; + + novels.push({ + extensionId: 7, + novelUrl, + novelName, + novelCover, + }); + }); + + res.json(novels); +}; + +module.exports = wuxiaWorldScraper = { + novelsScraper, + novelScraper, + chapterScraper, + searchScraper, +}; diff --git a/src/en/wuxiaworld/wuxiaworld.js b/src/en/wuxiaworld/wuxiaworld.js deleted file mode 100644 index 8f47498..0000000 --- a/src/en/wuxiaworld/wuxiaworld.js +++ /dev/null @@ -1,185 +0,0 @@ -const express = require("express"); -const cheerio = require("cheerio"); -const request = require("request"); - -const baseUrl = "https://www.wuxiaworld.com/"; - -const router = express.Router(); - -router.get("/novels/", async (req, res) => { - let url = `https://www.wuxiaworld.com/api/novels`; - - request(url, (err, response, body) => { - if (err) throw err; - - let novels = []; - - let data = JSON.parse(body); - data = data.items; - - data.map((novel) => { - let novelName = novel.name; - let novelCover = novel.coverUrl; - let novelUrl = novel.slug; - novelUrl = novelUrl + "/"; - - novels.push({ - extensionId: 7, - novelUrl, - novelName, - novelCover, - }); - }); - res.json(novels); - }); -}); - -// Novel - -router.get("/novel/:novelUrl", async (req, res) => { - let novelUrl = req.params.novelUrl; - let url = `${baseUrl}novel/${novelUrl}/`; - - request(url, (err, response, body) => { - if (err) throw err; - const $ = cheerio.load(body); - - let novel = {}; - - novel.extensionId = 7; - - novel.sourceName = "WuxiaWorld"; - - novel.sourceUrl = url; - - novel.novelUrl = `${novelUrl}/`; - - novel.novelName = $("h2").text(); - - novel.novelCover = $("img.img-thumbnail").attr("src"); - - novel.novelSummary = $(".fr-view > p").text(); - - novel["Author(s)"] = $("div > dt") - .filter(function () { - return $(this).text().trim() === "Author:"; - }) - .next() - .text(); - - let genres = []; - - $(".genres") - .find("div") - .each(function (res) { - genres.push($(this).find("a").text()); - }); - - novel["Genre(s)"] = genres.join(","); - - novel["Artist(s)"] = null; - - novel.Status = null; - - let novelChapters = []; - - $(".chapter-item").each(function (result) { - let chapterName = $(this).text(); - chapterName = chapterName.replace(/[\t\n]/g, ""); - - let releaseDate = null; - let chapterUrl = $(this).find("a").attr("href"); - chapterUrl = chapterUrl.replace(`/novel/${novelUrl}/`, ""); - - novelChapters.push({ - chapterName, - releaseDate, - chapterUrl, - }); - }); - - novel.novelChapters = novelChapters; - - res.json(novel); - }); -}); - -router.get("/novel/:novelUrl/:chapterUrl", async (req, res) => { - let url = `${baseUrl}novel/${req.params.novelUrl}/${req.params.chapterUrl}`; - - const novelUrl = `${req.params.novelUrl}/`; - const chapterUrl = `${req.params.chapterUrl}/`; - - request(url, (err, response, body) => { - if (err) throw err; - const $ = cheerio.load(body); - - let chapterName = $("#sidebar-toggler-container").next().text(); - chapterName = chapterName.replace(/[\t\n]/g, ""); - - let chapterText = $("#chapter-content").text(); - - let nextChapter = null; - - if ($("li.next > a").attr("href") !== "#") { - nextChapter = $("li.next > a") - .attr("href") - .replace(`/novel/${novelUrl}`, ""); - } - - let prevChapter = null; - - if ($("li.prev > a").attr("href") !== "#") { - prevChapter = $("li.prev > a") - .attr("href") - .replace(`/novel/${novelUrl}`, ""); - } - chapter = { - extensionId: 7, - novelUrl, - chapterUrl, - chapterName, - chapterText, - nextChapter, - prevChapter, - }; - - res.json(chapter); - }); -}); - -router.get("/search/", (req, res) => { - let searchTerm = req.query.s; - searchTerm = encodeURI(searchTerm); - const searchUrl = `https://www.wuxiaworld.com/api/novels/search?query=`; - - const url = `${searchUrl}${searchTerm}`; - - request(url, (err, response, body) => { - if (err) throw err; - - $ = cheerio.load(body); - - let novels = []; - - let data = JSON.parse(body); - data = data.items; - - data.map((novel) => { - let novelName = novel.name; - let novelCover = novel.coverUrl; - let novelUrl = novel.slug; - novelUrl = novelUrl + "/"; - - novels.push({ - extensionId: 7, - novelUrl, - novelName, - novelCover, - }); - }); - res.json(novels); - }); -}); - -module.exports = router; diff --git a/src/helper.js b/src/helper.js index 8443ef5..05568cb 100644 --- a/src/helper.js +++ b/src/helper.js @@ -1,7 +1,7 @@ const chromium = require("chrome-aws-lambda"); const puppeteer = require("puppeteer"); -async function scraper(url) { +const scraper = async (url) => { const browser = await puppeteer.launch({ args: [...chromium.args, "--hide-scrollbars", "--disable-web-security"], defaultViewport: chromium.defaultViewport, @@ -15,6 +15,6 @@ async function scraper(url) { const body = await page.content(); return body; -} +}; module.exports = { scraper };