From d12164dd2b74fdd471231874babe19fa3d9cca71 Mon Sep 17 00:00:00 2001 From: Rajarshee Chatterjee Date: Sun, 2 May 2021 21:10:53 +0530 Subject: [PATCH] Refactor BoxNovel --- index.js | 2 +- package.json | 1 + src/en/boxnovel/BoxNovelScraper.js | 206 +++++++++++++++++++++++++ src/en/boxnovel/boxnovel.js | 238 ++--------------------------- 4 files changed, 218 insertions(+), 229 deletions(-) create mode 100644 src/en/boxnovel/BoxNovelScraper.js diff --git a/index.js b/index.js index 00a89a6..a7de152 100644 --- a/index.js +++ b/index.js @@ -18,7 +18,7 @@ app.get("/", (req, res) => ); app.get("/api/", (req, res) => res.json(extensions)); -app.use("/api/1/", require("./src/en/boxnovel/boxnovel")); +app.use("/api/1/", require("./src/en/boxnovel/BoxNovel")); app.use("/api/2/", require("./src/en/readlightnovel/readlightnovel")); app.use("/api/3/", require("./src/en/fastnovel/fastnovel")); app.use("/api/4/", require("./src/en/readnovelfull/readnovelfull")); diff --git a/package.json b/package.json index 0598a53..a404de6 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,7 @@ "cheerio": "^1.0.0-rc.5", "chrome-aws-lambda": "^8.0.2", "express": "^4.17.1", + "node-fetch": "^2.6.1", "puppeteer": "^9.0.0", "request": "^2.88.2" }, diff --git a/src/en/boxnovel/BoxNovelScraper.js b/src/en/boxnovel/BoxNovelScraper.js new file mode 100644 index 0000000..347aa0f --- /dev/null +++ b/src/en/boxnovel/BoxNovelScraper.js @@ -0,0 +1,206 @@ +const cheerio = require("cheerio"); +const fetch = require("node-fetch"); + +const baseUrl = "https://boxnovel.com/novel"; +const searchUrl = "https://boxnovel.com/"; + +const novelsScraper = async (req, res) => { + const orderBy = req.params.orderBy; + + let url1 = `${baseUrl}/page/1/?m_orderby=${orderBy}`; + let url2 = `${baseUrl}/page/2/?m_orderby=${orderBy}`; + let url3 = `${baseUrl}/page/3/?m_orderby=${orderBy}`; + let url4 = `${baseUrl}/page/4/?m_orderby=${orderBy}`; + + const page1 = await scraper(url1); + const page2 = await scraper(url2); + const page3 = await scraper(url3); + const page4 = await scraper(url4); + + res.json([...page1, ...page2, ...page3, ...page4]); +}; + +const novelScraper = async (req, res) => { + const novelUrl = req.params.novelUrl; + const url = `${baseUrl}/${novelUrl}/`; + + const result = await fetch(url); + const body = await result.text(); + + $ = cheerio.load(body); + + let novel = {}; + + $(".post-title > h3 > span").remove(); + + novel.extensionId = 1; + + novel.sourceName = "BoxNovel"; + + novel.sourceUrl = url; + + novel.novelUrl = `${novelUrl}/`; + + novel.novelName = $(".post-title > h3") + .text() + .replace(/[\t\n]/g, "") + .trim(); + + novel.novelCover = $(".summary_image > a > img").attr("src"); + + $(".post-content_item").each(function (result) { + detailName = $(this) + .find(".summary-heading > h5") + .text() + .replace(/[\t\n]/g, "") + .trim(); + detail = $(this) + .find(".summary-content") + .text() + .replace(/[\t\n]/g, "") + .trim(); + + novel[detailName] = detail; + }); + + $(".description-summary > div.summary__content").find("em").remove(); + + novel.novelSummary = $(".description-summary > div.summary__content > div") + .text() + .replace(/[\t\n]/g, ""); + + let novelChapters = []; + + $(".wp-manga-chapter").each(function (result) { + chapterName = $(this) + .find("a") + .text() + .replace(/[\t\n]/g, "") + .trim(); + + releaseDate = $(this) + .find("span") + .text() + .replace(/[\t\n]/g, "") + .trim(); + + chapterUrl = $(this).find("a").attr("href").replace(url, ""); + + novelChapters.push({ chapterName, releaseDate, chapterUrl }); + }); + + novel.novelChapters = novelChapters.reverse(); + + res.json(novel); +}; + +const chapterScraper = async (req, res) => { + let novelUrl = req.params.novelUrl; + let chapterUrl = req.params.chapterUrl; + + const url = `${baseUrl}/${novelUrl}/${chapterUrl}`; + + const result = await fetch(url); + const body = await result.text(); + + $ = cheerio.load(body); + + const chapterName = $("div.text-left > h3").text(); + const chapterText = $(".reading-content").text(); + + let nextChapter = null; + + if ($(".nav-next").length) { + nextChapter = $(".nav-next") + .find("a") + .attr("href") + .replace(baseUrl + "/" + novelUrl + "/", ""); + } + + let prevChapter = null; + + if ($(".nav-previous").length) { + prevChapter = $(".nav-previous") + .find("a") + .attr("href") + .replace(baseUrl + "/" + novelUrl + "/", ""); + } + + novelUrl = novelUrl + "/"; + chapterUrl = chapterUrl + "/"; + + const chapter = { + extensionId: 1, + novelUrl, + chapterUrl, + chapterName, + chapterText, + nextChapter, + prevChapter, + }; + + res.json(chapter); +}; + +const searchScraper = async (req, res) => { + const searchTerm = req.query.s; + const orderBy = req.query.o; + + const url = `${searchUrl}?s=${searchTerm}&post_type=wp-manga&m_orderby=${orderBy}`; + + const result = await fetch(url); + const body = await result.text(); + + $ = cheerio.load(body); + + let novels = []; + + $(".c-tabs-item__content").each(function (result) { + const novelName = $(this).find("h4 > a").text(); + const novelCover = $(this).find("img").attr("src"); + + let novelUrl = $(this).find("h4 > a").attr("href"); + novelUrl = novelUrlreplace(`${baseUrl}/`, ""); + + const novel = { + extensionId: 1, + novelName, + novelCover, + novelUrl, + }; + + novels.push(novel); + }); + + res.json(novels); +}; + +module.exports = { novelsScraper, novelScraper, chapterScraper, searchScraper }; + +const scraper = async (url) => { + const result = await fetch(url); + const body = await result.text(); + + let novels = []; + + $ = cheerio.load(body); + + $(".page-item-detail").each(function (result) { + const novelName = $(this).find("h5 > a").text(); + const novelCover = $(this).find("img").attr("src"); + + let novelUrl = $(this).find("h5 > a").attr("href"); + novelUrl = novelUrl.replace(`${baseUrl}/`, ""); + + const novel = { + extensionId: 1, + novelName, + novelCover, + novelUrl, + }; + + novels.push(novel); + }); + + return novels; +}; diff --git a/src/en/boxnovel/boxnovel.js b/src/en/boxnovel/boxnovel.js index b61ac10..44b57d5 100644 --- a/src/en/boxnovel/boxnovel.js +++ b/src/en/boxnovel/boxnovel.js @@ -1,247 +1,29 @@ const express = require("express"); const cheerio = require("cheerio"); const request = require("request"); +const { + novelScraper, + searchScraper, + chapterScraper, + novelsScraper, +} = require("./boxnovelScraper"); const router = express.Router(); -const baseUrl = "https://boxnovel.com/novel"; - -const searchUrl = "https://boxnovel.com/"; - // Top novels -function apiCall(reqOps) { - return new Promise(function (resolve, reject) { - request(reqOps, (err, res, body) => { - if (!err && res.statusCode == 200) { - if (err) throw err; - - let novels = []; - - $ = cheerio.load(body); - - $(".page-item-detail").each(function (result) { - novelName = $(this).find("h5 > a").text(); - novelCover = $(this).find("img").attr("src"); - novelUrl = $(this) - .find("h5 > a") - .attr("href") - .replace(`${baseUrl}/`, ""); - novel = { - extensionId: 1, - novelName: novelName, - novelCover: novelCover, - novelUrl: novelUrl, - }; - - novels.push(novel); - }); - resolve(novels); - } - - reject(err); - }); - }); -} - -router.get("/novels/:pageNo/", (req, res) => { - orderBy = req.query.o; - pageNo = req.params.pageNo; - - let url = `${baseUrl}/page/1/?m_orderby=${orderBy}`; - - let page1, page2, page3, page4; - - apiCall(url) - .then((res) => { - page1 = res; - return apiCall(`${baseUrl}/page/2/?m_orderby=${orderBy}`); - }) - .then((res) => { - page2 = res; - return apiCall(`${baseUrl}/page/3/?m_orderby=${orderBy}`); - }) - .then((res) => { - page3 = res; - return apiCall(`${baseUrl}/page/4/?m_orderby=${orderBy}`); - }) - .then((result) => { - page4 = result; - - res.json([...page1, ...page2, ...page3]); - }) - .catch((err) => { - console.log("Error occured in one of the API call: ", err); - }); -}); +router.get("/novels/:pageNo/", novelsScraper); // Novel -router.get("/novel/:novelUrl", (req, res) => { - novelUrl = req.params.novelUrl; - url = `${baseUrl}/${novelUrl}/`; - - request(url, (err, response, body) => { - if (err) throw err; - - $ = cheerio.load(body); - - let novel = {}; - - $(".post-title > h3 > span").remove(); - - novel.extensionId = 1; - - novel.sourceName = "BoxNovel"; - - novel.sourceUrl = url; - - novel.novelUrl = `${novelUrl}/`; - - novel.novelName = $(".post-title > h3") - .text() - .replace(/[\t\n]/g, "") - .trim(); - - novel.novelCover = $(".summary_image > a > img").attr("src"); - - $(".post-content_item").each(function (result) { - detailName = $(this) - .find(".summary-heading > h5") - .text() - .replace(/[\t\n]/g, "") - .trim(); - detail = $(this) - .find(".summary-content") - .text() - .replace(/[\t\n]/g, "") - .trim(); - - novel[detailName] = detail; - }); - - $(".description-summary > div.summary__content").find("em").remove(); - - novel.novelSummary = $( - ".description-summary > div.summary__content > div" - ) - .text() - .replace(/[\t\n]/g, ""); - - // if (novel.novelSummary === "") { - // novel.novelSummary = $(".c_000").text(); - // } - - let novelChapters = []; - - $(".wp-manga-chapter").each(function (result) { - chapterName = $(this) - .find("a") - .text() - .replace(/[\t\n]/g, "") - .trim(); - - releaseDate = $(this) - .find("span") - .text() - .replace(/[\t\n]/g, "") - .trim(); - - chapterUrl = $(this).find("a").attr("href").replace(url, ""); - - novelChapters.push({ chapterName, releaseDate, chapterUrl }); - }); - - novel.novelChapters = novelChapters.reverse(); - - res.json(novel); - }); -}); +router.get("/novel/:novelUrl", novelScraper); // Chapter -router.get("/novel/:novelUrl/:chapterUrl", (req, res) => { - url = `${baseUrl}/${req.params.novelUrl}/${req.params.chapterUrl}`; - - request(url, (err, response, body) => { - if (err) throw err; - - $ = cheerio.load(body); - - chapterNameLower = req.params.chapterUrl.replace("-", " "); - - chapterName = - chapterNameLower.charAt(0).toUpperCase() + - chapterNameLower.slice(1); - - chapterText = $(".reading-content").text(); - - let nextChapter = null; - - if ($(".nav-next").length) { - nextChapter = $(".nav-next") - .find("a") - .attr("href") - .replace(baseUrl + "/" + req.params.novelUrl + "/", ""); - } - - let prevChapter = null; - - if ($(".nav-previous").length) { - prevChapter = $(".nav-previous") - .find("a") - .attr("href") - .replace(baseUrl + "/" + req.params.novelUrl + "/", ""); - } - - chapter = { - extensionId: 1, - novelUrl: `${req.params.novelUrl}/`, - chapterUrl: `${req.params.chapterUrl}/`, - chapterName, - chapterText, - nextChapter, - prevChapter, - }; - - res.json(chapter); - }); -}); +router.get("/novel/:novelUrl/:chapterUrl", chapterScraper); // Search -router.get("/search/", (req, res) => { - searchTerm = req.query.s; - orderBy = req.query.o; - - url = `${searchUrl}?s=${searchTerm}&post_type=wp-manga&m_orderby=${orderBy}`; - - request(url, (err, response, body) => { - if (err) throw err; - - $ = cheerio.load(body); - - let novels = []; - - $(".c-tabs-item__content").each(function (result) { - novelName = $(this).find("h4 > a").text(); - novelCover = $(this).find("img").attr("src"); - - novelUrl = $(this) - .find("h4 > a") - .attr("href") - .replace(`${baseUrl}/`, ""); - - novels.push({ - extensionId: 1, - novelName, - novelCover, - novelUrl, - }); - }); - - res.json(novels); - }); -}); +router.get("/search/", searchScraper); module.exports = router;