Refactor BoxNovel
This commit is contained in:
Rajarshee Chatterjee
@@ -18,7 +18,7 @@ app.get("/", (req, res) =>
|
||||
);
|
||||
|
||||
app.get("/api/", (req, res) => res.json(extensions));
|
||||
app.use("/api/1/", require("./src/en/boxnovel/boxnovel"));
|
||||
app.use("/api/1/", require("./src/en/boxnovel/BoxNovel"));
|
||||
app.use("/api/2/", require("./src/en/readlightnovel/readlightnovel"));
|
||||
app.use("/api/3/", require("./src/en/fastnovel/fastnovel"));
|
||||
app.use("/api/4/", require("./src/en/readnovelfull/readnovelfull"));
|
||||
|
||||
@@ -13,6 +13,7 @@
|
||||
"cheerio": "^1.0.0-rc.5",
|
||||
"chrome-aws-lambda": "^8.0.2",
|
||||
"express": "^4.17.1",
|
||||
"node-fetch": "^2.6.1",
|
||||
"puppeteer": "^9.0.0",
|
||||
"request": "^2.88.2"
|
||||
},
|
||||
|
||||
@@ -0,0 +1,206 @@
|
||||
const cheerio = require("cheerio");
|
||||
const fetch = require("node-fetch");
|
||||
|
||||
const baseUrl = "https://boxnovel.com/novel";
|
||||
const searchUrl = "https://boxnovel.com/";
|
||||
|
||||
const novelsScraper = async (req, res) => {
|
||||
const orderBy = req.params.orderBy;
|
||||
|
||||
let url1 = `${baseUrl}/page/1/?m_orderby=${orderBy}`;
|
||||
let url2 = `${baseUrl}/page/2/?m_orderby=${orderBy}`;
|
||||
let url3 = `${baseUrl}/page/3/?m_orderby=${orderBy}`;
|
||||
let url4 = `${baseUrl}/page/4/?m_orderby=${orderBy}`;
|
||||
|
||||
const page1 = await scraper(url1);
|
||||
const page2 = await scraper(url2);
|
||||
const page3 = await scraper(url3);
|
||||
const page4 = await scraper(url4);
|
||||
|
||||
res.json([...page1, ...page2, ...page3, ...page4]);
|
||||
};
|
||||
|
||||
const novelScraper = async (req, res) => {
|
||||
const novelUrl = req.params.novelUrl;
|
||||
const url = `${baseUrl}/${novelUrl}/`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
$(".post-title > h3 > span").remove();
|
||||
|
||||
novel.extensionId = 1;
|
||||
|
||||
novel.sourceName = "BoxNovel";
|
||||
|
||||
novel.sourceUrl = url;
|
||||
|
||||
novel.novelUrl = `${novelUrl}/`;
|
||||
|
||||
novel.novelName = $(".post-title > h3")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
novel.novelCover = $(".summary_image > a > img").attr("src");
|
||||
|
||||
$(".post-content_item").each(function (result) {
|
||||
detailName = $(this)
|
||||
.find(".summary-heading > h5")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
detail = $(this)
|
||||
.find(".summary-content")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
novel[detailName] = detail;
|
||||
});
|
||||
|
||||
$(".description-summary > div.summary__content").find("em").remove();
|
||||
|
||||
novel.novelSummary = $(".description-summary > div.summary__content > div")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "");
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
$(".wp-manga-chapter").each(function (result) {
|
||||
chapterName = $(this)
|
||||
.find("a")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
releaseDate = $(this)
|
||||
.find("span")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
chapterUrl = $(this).find("a").attr("href").replace(url, "");
|
||||
|
||||
novelChapters.push({ chapterName, releaseDate, chapterUrl });
|
||||
});
|
||||
|
||||
novel.novelChapters = novelChapters.reverse();
|
||||
|
||||
res.json(novel);
|
||||
};
|
||||
|
||||
const chapterScraper = async (req, res) => {
|
||||
let novelUrl = req.params.novelUrl;
|
||||
let chapterUrl = req.params.chapterUrl;
|
||||
|
||||
const url = `${baseUrl}/${novelUrl}/${chapterUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
const chapterName = $("div.text-left > h3").text();
|
||||
const chapterText = $(".reading-content").text();
|
||||
|
||||
let nextChapter = null;
|
||||
|
||||
if ($(".nav-next").length) {
|
||||
nextChapter = $(".nav-next")
|
||||
.find("a")
|
||||
.attr("href")
|
||||
.replace(baseUrl + "/" + novelUrl + "/", "");
|
||||
}
|
||||
|
||||
let prevChapter = null;
|
||||
|
||||
if ($(".nav-previous").length) {
|
||||
prevChapter = $(".nav-previous")
|
||||
.find("a")
|
||||
.attr("href")
|
||||
.replace(baseUrl + "/" + novelUrl + "/", "");
|
||||
}
|
||||
|
||||
novelUrl = novelUrl + "/";
|
||||
chapterUrl = chapterUrl + "/";
|
||||
|
||||
const chapter = {
|
||||
extensionId: 1,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
nextChapter,
|
||||
prevChapter,
|
||||
};
|
||||
|
||||
res.json(chapter);
|
||||
};
|
||||
|
||||
const searchScraper = async (req, res) => {
|
||||
const searchTerm = req.query.s;
|
||||
const orderBy = req.query.o;
|
||||
|
||||
const url = `${searchUrl}?s=${searchTerm}&post_type=wp-manga&m_orderby=${orderBy}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".c-tabs-item__content").each(function (result) {
|
||||
const novelName = $(this).find("h4 > a").text();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
|
||||
let novelUrl = $(this).find("h4 > a").attr("href");
|
||||
novelUrl = novelUrlreplace(`${baseUrl}/`, "");
|
||||
|
||||
const novel = {
|
||||
extensionId: 1,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
res.json(novels);
|
||||
};
|
||||
|
||||
module.exports = { novelsScraper, novelScraper, chapterScraper, searchScraper };
|
||||
|
||||
const scraper = async (url) => {
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
let novels = [];
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
$(".page-item-detail").each(function (result) {
|
||||
const novelName = $(this).find("h5 > a").text();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
|
||||
let novelUrl = $(this).find("h5 > a").attr("href");
|
||||
novelUrl = novelUrl.replace(`${baseUrl}/`, "");
|
||||
|
||||
const novel = {
|
||||
extensionId: 1,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
+10
-228
@@ -1,247 +1,29 @@
|
||||
const express = require("express");
|
||||
const cheerio = require("cheerio");
|
||||
const request = require("request");
|
||||
const {
|
||||
novelScraper,
|
||||
searchScraper,
|
||||
chapterScraper,
|
||||
novelsScraper,
|
||||
} = require("./boxnovelScraper");
|
||||
|
||||
const router = express.Router();
|
||||
|
||||
const baseUrl = "https://boxnovel.com/novel";
|
||||
|
||||
const searchUrl = "https://boxnovel.com/";
|
||||
|
||||
// Top novels
|
||||
|
||||
function apiCall(reqOps) {
|
||||
return new Promise(function (resolve, reject) {
|
||||
request(reqOps, (err, res, body) => {
|
||||
if (!err && res.statusCode == 200) {
|
||||
if (err) throw err;
|
||||
|
||||
let novels = [];
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
$(".page-item-detail").each(function (result) {
|
||||
novelName = $(this).find("h5 > a").text();
|
||||
novelCover = $(this).find("img").attr("src");
|
||||
novelUrl = $(this)
|
||||
.find("h5 > a")
|
||||
.attr("href")
|
||||
.replace(`${baseUrl}/`, "");
|
||||
novel = {
|
||||
extensionId: 1,
|
||||
novelName: novelName,
|
||||
novelCover: novelCover,
|
||||
novelUrl: novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
resolve(novels);
|
||||
}
|
||||
|
||||
reject(err);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
router.get("/novels/:pageNo/", (req, res) => {
|
||||
orderBy = req.query.o;
|
||||
pageNo = req.params.pageNo;
|
||||
|
||||
let url = `${baseUrl}/page/1/?m_orderby=${orderBy}`;
|
||||
|
||||
let page1, page2, page3, page4;
|
||||
|
||||
apiCall(url)
|
||||
.then((res) => {
|
||||
page1 = res;
|
||||
return apiCall(`${baseUrl}/page/2/?m_orderby=${orderBy}`);
|
||||
})
|
||||
.then((res) => {
|
||||
page2 = res;
|
||||
return apiCall(`${baseUrl}/page/3/?m_orderby=${orderBy}`);
|
||||
})
|
||||
.then((res) => {
|
||||
page3 = res;
|
||||
return apiCall(`${baseUrl}/page/4/?m_orderby=${orderBy}`);
|
||||
})
|
||||
.then((result) => {
|
||||
page4 = result;
|
||||
|
||||
res.json([...page1, ...page2, ...page3]);
|
||||
})
|
||||
.catch((err) => {
|
||||
console.log("Error occured in one of the API call: ", err);
|
||||
});
|
||||
});
|
||||
router.get("/novels/:pageNo/", novelsScraper);
|
||||
|
||||
// Novel
|
||||
|
||||
router.get("/novel/:novelUrl", (req, res) => {
|
||||
novelUrl = req.params.novelUrl;
|
||||
url = `${baseUrl}/${novelUrl}/`;
|
||||
|
||||
request(url, (err, response, body) => {
|
||||
if (err) throw err;
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
$(".post-title > h3 > span").remove();
|
||||
|
||||
novel.extensionId = 1;
|
||||
|
||||
novel.sourceName = "BoxNovel";
|
||||
|
||||
novel.sourceUrl = url;
|
||||
|
||||
novel.novelUrl = `${novelUrl}/`;
|
||||
|
||||
novel.novelName = $(".post-title > h3")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
novel.novelCover = $(".summary_image > a > img").attr("src");
|
||||
|
||||
$(".post-content_item").each(function (result) {
|
||||
detailName = $(this)
|
||||
.find(".summary-heading > h5")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
detail = $(this)
|
||||
.find(".summary-content")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
novel[detailName] = detail;
|
||||
});
|
||||
|
||||
$(".description-summary > div.summary__content").find("em").remove();
|
||||
|
||||
novel.novelSummary = $(
|
||||
".description-summary > div.summary__content > div"
|
||||
)
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "");
|
||||
|
||||
// if (novel.novelSummary === "") {
|
||||
// novel.novelSummary = $(".c_000").text();
|
||||
// }
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
$(".wp-manga-chapter").each(function (result) {
|
||||
chapterName = $(this)
|
||||
.find("a")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
releaseDate = $(this)
|
||||
.find("span")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
chapterUrl = $(this).find("a").attr("href").replace(url, "");
|
||||
|
||||
novelChapters.push({ chapterName, releaseDate, chapterUrl });
|
||||
});
|
||||
|
||||
novel.novelChapters = novelChapters.reverse();
|
||||
|
||||
res.json(novel);
|
||||
});
|
||||
});
|
||||
router.get("/novel/:novelUrl", novelScraper);
|
||||
|
||||
// Chapter
|
||||
|
||||
router.get("/novel/:novelUrl/:chapterUrl", (req, res) => {
|
||||
url = `${baseUrl}/${req.params.novelUrl}/${req.params.chapterUrl}`;
|
||||
|
||||
request(url, (err, response, body) => {
|
||||
if (err) throw err;
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
chapterNameLower = req.params.chapterUrl.replace("-", " ");
|
||||
|
||||
chapterName =
|
||||
chapterNameLower.charAt(0).toUpperCase() +
|
||||
chapterNameLower.slice(1);
|
||||
|
||||
chapterText = $(".reading-content").text();
|
||||
|
||||
let nextChapter = null;
|
||||
|
||||
if ($(".nav-next").length) {
|
||||
nextChapter = $(".nav-next")
|
||||
.find("a")
|
||||
.attr("href")
|
||||
.replace(baseUrl + "/" + req.params.novelUrl + "/", "");
|
||||
}
|
||||
|
||||
let prevChapter = null;
|
||||
|
||||
if ($(".nav-previous").length) {
|
||||
prevChapter = $(".nav-previous")
|
||||
.find("a")
|
||||
.attr("href")
|
||||
.replace(baseUrl + "/" + req.params.novelUrl + "/", "");
|
||||
}
|
||||
|
||||
chapter = {
|
||||
extensionId: 1,
|
||||
novelUrl: `${req.params.novelUrl}/`,
|
||||
chapterUrl: `${req.params.chapterUrl}/`,
|
||||
chapterName,
|
||||
chapterText,
|
||||
nextChapter,
|
||||
prevChapter,
|
||||
};
|
||||
|
||||
res.json(chapter);
|
||||
});
|
||||
});
|
||||
router.get("/novel/:novelUrl/:chapterUrl", chapterScraper);
|
||||
|
||||
// Search
|
||||
|
||||
router.get("/search/", (req, res) => {
|
||||
searchTerm = req.query.s;
|
||||
orderBy = req.query.o;
|
||||
|
||||
url = `${searchUrl}?s=${searchTerm}&post_type=wp-manga&m_orderby=${orderBy}`;
|
||||
|
||||
request(url, (err, response, body) => {
|
||||
if (err) throw err;
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".c-tabs-item__content").each(function (result) {
|
||||
novelName = $(this).find("h4 > a").text();
|
||||
novelCover = $(this).find("img").attr("src");
|
||||
|
||||
novelUrl = $(this)
|
||||
.find("h4 > a")
|
||||
.attr("href")
|
||||
.replace(`${baseUrl}/`, "");
|
||||
|
||||
novels.push({
|
||||
extensionId: 1,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
});
|
||||
});
|
||||
|
||||
res.json(novels);
|
||||
});
|
||||
});
|
||||
router.get("/search/", searchScraper);
|
||||
|
||||
module.exports = router;
|
||||
|
||||
Reference in New Issue
Block a user