Refactor (#4)
* refactor BoxNovel * refactor BoxNovel * refactor BoxNovel * refactor FastNovel * remove comments * try puppeteer for FastNovel * refactor MTLNovel * comment out MTLNovel * refactor MTLNovel * revert FastNovel and MTLNovel * possible fix for vercel application error * clear git cache * clear git cache * rename boxnovel in index.js * refactor NovelFull * refactor Novelhall * refactor rest of the sources * fix WuxiaWorld
This commit is contained in:
Rajarshee Chatterjee
committed by
GitHub
GitHub
parent
70e23ea459
commit
4b33834cfb
@@ -0,0 +1,14 @@
|
||||
const express = require("express");
|
||||
const novelhallScraper = require("./NovelhallScraper");
|
||||
|
||||
const router = express.Router();
|
||||
|
||||
router.get("/novels/", novelhallScraper.novelsScraper);
|
||||
|
||||
router.get("/novel/:novelUrl", novelhallScraper.novelScraper);
|
||||
|
||||
router.get("/novel/:novelUrl/:chapterUrl", novelhallScraper.chapterScraper);
|
||||
|
||||
router.get("/search/", novelhallScraper.searchScraper);
|
||||
|
||||
module.exports = router;
|
||||
@@ -0,0 +1,182 @@
|
||||
const cheerio = require("cheerio");
|
||||
const fetch = require("node-fetch");
|
||||
|
||||
const baseUrl = "https://www.novelhall.com/";
|
||||
|
||||
const novelsScraper = async (req, res) => {
|
||||
const result = await fetch(baseUrl);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$("div.section1")
|
||||
.find("li")
|
||||
.each(function (result) {
|
||||
const novelName = $(this).find(".book-info > h2 > a").text();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
const novelUrl = $(this).find("a").attr("href").substring(1);
|
||||
|
||||
const novel = {
|
||||
extensionId: 6,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
res.json(novels);
|
||||
};
|
||||
|
||||
const novelScraper = async (req, res) => {
|
||||
const novelUrl = req.params.novelUrl;
|
||||
const url = `${baseUrl}${novelUrl}/`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.extensionId = 6;
|
||||
|
||||
novel.sourceName = "NovelHall";
|
||||
|
||||
novel.sourceUrl = url;
|
||||
|
||||
novel.novelUrl = `${novelUrl}/`;
|
||||
|
||||
novel.novelName = $("h1").text();
|
||||
|
||||
novel.novelCover = $("div.book-img > img").attr("src");
|
||||
|
||||
novel.novelSummary = $("div.intro")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "");
|
||||
|
||||
novel["Author(s)"] = $("span.blue").first().text().replace("Author：", "");
|
||||
|
||||
novel["Genre(s)"] = $("a.red").text();
|
||||
|
||||
novel["Artist(s)"] = null;
|
||||
|
||||
novel.Status = $("span.blue").first().next().text().replace("Status：", "");
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
$("div.book-catalog.hidden-xs#morelist")
|
||||
.find("li.post-11")
|
||||
.each(function (result) {
|
||||
let chapterName = $(this).find("a").text();
|
||||
|
||||
let releaseDate = null;
|
||||
|
||||
let chapterUrl = $(this).find("a").attr("href");
|
||||
|
||||
if (chapterUrl) {
|
||||
chapterUrl = chapterUrl.replace(`/${novelUrl}/`, "");
|
||||
}
|
||||
|
||||
const chapter = {
|
||||
chapterName,
|
||||
releaseDate,
|
||||
chapterUrl,
|
||||
};
|
||||
|
||||
novelChapters.push(chapter);
|
||||
});
|
||||
|
||||
novel.novelChapters = novelChapters;
|
||||
|
||||
res.json(novel);
|
||||
};
|
||||
|
||||
const chapterScraper = async (req, res) => {
|
||||
let novelUrl = req.params.novelUrl.replace(".html/", "");
|
||||
let chapterUrl = req.params.chapterUrl;
|
||||
|
||||
const url = `${baseUrl}${novelUrl}/${chapterUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
const chapterName = $("h1").text();
|
||||
const chapterText = $("div.entry-content").text();
|
||||
|
||||
let nextChapter = null;
|
||||
nextChapter = $('a[rel="next"]').attr("href");
|
||||
if (nextChapter) {
|
||||
nextChapter = nextChapter.replace(`/${req.params.novelUrl}/`, "");
|
||||
}
|
||||
|
||||
let prevChapter = null;
|
||||
prevChapter = $('a[rel="prev"]').attr("href");
|
||||
if (prevChapter) {
|
||||
prevChapter = prevChapter.replace(`/${req.params.novelUrl}/`, "");
|
||||
}
|
||||
|
||||
novelUrl = novelUrl + "/";
|
||||
chapterUrl = chapterUrl + "/";
|
||||
|
||||
chapter = {
|
||||
extensionId: 6,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
nextChapter,
|
||||
prevChapter,
|
||||
};
|
||||
|
||||
res.json(chapter);
|
||||
};
|
||||
|
||||
const searchScraper = async (req, res) => {
|
||||
const searchTerm = req.query.s;
|
||||
const searchUrl = `${baseUrl}index.php?s=so&module=book&keyword=`;
|
||||
|
||||
const url = `${searchUrl}${searchTerm}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$("tr").each(function (result) {
|
||||
let novelName = $(this).find("td:nth-child(2)").text();
|
||||
novelName = novelName.replace(/[\t\n]/g, "");
|
||||
|
||||
const novelCover = "https://cdn.novelupdates.com/imgmid/noimagemid.jpg";
|
||||
|
||||
let novelUrl = $(this).find("td:nth-child(2) >").attr("href");
|
||||
if (novelUrl) {
|
||||
novelUrl = novelUrl.slice(1);
|
||||
}
|
||||
|
||||
novel = {
|
||||
extensionId: 6,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
res.json(novels);
|
||||
};
|
||||
|
||||
module.exports = novelhallScraper = {
|
||||
novelsScraper,
|
||||
novelScraper,
|
||||
chapterScraper,
|
||||
searchScraper,
|
||||
};
|
||||
@@ -1,191 +0,0 @@
|
||||
const express = require("express");
|
||||
const cheerio = require("cheerio");
|
||||
const request = require("request");
|
||||
|
||||
const baseUrl = "https://www.novelhall.com/";
|
||||
|
||||
const router = express.Router();
|
||||
router.get("/novels/", (req, res) => {
|
||||
request(baseUrl, (err, response, body) => {
|
||||
if (err) throw err;
|
||||
|
||||
let novels = [];
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
$("div.section1")
|
||||
.find("li")
|
||||
.each(function (result) {
|
||||
let novelName = $(this).find(".book-info > h2 > a").text();
|
||||
let novelCover = $(this).find("img").attr("src");
|
||||
let novelUrl = $(this).find("a").attr("href").substring(1);
|
||||
|
||||
novel = {
|
||||
extensionId: 6,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
res.json(novels);
|
||||
});
|
||||
});
|
||||
|
||||
// Novel
|
||||
|
||||
router.get("/novel/:novelUrl", (req, res) => {
|
||||
let novelUrl = req.params.novelUrl;
|
||||
const url = `${baseUrl}${novelUrl}/`;
|
||||
|
||||
request(url, (err, response, body) => {
|
||||
if (err) throw err;
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.extensionId = 6;
|
||||
|
||||
novel.sourceName = "NovelHall";
|
||||
|
||||
novel.sourceUrl = url;
|
||||
|
||||
novel.novelUrl = `${novelUrl}/`;
|
||||
|
||||
novel.novelName = $("h1").text();
|
||||
|
||||
novel.novelCover = $("div.book-img > img").attr("src");
|
||||
|
||||
novel.novelSummary = $("div.intro")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "");
|
||||
|
||||
novel["Author(s)"] = $("span.blue")
|
||||
.first()
|
||||
.text()
|
||||
.replace("Author：", "");
|
||||
|
||||
novel["Genre(s)"] = $("a.red").text();
|
||||
|
||||
novel["Artist(s)"] = null;
|
||||
|
||||
novel.Status = $("span.blue")
|
||||
.first()
|
||||
.next()
|
||||
.text()
|
||||
.replace("Status：", "");
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
$("div.book-catalog.hidden-xs#morelist")
|
||||
.find("li.post-11")
|
||||
.each(function (result) {
|
||||
let chapterName = $(this).find("a").text();
|
||||
|
||||
let releaseDate = null;
|
||||
|
||||
let chapterUrl = $(this).find("a").attr("href");
|
||||
|
||||
if (chapterUrl) {
|
||||
chapterUrl = chapterUrl.replace(`/${novelUrl}/`, "");
|
||||
}
|
||||
|
||||
novelChapters.push({
|
||||
chapterName,
|
||||
releaseDate,
|
||||
chapterUrl,
|
||||
});
|
||||
});
|
||||
|
||||
novel.novelChapters = novelChapters;
|
||||
|
||||
res.json(novel);
|
||||
});
|
||||
});
|
||||
|
||||
router.get("/novel/:novelUrl/:chapterUrl", (req, res) => {
|
||||
url = `${baseUrl}${req.params.novelUrl}/${req.params.chapterUrl}`;
|
||||
|
||||
request(url, (err, response, body) => {
|
||||
if (err) throw err;
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let chapterName = $("h1").text();
|
||||
|
||||
let chapterText = $("div.entry-content").text();
|
||||
|
||||
let nextChapter = null;
|
||||
|
||||
nextChapter = $('a[rel="next"]').attr("href");
|
||||
|
||||
if (nextChapter) {
|
||||
nextChapter = nextChapter.replace(`/${req.params.novelUrl}/`, "");
|
||||
}
|
||||
|
||||
let prevChapter = null;
|
||||
|
||||
prevChapter = $('a[rel="prev"]').attr("href");
|
||||
|
||||
if (prevChapter) {
|
||||
prevChapter = prevChapter.replace(`/${req.params.novelUrl}/`, "");
|
||||
}
|
||||
|
||||
chapter = {
|
||||
extensionId: 6,
|
||||
novelUrl: `${req.params.novelUrl}/`,
|
||||
chapterUrl: `${req.params.chapterUrl}/`,
|
||||
chapterName,
|
||||
chapterText,
|
||||
nextChapter,
|
||||
prevChapter,
|
||||
};
|
||||
|
||||
res.json(chapter);
|
||||
});
|
||||
});
|
||||
|
||||
router.get("/search/", (req, res) => {
|
||||
let searchTerm = req.query.s;
|
||||
searchTerm = encodeURI(searchTerm);
|
||||
const searchUrl = `https://www.novelhall.com/index.php?s=so&module=book&keyword=${searchTerm}`;
|
||||
|
||||
request(searchUrl, (err, response, body) => {
|
||||
if (err) throw err;
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$("tr").each(function (result) {
|
||||
let novelName = $(this)
|
||||
.find("td:nth-child(2)")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "");
|
||||
|
||||
let novelCover =
|
||||
"https://cdn.novelupdates.com/imgmid/noimagemid.jpg";
|
||||
let novelUrl = $(this).find("td:nth-child(2) >").attr("href");
|
||||
|
||||
if (novelUrl) {
|
||||
novelUrl = novelUrl.slice(1);
|
||||
}
|
||||
|
||||
novel = {
|
||||
extensionId: 6,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
res.json(novels);
|
||||
});
|
||||
});
|
||||
|
||||
module.exports = router;
|
||||
Reference in New Issue
Block a user