Refactor (#4)
* refactor BoxNovel * refactor BoxNovel * refactor BoxNovel * refactor FastNovel * remove comments * try puppeteer for FastNovel * refactor MTLNovel * comment out MTLNovel * refactor MTLNovel * revert FastNovel and MTLNovel * possible fix for vercel application error * clear git cache * clear git cache * rename boxnovel in index.js * refactor NovelFull * refactor Novelhall * refactor rest of the sources * fix WuxiaWorld
This commit is contained in:
Rajarshee Chatterjee
committed by
GitHub
GitHub
parent
70e23ea459
commit
4b33834cfb
@@ -0,0 +1,14 @@
|
||||
const express = require("express");
|
||||
const boxNovelScraper = require("./BoxNovelScraper");
|
||||
|
||||
const router = express.Router();
|
||||
|
||||
router.get("/novels/:pageNo", boxNovelScraper.novelsScraper);
|
||||
|
||||
router.get("/novel/:novelUrl", boxNovelScraper.novelScraper);
|
||||
|
||||
router.get("/novel/:novelUrl/:chapterUrl", boxNovelScraper.chapterScraper);
|
||||
|
||||
router.get("/search/", boxNovelScraper.searchScraper);
|
||||
|
||||
module.exports = router;
|
||||
@@ -0,0 +1,202 @@
|
||||
const cheerio = require("cheerio");
|
||||
const fetch = require("node-fetch");
|
||||
|
||||
const baseUrl = "https://boxnovel.com/novel";
|
||||
const searchUrl = "https://boxnovel.com/";
|
||||
|
||||
const novelsScraper = async (req, res) => {
|
||||
const orderBy = req.params.o;
|
||||
const pageNo = req.params.pageNo;
|
||||
|
||||
let url = `${baseUrl}/page/1/?m_orderby=${orderBy}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".page-item-detail").each(function (result) {
|
||||
const novelName = $(this).find("h5 > a").text();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
|
||||
let novelUrl = $(this).find("h5 > a").attr("href");
|
||||
novelUrl = novelUrl.replace(`${baseUrl}/`, "");
|
||||
|
||||
const novel = {
|
||||
extensionId: 1,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
res.json(novels);
|
||||
};
|
||||
|
||||
const novelScraper = async (req, res) => {
|
||||
const novelUrl = req.params.novelUrl;
|
||||
const url = `${baseUrl}/${novelUrl}/`;
|
||||
|
||||
console.log(url);
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
$(".post-title > h3 > span").remove();
|
||||
|
||||
novel.extensionId = 1;
|
||||
|
||||
novel.sourceName = "BoxNovel";
|
||||
|
||||
novel.sourceUrl = url;
|
||||
|
||||
novel.novelUrl = `${novelUrl}/`;
|
||||
|
||||
novel.novelName = $(".post-title > h3")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
novel.novelCover = $(".summary_image > a > img").attr("src");
|
||||
|
||||
$(".post-content_item").each(function (result) {
|
||||
detailName = $(this)
|
||||
.find(".summary-heading > h5")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
detail = $(this)
|
||||
.find(".summary-content")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
novel[detailName] = detail;
|
||||
});
|
||||
|
||||
$(".description-summary > div.summary__content").find("em").remove();
|
||||
|
||||
novel.novelSummary = $(".description-summary > div.summary__content > div")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "");
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
$(".wp-manga-chapter").each(function (result) {
|
||||
chapterName = $(this)
|
||||
.find("a")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
releaseDate = $(this)
|
||||
.find("span")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
chapterUrl = $(this).find("a").attr("href").replace(url, "");
|
||||
|
||||
novelChapters.push({ chapterName, releaseDate, chapterUrl });
|
||||
});
|
||||
|
||||
novel.novelChapters = novelChapters.reverse();
|
||||
|
||||
res.json(novel);
|
||||
};
|
||||
|
||||
const chapterScraper = async (req, res) => {
|
||||
let novelUrl = req.params.novelUrl;
|
||||
let chapterUrl = req.params.chapterUrl;
|
||||
|
||||
const url = `${baseUrl}/${novelUrl}/${chapterUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
const chapterName = $("div.text-left > h3").text();
|
||||
const chapterText = $(".reading-content").text();
|
||||
|
||||
let nextChapter = null;
|
||||
|
||||
if ($(".nav-next").length) {
|
||||
nextChapter = $(".nav-next")
|
||||
.find("a")
|
||||
.attr("href")
|
||||
.replace(baseUrl + "/" + novelUrl + "/", "");
|
||||
}
|
||||
|
||||
let prevChapter = null;
|
||||
|
||||
if ($(".nav-previous").length) {
|
||||
prevChapter = $(".nav-previous")
|
||||
.find("a")
|
||||
.attr("href")
|
||||
.replace(baseUrl + "/" + novelUrl + "/", "");
|
||||
}
|
||||
|
||||
novelUrl = novelUrl + "/";
|
||||
chapterUrl = chapterUrl + "/";
|
||||
|
||||
const chapter = {
|
||||
extensionId: 1,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
nextChapter,
|
||||
prevChapter,
|
||||
};
|
||||
|
||||
res.json(chapter);
|
||||
};
|
||||
|
||||
const searchScraper = async (req, res) => {
|
||||
const searchTerm = req.query.s;
|
||||
const orderBy = req.query.o;
|
||||
|
||||
const url = `${searchUrl}?s=${searchTerm}&post_type=wp-manga&m_orderby=${orderBy}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".c-tabs-item__content").each(function (result) {
|
||||
const novelName = $(this).find("h4 > a").text();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
|
||||
let novelUrl = $(this).find("h4 > a").attr("href");
|
||||
novelUrl = novelUrlreplace(`${baseUrl}/`, "");
|
||||
|
||||
const novel = {
|
||||
extensionId: 1,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
res.json(novels);
|
||||
};
|
||||
|
||||
module.exports = boxNovelScraper = {
|
||||
novelsScraper,
|
||||
novelScraper,
|
||||
chapterScraper,
|
||||
searchScraper,
|
||||
};
|
||||
@@ -1,247 +0,0 @@
|
||||
const express = require("express");
|
||||
const cheerio = require("cheerio");
|
||||
const request = require("request");
|
||||
|
||||
const router = express.Router();
|
||||
|
||||
const baseUrl = "https://boxnovel.com/novel";
|
||||
|
||||
const searchUrl = "https://boxnovel.com/";
|
||||
|
||||
// Top novels
|
||||
|
||||
function apiCall(reqOps) {
|
||||
return new Promise(function (resolve, reject) {
|
||||
request(reqOps, (err, res, body) => {
|
||||
if (!err && res.statusCode == 200) {
|
||||
if (err) throw err;
|
||||
|
||||
let novels = [];
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
$(".page-item-detail").each(function (result) {
|
||||
novelName = $(this).find("h5 > a").text();
|
||||
novelCover = $(this).find("img").attr("src");
|
||||
novelUrl = $(this)
|
||||
.find("h5 > a")
|
||||
.attr("href")
|
||||
.replace(`${baseUrl}/`, "");
|
||||
novel = {
|
||||
extensionId: 1,
|
||||
novelName: novelName,
|
||||
novelCover: novelCover,
|
||||
novelUrl: novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
resolve(novels);
|
||||
}
|
||||
|
||||
reject(err);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
router.get("/novels/:pageNo/", (req, res) => {
|
||||
orderBy = req.query.o;
|
||||
pageNo = req.params.pageNo;
|
||||
|
||||
let url = `${baseUrl}/page/1/?m_orderby=${orderBy}`;
|
||||
|
||||
let page1, page2, page3, page4;
|
||||
|
||||
apiCall(url)
|
||||
.then((res) => {
|
||||
page1 = res;
|
||||
return apiCall(`${baseUrl}/page/2/?m_orderby=${orderBy}`);
|
||||
})
|
||||
.then((res) => {
|
||||
page2 = res;
|
||||
return apiCall(`${baseUrl}/page/3/?m_orderby=${orderBy}`);
|
||||
})
|
||||
.then((res) => {
|
||||
page3 = res;
|
||||
return apiCall(`${baseUrl}/page/4/?m_orderby=${orderBy}`);
|
||||
})
|
||||
.then((result) => {
|
||||
page4 = result;
|
||||
|
||||
res.json([...page1, ...page2, ...page3, ...page4]);
|
||||
})
|
||||
.catch((err) => {
|
||||
console.log("Error occured in one of the API call: ", err);
|
||||
});
|
||||
});
|
||||
|
||||
// Novel
|
||||
|
||||
router.get("/novel/:novelUrl", (req, res) => {
|
||||
novelUrl = req.params.novelUrl;
|
||||
url = `${baseUrl}/${novelUrl}/`;
|
||||
|
||||
request(url, (err, response, body) => {
|
||||
if (err) throw err;
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
$(".post-title > h3 > span").remove();
|
||||
|
||||
novel.extensionId = 1;
|
||||
|
||||
novel.sourceName = "BoxNovel";
|
||||
|
||||
novel.sourceUrl = url;
|
||||
|
||||
novel.novelUrl = `${novelUrl}/`;
|
||||
|
||||
novel.novelName = $(".post-title > h3")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
novel.novelCover = $(".summary_image > a > img").attr("src");
|
||||
|
||||
$(".post-content_item").each(function (result) {
|
||||
detailName = $(this)
|
||||
.find(".summary-heading > h5")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
detail = $(this)
|
||||
.find(".summary-content")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
novel[detailName] = detail;
|
||||
});
|
||||
|
||||
$(".description-summary > div.summary__content").find("em").remove();
|
||||
|
||||
novel.novelSummary = $(
|
||||
".description-summary > div.summary__content > div"
|
||||
)
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "");
|
||||
|
||||
// if (novel.novelSummary === "") {
|
||||
// novel.novelSummary = $(".c_000").text();
|
||||
// }
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
$(".wp-manga-chapter").each(function (result) {
|
||||
chapterName = $(this)
|
||||
.find("a")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
releaseDate = $(this)
|
||||
.find("span")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
chapterUrl = $(this).find("a").attr("href").replace(url, "");
|
||||
|
||||
novelChapters.push({ chapterName, releaseDate, chapterUrl });
|
||||
});
|
||||
|
||||
novel.novelChapters = novelChapters.reverse();
|
||||
|
||||
res.json(novel);
|
||||
});
|
||||
});
|
||||
|
||||
// Chapter
|
||||
|
||||
router.get("/novel/:novelUrl/:chapterUrl", (req, res) => {
|
||||
url = `${baseUrl}/${req.params.novelUrl}/${req.params.chapterUrl}`;
|
||||
|
||||
request(url, (err, response, body) => {
|
||||
if (err) throw err;
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
chapterNameLower = req.params.chapterUrl.replace("-", " ");
|
||||
|
||||
chapterName =
|
||||
chapterNameLower.charAt(0).toUpperCase() +
|
||||
chapterNameLower.slice(1);
|
||||
|
||||
chapterText = $(".reading-content").text();
|
||||
|
||||
let nextChapter = null;
|
||||
|
||||
if ($(".nav-next").length) {
|
||||
nextChapter = $(".nav-next")
|
||||
.find("a")
|
||||
.attr("href")
|
||||
.replace(baseUrl + "/" + req.params.novelUrl + "/", "");
|
||||
}
|
||||
|
||||
let prevChapter = null;
|
||||
|
||||
if ($(".nav-previous").length) {
|
||||
prevChapter = $(".nav-previous")
|
||||
.find("a")
|
||||
.attr("href")
|
||||
.replace(baseUrl + "/" + req.params.novelUrl + "/", "");
|
||||
}
|
||||
|
||||
chapter = {
|
||||
extensionId: 1,
|
||||
novelUrl: `${req.params.novelUrl}/`,
|
||||
chapterUrl: `${req.params.chapterUrl}/`,
|
||||
chapterName,
|
||||
chapterText,
|
||||
nextChapter,
|
||||
prevChapter,
|
||||
};
|
||||
|
||||
res.json(chapter);
|
||||
});
|
||||
});
|
||||
|
||||
// Search
|
||||
|
||||
router.get("/search/", (req, res) => {
|
||||
searchTerm = req.query.s;
|
||||
orderBy = req.query.o;
|
||||
|
||||
url = `${searchUrl}?s=${searchTerm}&post_type=wp-manga&m_orderby=${orderBy}`;
|
||||
|
||||
request(url, (err, response, body) => {
|
||||
if (err) throw err;
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".c-tabs-item__content").each(function (result) {
|
||||
novelName = $(this).find("h4 > a").text();
|
||||
novelCover = $(this).find("img").attr("src");
|
||||
|
||||
novelUrl = $(this)
|
||||
.find("h4 > a")
|
||||
.attr("href")
|
||||
.replace(`${baseUrl}/`, "");
|
||||
|
||||
novels.push({
|
||||
extensionId: 1,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
});
|
||||
});
|
||||
|
||||
res.json(novels);
|
||||
});
|
||||
});
|
||||
|
||||
module.exports = router;
|
||||
@@ -0,0 +1,14 @@
|
||||
const express = require("express");
|
||||
const fastNovelScraper = require("./FastNovelScraper");
|
||||
|
||||
const router = express.Router();
|
||||
|
||||
router.get("/novels/", fastNovelScraper.novelsScraper);
|
||||
|
||||
router.get("/novel/:novelUrl", fastNovelScraper.novelScraper);
|
||||
|
||||
router.get("/novel/:novelUrl/:chapterUrl", fastNovelScraper.chapterScraper);
|
||||
|
||||
router.get("/search/", fastNovelScraper.searchScraper);
|
||||
|
||||
module.exports = router;
|
||||
@@ -0,0 +1,179 @@
|
||||
const cheerio = require("cheerio");
|
||||
const { scraper } = require("../../helper");
|
||||
|
||||
const baseUrl = "https://fastnovel.net";
|
||||
const searchUrl = `https://fastnovel.net/search/`;
|
||||
|
||||
const novelsScraper = async (req, res) => {
|
||||
const url = `${baseUrl}/list/most-popular.html`;
|
||||
|
||||
const body = await scraper(url);
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".film-item").each(function (result) {
|
||||
const novelName = $(this).find(".name").text();
|
||||
const novelCover = $(this).find(".img").attr("data-original");
|
||||
const novelUrl = $(this).find("a").attr("href").substring(1);
|
||||
|
||||
const novel = {
|
||||
extensionId: 3,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
res.json(novels);
|
||||
};
|
||||
|
||||
const novelScraper = async (req, res) => {
|
||||
const novelUrl = req.params.novelUrl;
|
||||
const url = `${baseUrl}/${novelUrl}/`;
|
||||
|
||||
const body = await scraper(url);
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.extensionId = 3;
|
||||
|
||||
novel.sourceName = "FastNovel";
|
||||
|
||||
novel.sourceUrl = url;
|
||||
|
||||
novel.novelUrl = novelUrl + "/";
|
||||
|
||||
novel.novelName = $("h1").text();
|
||||
|
||||
novel.novelCover = $(".book-cover").attr("data-original");
|
||||
|
||||
novel.novelSummary = $("div.film-content > p").text();
|
||||
|
||||
novel["Author(s)"] = $("li > label")
|
||||
.filter(function () {
|
||||
return $(this).text().trim() === "Author:";
|
||||
})
|
||||
.next()
|
||||
.text();
|
||||
|
||||
novel["Genre(s)"] = $("li")
|
||||
.filter(function () {
|
||||
return $(this).find("label").text().trim() === "Genre:";
|
||||
})
|
||||
.text()
|
||||
.replace("Genre:", "");
|
||||
|
||||
novel["Artist(s)"] = null;
|
||||
|
||||
novel.Status = null;
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
$(".chapter").each(function (result) {
|
||||
const chapterName = $(this).text();
|
||||
const releaseDate = null;
|
||||
let chapterUrl = $(this).attr("href");
|
||||
chapterUrl = chapterUrl.replace(`/${novelUrl}/`, "");
|
||||
|
||||
const novel = {
|
||||
chapterName,
|
||||
releaseDate,
|
||||
chapterUrl,
|
||||
};
|
||||
|
||||
novelChapters.push(novel);
|
||||
});
|
||||
|
||||
novel.novelChapters = novelChapters;
|
||||
|
||||
res.json(novel);
|
||||
};
|
||||
|
||||
const chapterScraper = async (req, res) => {
|
||||
let novelUrl = req.params.novelUrl;
|
||||
let chapterUrl = req.params.chapterUrl;
|
||||
|
||||
const url = `${baseUrl}/${novelUrl}/${chapterUrl}`;
|
||||
|
||||
const body = await scraper(url);
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
const chapterName = $(".episode-name").text();
|
||||
const chapterText = $("#chapter-body").text();
|
||||
|
||||
let nextChapter = null;
|
||||
|
||||
nextChapter = $("div.btn-group > a")
|
||||
.filter(function () {
|
||||
return $(this).text().trim() === "Next Chapter";
|
||||
})
|
||||
.attr("href")
|
||||
.replace("/", "");
|
||||
|
||||
let prevChapter = null;
|
||||
|
||||
prevChapter = $("div.btn-group > a")
|
||||
.filter(function () {
|
||||
return $(this).text().trim() === "Prev Chapter";
|
||||
})
|
||||
.attr("href")
|
||||
.replace("/", "");
|
||||
|
||||
novelUrl = novelUrl + "/";
|
||||
chapterUrl = chapterUrl + "/";
|
||||
|
||||
const chapter = {
|
||||
extensionId: 3,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
nextChapter,
|
||||
prevChapter,
|
||||
};
|
||||
|
||||
res.json(chapter);
|
||||
};
|
||||
|
||||
const searchScraper = async (req, res) => {
|
||||
const searchTerm = req.query.s;
|
||||
|
||||
const url = `${searchUrl}${searchTerm}`;
|
||||
|
||||
const body = await scraper(url);
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".film-item").each(function (result) {
|
||||
const novelName = $(this).find("div.title > p.name").text();
|
||||
const novelCover = $(this).find(".img").attr("data-original");
|
||||
const novelUrl = $(this).find("a").attr("href").substring(1);
|
||||
|
||||
const novel = {
|
||||
extensionId: 3,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
res.json(novels);
|
||||
};
|
||||
|
||||
module.exports = fastNovelScraper = {
|
||||
novelsScraper,
|
||||
novelScraper,
|
||||
chapterScraper,
|
||||
searchScraper,
|
||||
};
|
||||
@@ -1,182 +0,0 @@
|
||||
const express = require("express");
|
||||
const cheerio = require("cheerio");
|
||||
const request = require("request");
|
||||
|
||||
const baseUrl = "https://fastnovel.net";
|
||||
|
||||
const router = express.Router();
|
||||
router.get("/novels/", (req, res) => {
|
||||
url = `https://fastnovel.net/list/most-popular.html`;
|
||||
|
||||
request(url, (err, response, body) => {
|
||||
if (err) throw err;
|
||||
|
||||
let novels = [];
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
$(".film-item").each(function (result) {
|
||||
let novelName = $(this).find("div.title > p.name").text();
|
||||
let novelCover = $(this).find(".img").attr("data-original");
|
||||
let novelUrl = $(this).find("a").attr("href").substring(1);
|
||||
|
||||
novel = {
|
||||
extensionId: 3,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
res.json(novels);
|
||||
});
|
||||
});
|
||||
|
||||
// Novel
|
||||
|
||||
router.get("/novel/:novelUrl", (req, res) => {
|
||||
novelUrl = req.params.novelUrl;
|
||||
url = `${baseUrl}/${novelUrl}/`;
|
||||
|
||||
request(url, (err, response, body) => {
|
||||
if (err) throw err;
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.extensionId = 3;
|
||||
|
||||
novel.sourceName = "FastNovel";
|
||||
|
||||
novel.sourceUrl = url;
|
||||
|
||||
novel.novelUrl = `${novelUrl}/`;
|
||||
|
||||
novel.novelName = $("h1").text();
|
||||
|
||||
novel.novelCover = $(".book-cover").attr("data-original");
|
||||
|
||||
novel.novelSummary = $("div.film-content > p").text();
|
||||
|
||||
novel["Author(s)"] = $("li > label")
|
||||
.filter(function () {
|
||||
return $(this).text().trim() === "Author:";
|
||||
})
|
||||
.next()
|
||||
.text();
|
||||
|
||||
novel["Genre(s)"] = $("li")
|
||||
.filter(function () {
|
||||
return $(this).find("label").text().trim() === "Genre:";
|
||||
})
|
||||
.text()
|
||||
.replace("Genre:", "");
|
||||
|
||||
novel["Artist(s)"] = null;
|
||||
|
||||
novel.Status = null;
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
$(".chapter").each(function (result) {
|
||||
let chapterName = $(this).text();
|
||||
|
||||
let releaseDate = null;
|
||||
|
||||
let chapterUrl = $(this).attr("href");
|
||||
chapterUrl = chapterUrl.replace(`/${novelUrl}/`, "");
|
||||
|
||||
novelChapters.push({
|
||||
chapterName,
|
||||
releaseDate,
|
||||
chapterUrl,
|
||||
});
|
||||
});
|
||||
|
||||
novel.novelChapters = novelChapters;
|
||||
|
||||
res.json(novel);
|
||||
});
|
||||
});
|
||||
|
||||
router.get("/novel/:novelUrl/:chapterUrl", (req, res) => {
|
||||
url = `${baseUrl}/${req.params.novelUrl}/${req.params.chapterUrl}`;
|
||||
|
||||
request(url, (err, response, body) => {
|
||||
if (err) throw err;
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
chapterName = $(".episode-name").text();
|
||||
|
||||
chapterText = $("#chapter-body").text();
|
||||
|
||||
let nextChapter = null;
|
||||
|
||||
nextChapter = $("div.btn-group > a")
|
||||
.filter(function () {
|
||||
return $(this).text().trim() === "Next Chapter";
|
||||
})
|
||||
.attr("href")
|
||||
.replace("/", "");
|
||||
|
||||
let prevChapter = null;
|
||||
|
||||
prevChapter = $("div.btn-group > a")
|
||||
.filter(function () {
|
||||
return $(this).text().trim() === "Prev Chapter";
|
||||
})
|
||||
.attr("href")
|
||||
.replace("/", "");
|
||||
|
||||
chapter = {
|
||||
extensionId: 3,
|
||||
novelUrl: `${req.params.novelUrl}/`,
|
||||
chapterUrl: `${req.params.chapterUrl}/`,
|
||||
chapterName,
|
||||
chapterText,
|
||||
nextChapter,
|
||||
prevChapter,
|
||||
};
|
||||
|
||||
res.json(chapter);
|
||||
});
|
||||
});
|
||||
|
||||
router.get("/search/", (req, res) => {
|
||||
let searchTerm = req.query.s;
|
||||
searchTerm = encodeURI(searchTerm);
|
||||
searchUrl = `https://fastnovel.net/search/`;
|
||||
|
||||
url = `${searchUrl}${searchTerm}`;
|
||||
|
||||
request(url, (err, response, body) => {
|
||||
if (err) throw err;
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".film-item").each(function (result) {
|
||||
let novelName = $(this).find("div.title > p.name").text();
|
||||
let novelCover = $(this).find(".img").attr("data-original");
|
||||
let novelUrl = $(this).find("a").attr("href").substring(1);
|
||||
|
||||
novel = {
|
||||
extensionId: 3,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
res.json(novels);
|
||||
});
|
||||
});
|
||||
|
||||
module.exports = router;
|
||||
@@ -0,0 +1,14 @@
|
||||
const express = require("express");
|
||||
const mtlNovelScraper = require("./MTLNovelScraper");
|
||||
|
||||
const router = express.Router();
|
||||
|
||||
router.get("/novels/", mtlNovelScraper.novelsScraper);
|
||||
|
||||
router.get("/novel/:novelUrl", mtlNovelScraper.novelScraper);
|
||||
|
||||
router.get("/novel/:novelUrl/:chapterUrl", mtlNovelScraper.chapterScraper);
|
||||
|
||||
router.get("/search/", mtlNovelScraper.searchScraper);
|
||||
|
||||
module.exports = router;
|
||||
@@ -0,0 +1,196 @@
|
||||
const cheerio = require("cheerio");
|
||||
const fetch = require("node-fetch");
|
||||
const { scraper } = require("../../helper");
|
||||
|
||||
const baseUrl = "https://mtlnovel.com";
|
||||
|
||||
const novelsScraper = async (req, res) => {
|
||||
const url = `${baseUrl}/alltime-rank/`;
|
||||
|
||||
const body = await scraper(url);
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$("div.box.wide").each(function (result) {
|
||||
const novelName = $(this).find("a.list-title").text().slice(4);
|
||||
const novelCover = $(this).find("amp-img").attr("src");
|
||||
|
||||
let novelUrl = $(this).find("a.list-title").attr("href");
|
||||
novelUrl = novelUrl.replace("https://www.mtlnovel.com/", "");
|
||||
|
||||
const novel = {
|
||||
extensionId: 5,
|
||||
novelUrl,
|
||||
novelName,
|
||||
novelCover,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
res.json(novels);
|
||||
};
|
||||
|
||||
const novelScraper = async (req, res) => {
|
||||
const novelUrl = req.params.novelUrl;
|
||||
const url = `${baseUrl}/${novelUrl}`;
|
||||
|
||||
const body = await scraper(url);
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.extensionId = 5;
|
||||
|
||||
novel.sourceName = "MTLNovel";
|
||||
|
||||
novel.sourceUrl = url;
|
||||
|
||||
novel.novelUrl = novelUrl + "/";
|
||||
|
||||
novel.novelName = $("h1.entry-title").text();
|
||||
|
||||
novel.novelCover = $("amp-img").attr("src");
|
||||
|
||||
novel.novelSummary = $("div.desc > h2").next().text();
|
||||
|
||||
novel["Author(s)"] = $("tr > td")
|
||||
.filter(function () {
|
||||
return $(this).prev().text().trim() === "Author";
|
||||
})
|
||||
.next()
|
||||
.text()
|
||||
.replace("Auhtor:", "");
|
||||
|
||||
novel.Status = $("tr > td")
|
||||
.filter(function () {
|
||||
return $(this).prev().text().trim() === "Status";
|
||||
})
|
||||
.next()
|
||||
.text()
|
||||
.replace("Status:", "");
|
||||
|
||||
novel["Genre(s)"] = $("td")
|
||||
.filter(function () {
|
||||
return $(this).prev().text().trim() === "Genre";
|
||||
})
|
||||
.next()
|
||||
.text()
|
||||
.replace("Genre:", "");
|
||||
|
||||
novel["Artist(s)"] = null;
|
||||
|
||||
const chapterListUrl = url + "/chapter-list/";
|
||||
|
||||
const getChapters = async () => {
|
||||
const body = await scraper(chapterListUrl);
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
$("div.ch-list")
|
||||
.find("a.ch-link")
|
||||
.each(function (result) {
|
||||
const chapterName = $(this).text().replace("~ ", "");
|
||||
const releaseDate = null;
|
||||
|
||||
let chapterUrl = $(this).attr("href");
|
||||
chapterUrl = chapterUrl.replace(
|
||||
`https://www.mtlnovel.com/${novelUrl}/`,
|
||||
""
|
||||
);
|
||||
|
||||
novelChapters.push({
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
releaseDate,
|
||||
});
|
||||
});
|
||||
return novelChapters;
|
||||
};
|
||||
|
||||
novel.novelChapters = await getChapters();
|
||||
|
||||
res.json(novel);
|
||||
};
|
||||
|
||||
const chapterScraper = async (req, res) => {
|
||||
let novelUrl = req.params.novelUrl;
|
||||
let chapterUrl = req.params.chapterUrl;
|
||||
|
||||
const url = `${baseUrl}/${novelUrl}/${chapterUrl}/`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
const chapterName = $("h1.main-title").text();
|
||||
const chapterText = $("div.par").text();
|
||||
|
||||
let nextChapter = null;
|
||||
if ($("a.next").attr("href")) {
|
||||
nextChapter = $("a.next").attr("href").replace(`/${novelUrl}/`, "");
|
||||
}
|
||||
|
||||
let prevChapter = null;
|
||||
if ($("a.prev").attr("href")) {
|
||||
prevChapter = $("a.prev").attr("href").replace(`/${novelUrl}/`, "");
|
||||
}
|
||||
|
||||
chapter = {
|
||||
extensionId: 5,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
nextChapter,
|
||||
prevChapter,
|
||||
};
|
||||
|
||||
res.json(chapter);
|
||||
};
|
||||
|
||||
const searchScraper = async (req, res) => {
|
||||
const searchTerm = req.query.s;
|
||||
|
||||
const result = await fetch(baseUrl, {
|
||||
method: "POST",
|
||||
body: { s: searchTerm },
|
||||
});
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$("div.box").each(function (result) {
|
||||
const novelName = $(this).find("a.list-title").text();
|
||||
const novelCover = $(this).find("amp-img").attr("src");
|
||||
|
||||
let novelUrl = $(this).find("a").attr("href");
|
||||
novelUrl = novelUrl.replace("https://www.mtlnovel.com/", "");
|
||||
|
||||
const novel = {
|
||||
extensionId: 5,
|
||||
novelUrl,
|
||||
novelName,
|
||||
novelCover,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
res.json(novels);
|
||||
};
|
||||
|
||||
module.exports = mtlNovelScraper = {
|
||||
novelsScraper,
|
||||
novelScraper,
|
||||
chapterScraper,
|
||||
searchScraper,
|
||||
};
|
||||
@@ -1,202 +0,0 @@
|
||||
const express = require("express");
|
||||
const cheerio = require("cheerio");
|
||||
const request = require("request");
|
||||
|
||||
const router = express.Router();
|
||||
|
||||
const baseUrl = "https://mtlnovel.com";
|
||||
|
||||
// Top novels
|
||||
|
||||
router.get("/novels/", (req, res) => {
|
||||
const url = `${baseUrl}/alltime-rank/`;
|
||||
|
||||
request(url, (err, response, body) => {
|
||||
if (err) throw err;
|
||||
|
||||
let novels = [];
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
$("div.box.wide").each(function (result) {
|
||||
let novelUrl = $(this).find("a.list-title").attr("href");
|
||||
novelUrl = novelUrl.replace("https://www.mtlnovel.com/", "");
|
||||
|
||||
let novelName = $(this).find("a.list-title").text().slice(4);
|
||||
let novelCover = $(this).find("amp-img").attr("src");
|
||||
|
||||
novel = {
|
||||
extensionId: 5,
|
||||
novelUrl,
|
||||
novelName,
|
||||
novelCover,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
res.json(novels);
|
||||
});
|
||||
});
|
||||
|
||||
router.get("/novel/:novelUrl", (req, res) => {
|
||||
let novelUrl = req.params.novelUrl;
|
||||
const url = `${baseUrl}/${novelUrl}`;
|
||||
console.log(url);
|
||||
|
||||
request(url, (err, response, body) => {
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.extensionId = 5;
|
||||
|
||||
novel.sourceName = "MTLNovel";
|
||||
|
||||
novel.sourceUrl = url;
|
||||
|
||||
novel.novelUrl = `${novelUrl}/`;
|
||||
|
||||
novel.novelName = $("h1.entry-title").text();
|
||||
|
||||
novel.novelCover = $("amp-img").attr("src");
|
||||
|
||||
novel.novelSummary = $("div.desc > h2").next().text();
|
||||
|
||||
novel["Author(s)"] = $("tr > td")
|
||||
.filter(function () {
|
||||
return $(this).prev().text().trim() === "Author";
|
||||
})
|
||||
.next()
|
||||
.text()
|
||||
.replace("Auhtor:", "");
|
||||
|
||||
novel.Status = $("tr > td")
|
||||
.filter(function () {
|
||||
return $(this).prev().text().trim() === "Status";
|
||||
})
|
||||
.next()
|
||||
.text()
|
||||
.replace("Status:", "");
|
||||
|
||||
novel["Genre(s)"] = $("td")
|
||||
.filter(function () {
|
||||
return $(this).prev().text().trim() === "Genre";
|
||||
})
|
||||
.next()
|
||||
.text()
|
||||
.replace("Genre:", "");
|
||||
|
||||
novel["Artist(s)"] = null;
|
||||
|
||||
function callback(res) {
|
||||
console.log(res);
|
||||
novel.novelChapters = res;
|
||||
}
|
||||
|
||||
request(`${url}/chapter-list/`, (err, response, body) => {
|
||||
$ = cheerio.load(body);
|
||||
console.log(`${url}/chapter-list/`);
|
||||
// console.log($("div.ch-list").find("a.ch-link").next().html());
|
||||
let novelChapters = [];
|
||||
|
||||
$("div.ch-list")
|
||||
.find("a.ch-link")
|
||||
.each(function (result) {
|
||||
let chapterName = $(this).text().replace("~ ", "");
|
||||
|
||||
let releaseDate = null;
|
||||
|
||||
let chapterUrl = $(this).attr("href");
|
||||
chapterUrl = chapterUrl.replace(
|
||||
`https://www.mtlnovel.com/${novelUrl}/`,
|
||||
""
|
||||
);
|
||||
|
||||
novelChapters.push({
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
releaseDate,
|
||||
});
|
||||
});
|
||||
|
||||
callback(novelChapters);
|
||||
|
||||
return res.json(novel);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
router.get("/novel/:novelUrl/:chapterUrl", (req, res) => {
|
||||
let novelUrl = req.params.novelUrl;
|
||||
const url = `https://www.mtlnovel.com/${novelUrl}/${req.params.chapterUrl}/`;
|
||||
console.log(url);
|
||||
request(url, (err, response, body) => {
|
||||
if (err) throw err;
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let chapterName = $("h1.main-title").text();
|
||||
|
||||
let chapterText = $("div.par").text();
|
||||
|
||||
let nextChapter = null;
|
||||
if ($("a.next").attr("href")) {
|
||||
nextChapter = $("a.next").attr("href").replace(`/${novelUrl}/`, "");
|
||||
}
|
||||
|
||||
let prevChapter = null;
|
||||
if ($("a.prev").attr("href")) {
|
||||
prevChapter = $("a.prev").attr("href").replace(`/${novelUrl}/`, "");
|
||||
}
|
||||
|
||||
chapter = {
|
||||
extensionId: 5,
|
||||
novelUrl: `${req.params.novelUrl}`,
|
||||
chapterUrl: `${req.params.chapterUrl}`,
|
||||
chapterName,
|
||||
chapterText,
|
||||
nextChapter,
|
||||
prevChapter,
|
||||
};
|
||||
|
||||
res.json(chapter);
|
||||
});
|
||||
});
|
||||
|
||||
router.get("/search/", (req, res) => {
|
||||
searchTerm = req.query.s;
|
||||
|
||||
const searchUrl = "https://www.mtlnovel.com";
|
||||
|
||||
request.post(
|
||||
{ url: searchUrl, form: { s: searchTerm } },
|
||||
(err, response, body) => {
|
||||
if (err) throw err;
|
||||
|
||||
$ = cheerio.load(body);
|
||||
console.log(body);
|
||||
let novels = [];
|
||||
|
||||
$("div.box").each(function (result) {
|
||||
let novelName = $(this).find("a.list-title").text();
|
||||
let novelCover = $(this).find("amp-img").attr("src");
|
||||
let novelUrl = $(this).find("a").attr("href");
|
||||
novelUrl = novelUrl.replace("https://www.mtlnovel.com/", "");
|
||||
|
||||
novel = {
|
||||
extensionId: 5,
|
||||
novelUrl,
|
||||
novelName,
|
||||
novelCover,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
res.json(novels);
|
||||
}
|
||||
);
|
||||
});
|
||||
|
||||
module.exports = router;
|
||||
@@ -0,0 +1,13 @@
|
||||
const express = require("express");
|
||||
const novelFullScraper = require("./NovelFullScraper");
|
||||
const router = express.Router();
|
||||
|
||||
router.get("/novels/", novelFullScraper.novelsScraper);
|
||||
|
||||
router.get("/novel/:novelUrl", novelFullScraper.novelScraper);
|
||||
|
||||
router.get("/novel/:novelUrl/:chapterUrl", novelFullScraper.chapterScraper);
|
||||
|
||||
router.get("/search/", novelFullScraper.searchScraper);
|
||||
|
||||
module.exports = router;
|
||||
@@ -0,0 +1,206 @@
|
||||
const cheerio = require("cheerio");
|
||||
const fetch = require("node-fetch");
|
||||
|
||||
const baseUrl = "https://novelfull.com";
|
||||
|
||||
const novelsScraper = async (req, res) => {
|
||||
const url = `${baseUrl}/most-popular`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$("div.col-truyen-main > div.list-truyen > .row").each(function (result) {
|
||||
const novelName = $(this).find("h3.truyen-title > a").text();
|
||||
|
||||
let novelCover = $(this).find("img").attr("src");
|
||||
novelCover = baseUrl + novelCover;
|
||||
|
||||
let novelUrl = $(this)
|
||||
.find("h3.truyen-title > a")
|
||||
.attr("href")
|
||||
.replace(".html", "")
|
||||
.substring(1);
|
||||
novelUrl = `${novelUrl}/`;
|
||||
|
||||
const novel = {
|
||||
extensionId: 8,
|
||||
novelUrl,
|
||||
novelName,
|
||||
novelCover,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
res.json(novels);
|
||||
};
|
||||
|
||||
const novelScraper = async (req, res) => {
|
||||
const novelUrl = req.params.novelUrl;
|
||||
const url = `${baseUrl}/${novelUrl}.html`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.extensionId = 8;
|
||||
|
||||
novel.sourceName = "NovelFull";
|
||||
|
||||
novel.sourceUrl = url;
|
||||
|
||||
novel.novelUrl = `${novelUrl}/`;
|
||||
|
||||
novel.novelName = $("div.book > img").attr("alt");
|
||||
|
||||
novel.novelCover = baseUrl + $("div.book > img").attr("src");
|
||||
|
||||
novel.novelSummary = $("div.desc-text").text();
|
||||
|
||||
novel["Author(s)"] = $("div.info > div > h3")
|
||||
.filter(function () {
|
||||
return $(this).text().trim() === "Author:";
|
||||
})
|
||||
.siblings()
|
||||
.text();
|
||||
|
||||
novel["Genre(s)"] = $("div.info > div")
|
||||
.filter(function () {
|
||||
return $(this).find("h3").text().trim() === "Genre:";
|
||||
})
|
||||
.text()
|
||||
.replace("Genre:", "");
|
||||
|
||||
novel["Artist(s)"] = null;
|
||||
|
||||
novel.Status = $("div.info > div > h3")
|
||||
.filter(function () {
|
||||
return $(this).text().trim() === "Status:";
|
||||
})
|
||||
.next()
|
||||
.text();
|
||||
|
||||
const novelId = $("#rating").attr("data-novel-id");
|
||||
|
||||
const getChapters = async (novelId) => {
|
||||
const chapterListUrl =
|
||||
baseUrl + "/ajax/chapter-option?novelId=" + novelId;
|
||||
|
||||
const data = await fetch(chapterListUrl);
|
||||
const chapters = await data.text();
|
||||
|
||||
$ = cheerio.load(chapters);
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
$("select > option").each(function (result) {
|
||||
let chapterName = $(this).text();
|
||||
let releaseDate = null;
|
||||
let chapterUrl = $(this).attr("value");
|
||||
chapterUrl = chapterUrl.replace(`/${novelUrl}/`, "");
|
||||
|
||||
novelChapters.push({
|
||||
chapterName,
|
||||
releaseDate,
|
||||
chapterUrl,
|
||||
});
|
||||
});
|
||||
return novelChapters;
|
||||
};
|
||||
|
||||
novel.novelChapters = await getChapters(novelId);
|
||||
|
||||
res.json(novel);
|
||||
};
|
||||
|
||||
const chapterScraper = async (req, res) => {
|
||||
const novelUrl = req.params.novelUrl.replace(".html/", "");
|
||||
const chapterUrl = req.params.chapterUrl;
|
||||
|
||||
const url = `${baseUrl}/${novelUrl}/${chapterUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
const chapterName = $(".chapter-title").attr("title");
|
||||
const chapterText = $("#chapter-content").text();
|
||||
|
||||
let nextChapter = null;
|
||||
if ($("a#next_chap").attr("href")) {
|
||||
nextChapter = $("a#next_chap")
|
||||
.attr("href")
|
||||
.replace(`/${novelUrl}/`, "");
|
||||
}
|
||||
|
||||
let prevChapter = null;
|
||||
if ($("a#prev_chap").attr("href")) {
|
||||
prevChapter = $("a#prev_chap")
|
||||
.attr("href")
|
||||
.replace(`/${novelUrl}/`, "");
|
||||
}
|
||||
|
||||
const chapter = {
|
||||
extensionId: 8,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
nextChapter,
|
||||
prevChapter,
|
||||
};
|
||||
|
||||
res.json(chapter);
|
||||
};
|
||||
|
||||
const searchScraper = async (req, res) => {
|
||||
const searchTerm = req.query.s;
|
||||
const searchUrl = `https://novelfull.com/search?keyword=`;
|
||||
|
||||
const url = `${searchUrl}${searchTerm}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$("div.col-truyen-main > div.list-truyen > .row").each(function (result) {
|
||||
let novelUrl = $(this)
|
||||
.find("h3.truyen-title > a")
|
||||
.attr("href")
|
||||
.replace(".html", "")
|
||||
.substring(1);
|
||||
novelUrl = `${novelUrl}/`;
|
||||
|
||||
const novelName = $(this).find("h3.truyen-title > a").text();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
|
||||
const novel = {
|
||||
extensionId: 8,
|
||||
novelUrl,
|
||||
novelName,
|
||||
novelCover,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
res.json(novels);
|
||||
};
|
||||
|
||||
module.exports = novelFullScraper = {
|
||||
novelsScraper,
|
||||
novelScraper,
|
||||
chapterScraper,
|
||||
searchScraper,
|
||||
};
|
||||
@@ -1,210 +0,0 @@
|
||||
const express = require("express");
|
||||
const cheerio = require("cheerio");
|
||||
const request = require("request");
|
||||
|
||||
const router = express.Router();
|
||||
|
||||
const baseUrl = "https://novelfull.com";
|
||||
|
||||
// Top novels
|
||||
|
||||
router.get("/novels/", (req, res) => {
|
||||
let url = `${baseUrl}/most-popular`;
|
||||
|
||||
console.log(url);
|
||||
|
||||
request(url, (err, response, body) => {
|
||||
if (err) throw err;
|
||||
|
||||
let novels = [];
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
$("div.col-truyen-main > div.list-truyen > .row").each(function (
|
||||
result
|
||||
) {
|
||||
let novelUrl = $(this)
|
||||
.find("h3.truyen-title > a")
|
||||
.attr("href")
|
||||
.replace(".html", "")
|
||||
.substring(1);
|
||||
novelUrl = `${novelUrl}/`;
|
||||
let novelName = $(this).find("h3.truyen-title > a").text();
|
||||
let novelCover = $(this).find("img").attr("src");
|
||||
novelCover = baseUrl + novelCover;
|
||||
|
||||
novel = {
|
||||
extensionId: 8,
|
||||
novelUrl,
|
||||
novelName,
|
||||
novelCover,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
res.json(novels);
|
||||
});
|
||||
});
|
||||
|
||||
router.get("/novel/:novelUrl", (req, res) => {
|
||||
let novelUrl = req.params.novelUrl;
|
||||
url = `${baseUrl}/${novelUrl}.html`;
|
||||
|
||||
request(url, (err, response, body) => {
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.extensionId = 8;
|
||||
|
||||
novel.sourceName = "NovelFull";
|
||||
|
||||
novel.sourceUrl = url;
|
||||
|
||||
novel.novelUrl = `${novelUrl}/`;
|
||||
|
||||
novel.novelName = $("div.book > img").attr("alt");
|
||||
|
||||
novel.novelCover = baseUrl + $("div.book > img").attr("src");
|
||||
|
||||
novel.novelSummary = $("div.desc-text").text();
|
||||
|
||||
novel["Author(s)"] = $("div.info > div > h3")
|
||||
.filter(function () {
|
||||
return $(this).text().trim() === "Author:";
|
||||
})
|
||||
.siblings()
|
||||
.text();
|
||||
|
||||
novel["Genre(s)"] = $("div.info > div")
|
||||
.filter(function () {
|
||||
return $(this).find("h3").text().trim() === "Genre:";
|
||||
})
|
||||
.text()
|
||||
.replace("Genre:", "");
|
||||
|
||||
novel["Artist(s)"] = null;
|
||||
|
||||
novel.Status = $("div.info > div > h3")
|
||||
.filter(function () {
|
||||
return $(this).text().trim() === "Status:";
|
||||
})
|
||||
.next()
|
||||
.text();
|
||||
|
||||
const novelId = $("#rating").attr("data-novel-id");
|
||||
console.log(novelId);
|
||||
|
||||
const chapterListUrl =
|
||||
baseUrl + "/ajax/chapter-option?novelId=" + novelId;
|
||||
|
||||
function callback(res) {
|
||||
console.log(res);
|
||||
novel.novelChapters = res;
|
||||
}
|
||||
|
||||
request(chapterListUrl, (err, response, body) => {
|
||||
$ = cheerio.load(body);
|
||||
let novelChapters = [];
|
||||
|
||||
$("select > option").each(function (result) {
|
||||
let chapterName = $(this).text();
|
||||
let releaseDate = null;
|
||||
let chapterUrl = $(this).attr("value");
|
||||
chapterUrl = chapterUrl.replace(`/${novelUrl}/`, "");
|
||||
|
||||
novelChapters.push({
|
||||
chapterName,
|
||||
releaseDate,
|
||||
chapterUrl,
|
||||
});
|
||||
});
|
||||
callback(novelChapters);
|
||||
|
||||
res.json(novel);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
router.get("/novel/:novelUrl/:chapterUrl", (req, res) => {
|
||||
let novelUrl = req.params.novelUrl.replace(".html/", "");
|
||||
url = `${baseUrl}/${novelUrl}/${req.params.chapterUrl}`;
|
||||
|
||||
request(url, (err, response, body) => {
|
||||
if (err) throw err;
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let chapterName = $(".chapter-title").attr("title");
|
||||
|
||||
let chapterText = $("#chapter-content").text();
|
||||
|
||||
let nextChapter = null;
|
||||
if ($("a#next_chap").attr("href")) {
|
||||
nextChapter = $("a#next_chap")
|
||||
.attr("href")
|
||||
.replace(`/${novelUrl}/`, "");
|
||||
}
|
||||
|
||||
let prevChapter = null;
|
||||
if ($("a#prev_chap").attr("href")) {
|
||||
prevChapter = $("a#prev_chap")
|
||||
.attr("href")
|
||||
.replace(`/${novelUrl}/`, "");
|
||||
}
|
||||
|
||||
chapter = {
|
||||
extensionId: 8,
|
||||
novelUrl: `${req.params.novelUrl}`,
|
||||
chapterUrl: `${req.params.chapterUrl}`,
|
||||
chapterName,
|
||||
chapterText,
|
||||
nextChapter,
|
||||
prevChapter,
|
||||
};
|
||||
|
||||
res.json(chapter);
|
||||
});
|
||||
});
|
||||
|
||||
router.get("/search/", (req, res) => {
|
||||
let searchTerm = req.query.s;
|
||||
searchUrl = `https://novelfull.com/search?keyword=`;
|
||||
|
||||
url = `${searchUrl}${searchTerm}`;
|
||||
|
||||
request(url, (err, response, body) => {
|
||||
if (err) throw err;
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$("div.col-truyen-main > div.list-truyen > .row").each(function (
|
||||
result
|
||||
) {
|
||||
let novelUrl = $(this)
|
||||
.find("h3.truyen-title > a")
|
||||
.attr("href")
|
||||
.replace(".html", "")
|
||||
.substring(1);
|
||||
novelUrl = `${novelUrl}/`;
|
||||
let novelName = $(this).find("h3.truyen-title > a").text();
|
||||
let novelCover = $(this).find("img").attr("src");
|
||||
|
||||
novel = {
|
||||
extensionId: 8,
|
||||
novelUrl,
|
||||
novelName,
|
||||
novelCover,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
res.json(novels);
|
||||
});
|
||||
});
|
||||
|
||||
module.exports = router;
|
||||
@@ -0,0 +1,14 @@
|
||||
const express = require("express");
|
||||
const novelhallScraper = require("./NovelhallScraper");
|
||||
|
||||
const router = express.Router();
|
||||
|
||||
router.get("/novels/", novelhallScraper.novelsScraper);
|
||||
|
||||
router.get("/novel/:novelUrl", novelhallScraper.novelScraper);
|
||||
|
||||
router.get("/novel/:novelUrl/:chapterUrl", novelhallScraper.chapterScraper);
|
||||
|
||||
router.get("/search/", novelhallScraper.searchScraper);
|
||||
|
||||
module.exports = router;
|
||||
@@ -0,0 +1,182 @@
|
||||
const cheerio = require("cheerio");
|
||||
const fetch = require("node-fetch");
|
||||
|
||||
const baseUrl = "https://www.novelhall.com/";
|
||||
|
||||
const novelsScraper = async (req, res) => {
|
||||
const result = await fetch(baseUrl);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$("div.section1")
|
||||
.find("li")
|
||||
.each(function (result) {
|
||||
const novelName = $(this).find(".book-info > h2 > a").text();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
const novelUrl = $(this).find("a").attr("href").substring(1);
|
||||
|
||||
const novel = {
|
||||
extensionId: 6,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
res.json(novels);
|
||||
};
|
||||
|
||||
const novelScraper = async (req, res) => {
|
||||
const novelUrl = req.params.novelUrl;
|
||||
const url = `${baseUrl}${novelUrl}/`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.extensionId = 6;
|
||||
|
||||
novel.sourceName = "NovelHall";
|
||||
|
||||
novel.sourceUrl = url;
|
||||
|
||||
novel.novelUrl = `${novelUrl}/`;
|
||||
|
||||
novel.novelName = $("h1").text();
|
||||
|
||||
novel.novelCover = $("div.book-img > img").attr("src");
|
||||
|
||||
novel.novelSummary = $("div.intro")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "");
|
||||
|
||||
novel["Author(s)"] = $("span.blue").first().text().replace("Author：", "");
|
||||
|
||||
novel["Genre(s)"] = $("a.red").text();
|
||||
|
||||
novel["Artist(s)"] = null;
|
||||
|
||||
novel.Status = $("span.blue").first().next().text().replace("Status：", "");
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
$("div.book-catalog.hidden-xs#morelist")
|
||||
.find("li.post-11")
|
||||
.each(function (result) {
|
||||
let chapterName = $(this).find("a").text();
|
||||
|
||||
let releaseDate = null;
|
||||
|
||||
let chapterUrl = $(this).find("a").attr("href");
|
||||
|
||||
if (chapterUrl) {
|
||||
chapterUrl = chapterUrl.replace(`/${novelUrl}/`, "");
|
||||
}
|
||||
|
||||
const chapter = {
|
||||
chapterName,
|
||||
releaseDate,
|
||||
chapterUrl,
|
||||
};
|
||||
|
||||
novelChapters.push(chapter);
|
||||
});
|
||||
|
||||
novel.novelChapters = novelChapters;
|
||||
|
||||
res.json(novel);
|
||||
};
|
||||
|
||||
const chapterScraper = async (req, res) => {
|
||||
let novelUrl = req.params.novelUrl.replace(".html/", "");
|
||||
let chapterUrl = req.params.chapterUrl;
|
||||
|
||||
const url = `${baseUrl}${novelUrl}/${chapterUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
const chapterName = $("h1").text();
|
||||
const chapterText = $("div.entry-content").text();
|
||||
|
||||
let nextChapter = null;
|
||||
nextChapter = $('a[rel="next"]').attr("href");
|
||||
if (nextChapter) {
|
||||
nextChapter = nextChapter.replace(`/${req.params.novelUrl}/`, "");
|
||||
}
|
||||
|
||||
let prevChapter = null;
|
||||
prevChapter = $('a[rel="prev"]').attr("href");
|
||||
if (prevChapter) {
|
||||
prevChapter = prevChapter.replace(`/${req.params.novelUrl}/`, "");
|
||||
}
|
||||
|
||||
novelUrl = novelUrl + "/";
|
||||
chapterUrl = chapterUrl + "/";
|
||||
|
||||
chapter = {
|
||||
extensionId: 6,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
nextChapter,
|
||||
prevChapter,
|
||||
};
|
||||
|
||||
res.json(chapter);
|
||||
};
|
||||
|
||||
const searchScraper = async (req, res) => {
|
||||
const searchTerm = req.query.s;
|
||||
const searchUrl = `${baseUrl}index.php?s=so&module=book&keyword=`;
|
||||
|
||||
const url = `${searchUrl}${searchTerm}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$("tr").each(function (result) {
|
||||
let novelName = $(this).find("td:nth-child(2)").text();
|
||||
novelName = novelName.replace(/[\t\n]/g, "");
|
||||
|
||||
const novelCover = "https://cdn.novelupdates.com/imgmid/noimagemid.jpg";
|
||||
|
||||
let novelUrl = $(this).find("td:nth-child(2) >").attr("href");
|
||||
if (novelUrl) {
|
||||
novelUrl = novelUrl.slice(1);
|
||||
}
|
||||
|
||||
novel = {
|
||||
extensionId: 6,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
res.json(novels);
|
||||
};
|
||||
|
||||
module.exports = novelhallScraper = {
|
||||
novelsScraper,
|
||||
novelScraper,
|
||||
chapterScraper,
|
||||
searchScraper,
|
||||
};
|
||||
@@ -1,191 +0,0 @@
|
||||
const express = require("express");
|
||||
const cheerio = require("cheerio");
|
||||
const request = require("request");
|
||||
|
||||
const baseUrl = "https://www.novelhall.com/";
|
||||
|
||||
const router = express.Router();
|
||||
router.get("/novels/", (req, res) => {
|
||||
request(baseUrl, (err, response, body) => {
|
||||
if (err) throw err;
|
||||
|
||||
let novels = [];
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
$("div.section1")
|
||||
.find("li")
|
||||
.each(function (result) {
|
||||
let novelName = $(this).find(".book-info > h2 > a").text();
|
||||
let novelCover = $(this).find("img").attr("src");
|
||||
let novelUrl = $(this).find("a").attr("href").substring(1);
|
||||
|
||||
novel = {
|
||||
extensionId: 6,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
res.json(novels);
|
||||
});
|
||||
});
|
||||
|
||||
// Novel
|
||||
|
||||
router.get("/novel/:novelUrl", (req, res) => {
|
||||
let novelUrl = req.params.novelUrl;
|
||||
const url = `${baseUrl}${novelUrl}/`;
|
||||
|
||||
request(url, (err, response, body) => {
|
||||
if (err) throw err;
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.extensionId = 6;
|
||||
|
||||
novel.sourceName = "NovelHall";
|
||||
|
||||
novel.sourceUrl = url;
|
||||
|
||||
novel.novelUrl = `${novelUrl}/`;
|
||||
|
||||
novel.novelName = $("h1").text();
|
||||
|
||||
novel.novelCover = $("div.book-img > img").attr("src");
|
||||
|
||||
novel.novelSummary = $("div.intro")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "");
|
||||
|
||||
novel["Author(s)"] = $("span.blue")
|
||||
.first()
|
||||
.text()
|
||||
.replace("Author：", "");
|
||||
|
||||
novel["Genre(s)"] = $("a.red").text();
|
||||
|
||||
novel["Artist(s)"] = null;
|
||||
|
||||
novel.Status = $("span.blue")
|
||||
.first()
|
||||
.next()
|
||||
.text()
|
||||
.replace("Status：", "");
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
$("div.book-catalog.hidden-xs#morelist")
|
||||
.find("li.post-11")
|
||||
.each(function (result) {
|
||||
let chapterName = $(this).find("a").text();
|
||||
|
||||
let releaseDate = null;
|
||||
|
||||
let chapterUrl = $(this).find("a").attr("href");
|
||||
|
||||
if (chapterUrl) {
|
||||
chapterUrl = chapterUrl.replace(`/${novelUrl}/`, "");
|
||||
}
|
||||
|
||||
novelChapters.push({
|
||||
chapterName,
|
||||
releaseDate,
|
||||
chapterUrl,
|
||||
});
|
||||
});
|
||||
|
||||
novel.novelChapters = novelChapters;
|
||||
|
||||
res.json(novel);
|
||||
});
|
||||
});
|
||||
|
||||
router.get("/novel/:novelUrl/:chapterUrl", (req, res) => {
|
||||
url = `${baseUrl}${req.params.novelUrl}/${req.params.chapterUrl}`;
|
||||
|
||||
request(url, (err, response, body) => {
|
||||
if (err) throw err;
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let chapterName = $("h1").text();
|
||||
|
||||
let chapterText = $("div.entry-content").text();
|
||||
|
||||
let nextChapter = null;
|
||||
|
||||
nextChapter = $('a[rel="next"]').attr("href");
|
||||
|
||||
if (nextChapter) {
|
||||
nextChapter = nextChapter.replace(`/${req.params.novelUrl}/`, "");
|
||||
}
|
||||
|
||||
let prevChapter = null;
|
||||
|
||||
prevChapter = $('a[rel="prev"]').attr("href");
|
||||
|
||||
if (prevChapter) {
|
||||
prevChapter = prevChapter.replace(`/${req.params.novelUrl}/`, "");
|
||||
}
|
||||
|
||||
chapter = {
|
||||
extensionId: 6,
|
||||
novelUrl: `${req.params.novelUrl}/`,
|
||||
chapterUrl: `${req.params.chapterUrl}/`,
|
||||
chapterName,
|
||||
chapterText,
|
||||
nextChapter,
|
||||
prevChapter,
|
||||
};
|
||||
|
||||
res.json(chapter);
|
||||
});
|
||||
});
|
||||
|
||||
router.get("/search/", (req, res) => {
|
||||
let searchTerm = req.query.s;
|
||||
searchTerm = encodeURI(searchTerm);
|
||||
const searchUrl = `https://www.novelhall.com/index.php?s=so&module=book&keyword=${searchTerm}`;
|
||||
|
||||
request(searchUrl, (err, response, body) => {
|
||||
if (err) throw err;
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$("tr").each(function (result) {
|
||||
let novelName = $(this)
|
||||
.find("td:nth-child(2)")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "");
|
||||
|
||||
let novelCover =
|
||||
"https://cdn.novelupdates.com/imgmid/noimagemid.jpg";
|
||||
let novelUrl = $(this).find("td:nth-child(2) >").attr("href");
|
||||
|
||||
if (novelUrl) {
|
||||
novelUrl = novelUrl.slice(1);
|
||||
}
|
||||
|
||||
novel = {
|
||||
extensionId: 6,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
res.json(novels);
|
||||
});
|
||||
});
|
||||
|
||||
module.exports = router;
|
||||
@@ -0,0 +1,17 @@
|
||||
const express = require("express");
|
||||
const readLightNovelScraper = require("./ReadLightNovelScraper.js");
|
||||
|
||||
const router = express.Router();
|
||||
|
||||
router.get("/novels/", readLightNovelScraper.novelsScraper);
|
||||
|
||||
router.get("/novel/:novelUrl", readLightNovelScraper.novelScraper);
|
||||
|
||||
router.get(
|
||||
"/novel/:novelUrl/:chapterUrl/:volumeUrl?",
|
||||
readLightNovelScraper.chapterScraper
|
||||
);
|
||||
|
||||
router.get("/search/", readLightNovelScraper.searchScraper);
|
||||
|
||||
module.exports = router;
|
||||
@@ -0,0 +1,222 @@
|
||||
const cheerio = require("cheerio");
|
||||
const fetch = require("node-fetch");
|
||||
const request = require("request");
|
||||
|
||||
const baseUrl = "https://www.readlightnovel.org";
|
||||
const searchUrl = "https://www.readlightnovel.org/detailed-search";
|
||||
|
||||
const novelsScraper = async (req, res) => {
|
||||
const url = `${baseUrl}/top-novel?change_type=top_rated`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".top-novel-block").each(function (result) {
|
||||
const novelName = $(this).find(".top-novel-header > h2 > a").text();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
|
||||
let novelUrl = $(this)
|
||||
.find(".top-novel-header > h2 > a")
|
||||
.attr("href")
|
||||
.replace(`${baseUrl}/`, "");
|
||||
novelUrl = novelUrl + "/";
|
||||
|
||||
const novel = {
|
||||
extensionId: 2,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
res.json(novels);
|
||||
};
|
||||
|
||||
const novelScraper = async (req, res) => {
|
||||
const novelUrl = req.params.novelUrl;
|
||||
const url = `${baseUrl}/${novelUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.extensionId = 2;
|
||||
|
||||
novel.sourceName = "ReadLightNovel";
|
||||
|
||||
novel.sourceUrl = url;
|
||||
|
||||
novel.novelUrl = novelUrl + "/";
|
||||
|
||||
novel.novelName = $(".block-title > h1").text();
|
||||
|
||||
novel.novelCover = $(".novel-cover > a > img").attr("src");
|
||||
|
||||
$(".novel-detail-item").each(function (result) {
|
||||
const detailName = $(this)
|
||||
.find(".novel-detail-header > h6")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
const detail = $(this).find(".novel-detail-body").text().trim();
|
||||
|
||||
novel[detailName] = detail;
|
||||
});
|
||||
|
||||
novel.Alternative = novel["Alternative Names"];
|
||||
novel.novelSummary = novel.Description;
|
||||
novel["Genre(s)"] = novel.Genre.replace(/[\t\n]/g, ", ");
|
||||
novel.Release = novel.Year;
|
||||
|
||||
delete novel["Alternative Names"];
|
||||
delete novel.Description;
|
||||
delete novel.Genre;
|
||||
delete novel.Year;
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
$(".panel").each(function (res) {
|
||||
let volumeName = $(this).find("h4.panel-title").text();
|
||||
|
||||
$(this)
|
||||
.find("ul.chapter-chs > li")
|
||||
.each(function (result) {
|
||||
chapterName = $(this)
|
||||
.find("a")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
releaseDate = null;
|
||||
|
||||
chapterUrl = $(this).find("a").attr("href");
|
||||
|
||||
chapterUrl = chapterUrl.replace(`${baseUrl}/${novelUrl}/`, "");
|
||||
|
||||
if (volumeName.includes("Volume")) {
|
||||
chapterName = volumeName + " " + chapterName;
|
||||
}
|
||||
|
||||
const chapter = {
|
||||
chapterName,
|
||||
releaseDate,
|
||||
chapterUrl,
|
||||
};
|
||||
|
||||
novelChapters.push(chapter);
|
||||
});
|
||||
});
|
||||
|
||||
novel.novelChapters = novelChapters;
|
||||
|
||||
res.json(novel);
|
||||
};
|
||||
|
||||
const chapterScraper = async (req, res) => {
|
||||
const novelUrl = req.params.novelUrl;
|
||||
let chapterUrl = req.params.chapterUrl;
|
||||
const volumeUrl = req.params.volumeUrl;
|
||||
|
||||
let optionalUrl;
|
||||
|
||||
if (volumeUrl) {
|
||||
optionalUrl = volumeUrl;
|
||||
} else {
|
||||
optionalUrl = "";
|
||||
}
|
||||
|
||||
const url = `${baseUrl}/${novelUrl}/${chapterUrl}/${optionalUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
$(".block-title > h1").find("a").remove();
|
||||
|
||||
const chapterName = $(".block-title > h1").text().replace(" - ", "");
|
||||
const chapterText = $(".desc").text();
|
||||
|
||||
let nextChapter = null;
|
||||
|
||||
if ($("a.next.next-link").length) {
|
||||
nextChapter = $("a.next.next-link")
|
||||
.attr("href")
|
||||
.replace(`${baseUrl}/${novelUrl}/`, "");
|
||||
}
|
||||
|
||||
let prevChapter = null;
|
||||
|
||||
if ($("a.prev.prev-link").length) {
|
||||
prevChapter = $("a.prev.prev-link")
|
||||
.attr("href")
|
||||
.replace(`${baseUrl}/${novelUrl}/`, "");
|
||||
}
|
||||
|
||||
chapterUrl = chapterUrl + "/" + optionalUrl;
|
||||
|
||||
const chapter = {
|
||||
extensionId: 2,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
nextChapter,
|
||||
prevChapter,
|
||||
};
|
||||
|
||||
res.json(chapter);
|
||||
};
|
||||
|
||||
const searchScraper = async (req, res) => {
|
||||
const searchTerm = req.query.s;
|
||||
|
||||
request.post(
|
||||
{ url: searchUrl, form: { keyword: searchTerm, search: 1 } },
|
||||
(err, response, body) => {
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".top-novel-block").each(function (result) {
|
||||
const novelName = $(this)
|
||||
.find(".top-novel-header > h2 > a")
|
||||
.text();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
|
||||
let novelUrl = $(this)
|
||||
.find(".top-novel-header > h2 > a")
|
||||
.attr("href")
|
||||
.replace(`${baseUrl}/`, "");
|
||||
novelUrl = novelUrl + "/";
|
||||
|
||||
const novel = {
|
||||
extensionId: 2,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
res.json(novels);
|
||||
}
|
||||
);
|
||||
};
|
||||
|
||||
module.exports = readLightNovelScraper = {
|
||||
novelsScraper,
|
||||
novelScraper,
|
||||
chapterScraper,
|
||||
searchScraper,
|
||||
};
|
||||
@@ -1,223 +0,0 @@
|
||||
const express = require("express");
|
||||
const cheerio = require("cheerio");
|
||||
const request = require("request");
|
||||
|
||||
const router = express.Router();
|
||||
|
||||
const baseUrl = "https://www.readlightnovel.org";
|
||||
|
||||
const searchUrl = "https://www.readlightnovel.org/detailed-search";
|
||||
|
||||
// Top Novels
|
||||
|
||||
router.get("/novels/", (req, res) => {
|
||||
// orderBy = req.query.o;
|
||||
|
||||
url = `${baseUrl}/top-novel?change_type=top_rated`;
|
||||
|
||||
request(url, (err, response, body) => {
|
||||
if (err) throw err;
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".top-novel-block").each(function (result) {
|
||||
novelName = $(this).find(".top-novel-header > h2 > a").text();
|
||||
novelCover = $(this).find("img").attr("src");
|
||||
novelUrl = $(this)
|
||||
.find(".top-novel-header > h2 > a")
|
||||
.attr("href")
|
||||
.replace(`${baseUrl}/`, "");
|
||||
novel = {
|
||||
extensionId: 2,
|
||||
novelName: novelName,
|
||||
novelCover: novelCover,
|
||||
novelUrl: `${novelUrl}/`,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
res.json(novels);
|
||||
});
|
||||
});
|
||||
|
||||
// Novel
|
||||
|
||||
router.get("/novel/:novelUrl", (req, res) => {
|
||||
novelUrl = req.params.novelUrl;
|
||||
url = `${baseUrl}/${novelUrl}`;
|
||||
|
||||
request(url, (err, response, body) => {
|
||||
if (err) throw err;
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.extensionId = 2;
|
||||
|
||||
novel.sourceName = "ReadLightNovel";
|
||||
|
||||
novel.sourceUrl = url;
|
||||
|
||||
novel.novelUrl = `${novelUrl}/`;
|
||||
|
||||
novel.novelName = $(".block-title > h1").text();
|
||||
|
||||
novel.novelCover = $(".novel-cover > a > img").attr("src");
|
||||
|
||||
$(".novel-detail-item").each(function (result) {
|
||||
detailName = $(this)
|
||||
.find(".novel-detail-header > h6")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
detail = $(this)
|
||||
.find(".novel-detail-body")
|
||||
.text()
|
||||
// .replace(/[\t\n]/g, " ")
|
||||
.trim();
|
||||
|
||||
novel[detailName] = detail;
|
||||
});
|
||||
|
||||
novel.Alternative = novel["Alternative Names"];
|
||||
novel.novelSummary = novel.Description;
|
||||
novel["Genre(s)"] = novel.Genre.replace(/[\t\n]/g, ", ");
|
||||
novel.Release = novel.Year;
|
||||
|
||||
delete novel["Alternative Names"];
|
||||
delete novel.Description;
|
||||
delete novel.Genre;
|
||||
delete novel.Year;
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
$(".panel").each(function (res) {
|
||||
let volumeName = $(this).find("h4.panel-title").text();
|
||||
|
||||
$(this)
|
||||
.find("ul.chapter-chs > li")
|
||||
.each(function (result) {
|
||||
chapterName = $(this)
|
||||
.find("a")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
releaseDate = null;
|
||||
|
||||
chapterUrl = $(this).find("a").attr("href");
|
||||
|
||||
chapterUrl = chapterUrl.replace(
|
||||
`${baseUrl}/${novelUrl}/`,
|
||||
""
|
||||
);
|
||||
|
||||
if (volumeName.includes("Volume")) {
|
||||
chapterName = volumeName + " " + chapterName;
|
||||
}
|
||||
|
||||
novelChapters.push({
|
||||
chapterName,
|
||||
releaseDate,
|
||||
chapterUrl,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
novel.novelChapters = novelChapters;
|
||||
|
||||
res.json(novel);
|
||||
});
|
||||
});
|
||||
|
||||
// Chapter
|
||||
|
||||
router.get("/novel/:novelUrl/:chapterUrl/:volumeUrl?", (req, res) => {
|
||||
req.params.volumeUrl
|
||||
? (optionalUrl = req.params.volumeUrl)
|
||||
: (optionalUrl = "");
|
||||
|
||||
url = `${baseUrl}/${req.params.novelUrl}/${req.params.chapterUrl}/${optionalUrl}`;
|
||||
|
||||
request(url, (err, response, body) => {
|
||||
if (err) throw err;
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
$(".block-title > h1").find("a").remove();
|
||||
|
||||
chapterName = $(".block-title > h1").text().replace(" - ", "");
|
||||
|
||||
// $(".desc").find(".trinity-player-iframe-wrapper").remove();
|
||||
|
||||
chapterText = $(".desc").text();
|
||||
|
||||
let nextChapter = null;
|
||||
|
||||
if ($("a.next.next-link").length) {
|
||||
nextChapter = $("a.next.next-link")
|
||||
.attr("href")
|
||||
.replace(`${baseUrl}/${req.params.novelUrl}/`, "");
|
||||
}
|
||||
|
||||
let prevChapter = null;
|
||||
|
||||
if ($("a.prev.prev-link").length) {
|
||||
prevChapter = $("a.prev.prev-link")
|
||||
.attr("href")
|
||||
.replace(`${baseUrl}/${req.params.novelUrl}/`, "");
|
||||
}
|
||||
|
||||
chapter = {
|
||||
extensionId: 2,
|
||||
novelUrl: req.params.novelUrl,
|
||||
chapterUrl: `${req.params.chapterUrl}/${optionalUrl}`,
|
||||
chapterName,
|
||||
chapterText,
|
||||
nextChapter,
|
||||
prevChapter,
|
||||
};
|
||||
|
||||
res.json(chapter);
|
||||
});
|
||||
});
|
||||
|
||||
router.get("/search/", (req, res) => {
|
||||
searchTerm = req.query.s;
|
||||
|
||||
request.post(
|
||||
{ url: searchUrl, form: { keyword: searchTerm, search: 1 } },
|
||||
(err, response, body) => {
|
||||
if (err) throw err;
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".top-novel-block").each(function (result) {
|
||||
novelName = $(this).find(".top-novel-header > h2 > a").text();
|
||||
novelCover = $(this).find("img").attr("src");
|
||||
novelUrl = $(this)
|
||||
.find(".top-novel-header > h2 > a")
|
||||
.attr("href")
|
||||
.replace(`${baseUrl}/`, "");
|
||||
novel = {
|
||||
extensionId: 2,
|
||||
novelName: novelName,
|
||||
novelCover: novelCover,
|
||||
novelUrl: `${novelUrl}/`,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
res.json(novels);
|
||||
}
|
||||
);
|
||||
});
|
||||
|
||||
module.exports = router;
|
||||
@@ -0,0 +1,14 @@
|
||||
const express = require("express");
|
||||
const readNovelFullScraper = require("./ReadNovelFullScraper");
|
||||
|
||||
const router = express.Router();
|
||||
|
||||
router.get("/novels/", readNovelFullScraper.novelsScraper);
|
||||
|
||||
router.get("/novel/:novelUrl", readNovelFullScraper.novelScraper);
|
||||
|
||||
router.get("/novel/:novelUrl/:chapterUrl", readNovelFullScraper.chapterScraper);
|
||||
|
||||
router.get("/search/", readNovelFullScraper.searchScraper);
|
||||
|
||||
module.exports = router;
|
||||
@@ -0,0 +1,206 @@
|
||||
const cheerio = require("cheerio");
|
||||
const fetch = require("node-fetch");
|
||||
|
||||
const baseUrl = "https://readnovelfull.com";
|
||||
|
||||
const novelsScraper = async (req, res) => {
|
||||
const url = `${baseUrl}/most-popular-novel`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$("div.col-novel-main > div.list-novel > .row").each(function (result) {
|
||||
let novelUrl = $(this)
|
||||
.find("h3.novel-title > a")
|
||||
.attr("href")
|
||||
.replace(".html", "")
|
||||
.substring(1);
|
||||
novelUrl = `${novelUrl}/`;
|
||||
|
||||
const novelName = $(this).find("h3.novel-title > a").text();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
|
||||
const novel = {
|
||||
extensionId: 4,
|
||||
novelUrl,
|
||||
novelName,
|
||||
novelCover,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
res.json(novels);
|
||||
};
|
||||
|
||||
const novelScraper = async (req, res) => {
|
||||
const novelUrl = req.params.novelUrl;
|
||||
const url = `${baseUrl}/${novelUrl}.html`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.extensionId = 4;
|
||||
|
||||
novel.sourceName = "ReadNovelFull";
|
||||
|
||||
novel.sourceUrl = url;
|
||||
|
||||
novel.novelUrl = novelUrl + "/";
|
||||
|
||||
novel.novelName = $("div.book > img").attr("alt");
|
||||
|
||||
novel.novelCover = $("div.book > img").attr("src");
|
||||
|
||||
novel.novelSummary = $("div.desc-text").text();
|
||||
|
||||
novel["Author(s)"] = $("li > h3")
|
||||
.filter(function () {
|
||||
return $(this).text().trim() === "Author:";
|
||||
})
|
||||
.siblings()
|
||||
.text();
|
||||
|
||||
novel["Genre(s)"] = $("li")
|
||||
.filter(function () {
|
||||
return $(this).find("h3").text().trim() === "Genre:";
|
||||
})
|
||||
.text()
|
||||
.replace("Genre:", "");
|
||||
|
||||
novel["Artist(s)"] = null;
|
||||
|
||||
novel.Status = $("li > h3")
|
||||
.filter(function () {
|
||||
return $(this).text().trim() === "Status:";
|
||||
})
|
||||
.siblings()
|
||||
.text();
|
||||
|
||||
const novelId = $("#rating").attr("data-novel-id");
|
||||
|
||||
const getChapters = async (novelId) => {
|
||||
const chapterListUrl =
|
||||
"https://readnovelfull.com/ajax/chapter-archive?novelId=" + novelId;
|
||||
|
||||
const data = await fetch(chapterListUrl);
|
||||
const chapters = await data.text();
|
||||
|
||||
$ = cheerio.load(chapters);
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
$(".panel-body")
|
||||
.find("li")
|
||||
.each(function (result) {
|
||||
let chapterName = $(this).find("a").attr("title");
|
||||
let releaseDate = null;
|
||||
let chapterUrl = $(this).find("a").attr("href");
|
||||
chapterUrl = chapterUrl.replace(`/${novelUrl}/`, "");
|
||||
|
||||
novelChapters.push({
|
||||
chapterName,
|
||||
releaseDate,
|
||||
chapterUrl,
|
||||
});
|
||||
});
|
||||
return novelChapters;
|
||||
};
|
||||
|
||||
novel.novelChapters = await getChapters(novelId);
|
||||
|
||||
res.json(novel);
|
||||
};
|
||||
|
||||
const chapterScraper = async (req, res) => {
|
||||
let novelUrl = req.params.novelUrl.replace(".html/", "");
|
||||
let chapterUrl = req.params.chapterUrl;
|
||||
|
||||
const url = `${baseUrl}/${novelUrl}/${chapterUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
const chapterName = $(".chr-title").attr("title");
|
||||
const chapterText = $("#chr-content").text();
|
||||
|
||||
let nextChapter = null;
|
||||
if ($("a#next_chap").attr("href")) {
|
||||
nextChapter = $("a#next_chap")
|
||||
.attr("href")
|
||||
.replace(`/${novelUrl}/`, "");
|
||||
}
|
||||
|
||||
let prevChapter = null;
|
||||
if ($("a#prev_chap").attr("href")) {
|
||||
prevChapter = $("a#prev_chap")
|
||||
.attr("href")
|
||||
.replace(`/${novelUrl}/`, "");
|
||||
}
|
||||
|
||||
const chapter = {
|
||||
extensionId: 4,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
nextChapter,
|
||||
prevChapter,
|
||||
};
|
||||
|
||||
res.json(chapter);
|
||||
};
|
||||
|
||||
const searchScraper = async (req, res) => {
|
||||
const searchTerm = req.query.s;
|
||||
const searchUrl = `https://readnovelfull.com/search?keyword=`;
|
||||
|
||||
const url = `${searchUrl}${searchTerm}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$("div.col-novel-main > div.list-novel > .row").each(function (result) {
|
||||
let novelUrl = $(this)
|
||||
.find("h3.novel-title > a")
|
||||
.attr("href")
|
||||
.replace(".html", "")
|
||||
.substring(1);
|
||||
novelUrl = `${novelUrl}/`;
|
||||
|
||||
const novelName = $(this).find("h3.novel-title > a").text();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
|
||||
const novel = {
|
||||
extensionId: 4,
|
||||
novelUrl,
|
||||
novelName,
|
||||
novelCover,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
res.json(novels);
|
||||
};
|
||||
|
||||
module.exports = readNovelFullScraper = {
|
||||
novelsScraper,
|
||||
novelScraper,
|
||||
chapterScraper,
|
||||
searchScraper,
|
||||
};
|
||||
@@ -1,204 +0,0 @@
|
||||
const express = require("express");
|
||||
const cheerio = require("cheerio");
|
||||
const request = require("request");
|
||||
|
||||
const router = express.Router();
|
||||
|
||||
const baseUrl = "https://readnovelfull.com";
|
||||
|
||||
// Top novels
|
||||
|
||||
router.get("/novels/", (req, res) => {
|
||||
url = `${baseUrl}/most-popular-novel`;
|
||||
|
||||
request(url, (err, response, body) => {
|
||||
if (err) throw err;
|
||||
|
||||
let novels = [];
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
$("div.col-novel-main > div.list-novel > .row").each(function (result) {
|
||||
let novelUrl = $(this)
|
||||
.find("h3.novel-title > a")
|
||||
.attr("href")
|
||||
.replace(".html", "")
|
||||
.substring(1);
|
||||
novelUrl = `${novelUrl}/`;
|
||||
let novelName = $(this).find("h3.novel-title > a").text();
|
||||
let novelCover = $(this).find("img").attr("src");
|
||||
|
||||
novel = {
|
||||
extensionId: 4,
|
||||
novelUrl,
|
||||
novelName,
|
||||
novelCover,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
res.json(novels);
|
||||
});
|
||||
});
|
||||
|
||||
router.get("/novel/:novelUrl", (req, res) => {
|
||||
let novelUrl = req.params.novelUrl;
|
||||
url = `${baseUrl}/${novelUrl}.html`;
|
||||
|
||||
request(url, (err, response, body) => {
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.extensionId = 4;
|
||||
|
||||
novel.sourceName = "ReadNovelFull";
|
||||
|
||||
novel.sourceUrl = url;
|
||||
|
||||
novel.novelUrl = `${novelUrl}/`;
|
||||
|
||||
novel.novelName = $("div.book > img").attr("alt");
|
||||
|
||||
novel.novelCover = $("div.book > img").attr("src");
|
||||
|
||||
novel.novelSummary = $("div.desc-text").text();
|
||||
|
||||
novel["Author(s)"] = $("li > h3")
|
||||
.filter(function () {
|
||||
return $(this).text().trim() === "Author:";
|
||||
})
|
||||
.siblings()
|
||||
.text();
|
||||
|
||||
novel["Genre(s)"] = $("li")
|
||||
.filter(function () {
|
||||
return $(this).find("h3").text().trim() === "Genre:";
|
||||
})
|
||||
.text()
|
||||
.replace("Genre:", "");
|
||||
|
||||
novel["Artist(s)"] = null;
|
||||
|
||||
novel.Status = $("li > h3")
|
||||
.filter(function () {
|
||||
return $(this).text().trim() === "Status:";
|
||||
})
|
||||
.siblings()
|
||||
.text();
|
||||
|
||||
const novelId = $("#rating").attr("data-novel-id");
|
||||
|
||||
const chapterListUrl =
|
||||
"https://readnovelfull.com/ajax/chapter-archive?novelId=" + novelId;
|
||||
|
||||
function callback(res) {
|
||||
console.log(res);
|
||||
novel.novelChapters = res;
|
||||
}
|
||||
|
||||
request(chapterListUrl, (err, response, body) => {
|
||||
$ = cheerio.load(body);
|
||||
let novelChapters = [];
|
||||
|
||||
$(".panel-body")
|
||||
.find("li")
|
||||
.each(function (result) {
|
||||
let chapterName = $(this).find("a").attr("title");
|
||||
let releaseDate = null;
|
||||
let chapterUrl = $(this).find("a").attr("href");
|
||||
chapterUrl = chapterUrl.replace(`/${novelUrl}/`, "");
|
||||
|
||||
novelChapters.push({
|
||||
chapterName,
|
||||
releaseDate,
|
||||
chapterUrl,
|
||||
});
|
||||
});
|
||||
callback(novelChapters);
|
||||
|
||||
res.json(novel);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
router.get("/novel/:novelUrl/:chapterUrl", (req, res) => {
|
||||
let novelUrl = req.params.novelUrl.replace(".html/", "");
|
||||
url = `${baseUrl}/${novelUrl}/${req.params.chapterUrl}`;
|
||||
|
||||
request(url, (err, response, body) => {
|
||||
if (err) throw err;
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let chapterName = $(".chr-title").attr("title");
|
||||
|
||||
chapterText = $("#chr-content").text();
|
||||
|
||||
let nextChapter = null;
|
||||
if ($("a#next_chap").attr("href")) {
|
||||
nextChapter = $("a#next_chap")
|
||||
.attr("href")
|
||||
.replace(`/${novelUrl}/`, "");
|
||||
}
|
||||
|
||||
let prevChapter = null;
|
||||
if ($("a#prev_chap").attr("href")) {
|
||||
prevChapter = $("a#prev_chap")
|
||||
.attr("href")
|
||||
.replace(`/${novelUrl}/`, "");
|
||||
}
|
||||
|
||||
chapter = {
|
||||
extensionId: 4,
|
||||
novelUrl: `${req.params.novelUrl}`,
|
||||
chapterUrl: `${req.params.chapterUrl}`,
|
||||
chapterName,
|
||||
chapterText,
|
||||
nextChapter,
|
||||
prevChapter,
|
||||
};
|
||||
|
||||
res.json(chapter);
|
||||
});
|
||||
});
|
||||
|
||||
router.get("/search/", (req, res) => {
|
||||
let searchTerm = req.query.s;
|
||||
searchUrl = `https://readnovelfull.com/search?keyword=`;
|
||||
|
||||
url = `${searchUrl}${searchTerm}`;
|
||||
|
||||
request(url, (err, response, body) => {
|
||||
if (err) throw err;
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$("div.col-novel-main > div.list-novel > .row").each(function (result) {
|
||||
let novelUrl = $(this)
|
||||
.find("h3.novel-title > a")
|
||||
.attr("href")
|
||||
.replace(".html", "")
|
||||
.substring(1);
|
||||
novelUrl = `${novelUrl}/`;
|
||||
let novelName = $(this).find("h3.novel-title > a").text();
|
||||
let novelCover = $(this).find("img").attr("src");
|
||||
|
||||
novel = {
|
||||
extensionId: 4,
|
||||
novelUrl,
|
||||
novelName,
|
||||
novelCover,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
res.json(novels);
|
||||
});
|
||||
});
|
||||
|
||||
module.exports = router;
|
||||
@@ -0,0 +1,14 @@
|
||||
const express = require("express");
|
||||
const wuxiaWorldScraper = require("./WuxiaWorldScraper");
|
||||
|
||||
const router = express.Router();
|
||||
|
||||
router.get("/novels/", wuxiaWorldScraper.novelsScraper);
|
||||
|
||||
router.get("/novel/:novelUrl", wuxiaWorldScraper.novelScraper);
|
||||
|
||||
router.get("/novel/:novelUrl/:chapterUrl", wuxiaWorldScraper.chapterScraper);
|
||||
|
||||
router.get("/search/", wuxiaWorldScraper.searchScraper);
|
||||
|
||||
module.exports = router;
|
||||
@@ -0,0 +1,180 @@
|
||||
const cheerio = require("cheerio");
|
||||
const fetch = require("node-fetch");
|
||||
|
||||
const baseUrl = "https://www.wuxiaworld.com/";
|
||||
|
||||
const novelsScraper = async (req, res) => {
|
||||
const url = `${baseUrl}api/novels`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const data = await result.json();
|
||||
|
||||
let novels = [];
|
||||
|
||||
data.items.map((novel) => {
|
||||
let novelName = novel.name;
|
||||
let novelCover = novel.coverUrl;
|
||||
let novelUrl = novel.slug;
|
||||
novelUrl = novelUrl + "/";
|
||||
|
||||
novels.push({
|
||||
extensionId: 7,
|
||||
novelUrl,
|
||||
novelName,
|
||||
novelCover,
|
||||
});
|
||||
});
|
||||
|
||||
res.json(novels);
|
||||
};
|
||||
|
||||
const novelScraper = async (req, res) => {
|
||||
const novelUrl = req.params.novelUrl;
|
||||
const url = `${baseUrl}novel/${novelUrl}/`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.extensionId = 7;
|
||||
|
||||
novel.sourceName = "WuxiaWorld";
|
||||
|
||||
novel.sourceUrl = url;
|
||||
|
||||
novel.novelUrl = `${novelUrl}/`;
|
||||
|
||||
novel.novelName = $("h2").text();
|
||||
|
||||
novel.novelCover = $("img.img-thumbnail").attr("src");
|
||||
|
||||
novel.novelSummary = $(".fr-view > p").text();
|
||||
|
||||
novel["Author(s)"] = $("div > dt")
|
||||
.filter(function () {
|
||||
return $(this).text().trim() === "Author:";
|
||||
})
|
||||
.next()
|
||||
.text();
|
||||
|
||||
let genres = [];
|
||||
|
||||
$(".genres")
|
||||
.find("div")
|
||||
.each(function (res) {
|
||||
genres.push($(this).find("a").text());
|
||||
});
|
||||
|
||||
novel["Genre(s)"] = genres.join(",");
|
||||
|
||||
novel["Artist(s)"] = null;
|
||||
|
||||
novel.Status = null;
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
$(".chapter-item").each(function (result) {
|
||||
let chapterName = $(this).text();
|
||||
chapterName = chapterName.replace(/[\t\n]/g, "");
|
||||
|
||||
const releaseDate = null;
|
||||
|
||||
let chapterUrl = $(this).find("a").attr("href");
|
||||
chapterUrl = chapterUrl.replace(`/novel/${novelUrl}/`, "");
|
||||
|
||||
const chapter = {
|
||||
chapterName,
|
||||
releaseDate,
|
||||
chapterUrl,
|
||||
};
|
||||
|
||||
novelChapters.push(chapter);
|
||||
});
|
||||
|
||||
novel.novelChapters = novelChapters;
|
||||
|
||||
res.json(novel);
|
||||
};
|
||||
|
||||
const chapterScraper = async (req, res) => {
|
||||
let novelUrl = req.params.novelUrl;
|
||||
let chapterUrl = req.params.chapterUrl;
|
||||
|
||||
let url = `${baseUrl}novel/${novelUrl}/${chapterUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let chapterName = $("#sidebar-toggler-container").next().text();
|
||||
chapterName = chapterName.replace(/[\t\n]/g, "");
|
||||
|
||||
const chapterText = $("#chapter-content").text();
|
||||
|
||||
let nextChapter = null;
|
||||
|
||||
if ($("li.next > a").attr("href") !== "#") {
|
||||
nextChapter = $("li.next > a")
|
||||
.attr("href")
|
||||
.replace(`/novel/${novelUrl}`, "");
|
||||
}
|
||||
|
||||
let prevChapter = null;
|
||||
|
||||
if ($("li.prev > a").attr("href") !== "#") {
|
||||
prevChapter = $("li.prev > a")
|
||||
.attr("href")
|
||||
.replace(`/novel/${novelUrl}`, "");
|
||||
}
|
||||
|
||||
const chapter = {
|
||||
extensionId: 7,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
nextChapter,
|
||||
prevChapter,
|
||||
};
|
||||
|
||||
res.json(chapter);
|
||||
};
|
||||
|
||||
const searchScraper = async (req, res) => {
|
||||
const searchTerm = req.query.s;
|
||||
const searchUrl = `https://www.wuxiaworld.com/api/novels/search?query=`;
|
||||
|
||||
const url = `${searchUrl}${searchTerm}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const data = await result.json();
|
||||
|
||||
let novels = [];
|
||||
|
||||
data.items.map((novel) => {
|
||||
let novelName = novel.name;
|
||||
let novelCover = novel.coverUrl;
|
||||
let novelUrl = novel.slug;
|
||||
novelUrl = novelUrl + "/";
|
||||
|
||||
novels.push({
|
||||
extensionId: 7,
|
||||
novelUrl,
|
||||
novelName,
|
||||
novelCover,
|
||||
});
|
||||
});
|
||||
|
||||
res.json(novels);
|
||||
};
|
||||
|
||||
module.exports = wuxiaWorldScraper = {
|
||||
novelsScraper,
|
||||
novelScraper,
|
||||
chapterScraper,
|
||||
searchScraper,
|
||||
};
|
||||
@@ -1,185 +0,0 @@
|
||||
const express = require("express");
|
||||
const cheerio = require("cheerio");
|
||||
const request = require("request");
|
||||
|
||||
const baseUrl = "https://www.wuxiaworld.com/";
|
||||
|
||||
const router = express.Router();
|
||||
|
||||
router.get("/novels/", async (req, res) => {
|
||||
let url = `https://www.wuxiaworld.com/api/novels`;
|
||||
|
||||
request(url, (err, response, body) => {
|
||||
if (err) throw err;
|
||||
|
||||
let novels = [];
|
||||
|
||||
let data = JSON.parse(body);
|
||||
data = data.items;
|
||||
|
||||
data.map((novel) => {
|
||||
let novelName = novel.name;
|
||||
let novelCover = novel.coverUrl;
|
||||
let novelUrl = novel.slug;
|
||||
novelUrl = novelUrl + "/";
|
||||
|
||||
novels.push({
|
||||
extensionId: 7,
|
||||
novelUrl,
|
||||
novelName,
|
||||
novelCover,
|
||||
});
|
||||
});
|
||||
res.json(novels);
|
||||
});
|
||||
});
|
||||
|
||||
// Novel
|
||||
|
||||
router.get("/novel/:novelUrl", async (req, res) => {
|
||||
let novelUrl = req.params.novelUrl;
|
||||
let url = `${baseUrl}novel/${novelUrl}/`;
|
||||
|
||||
request(url, (err, response, body) => {
|
||||
if (err) throw err;
|
||||
const $ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.extensionId = 7;
|
||||
|
||||
novel.sourceName = "WuxiaWorld";
|
||||
|
||||
novel.sourceUrl = url;
|
||||
|
||||
novel.novelUrl = `${novelUrl}/`;
|
||||
|
||||
novel.novelName = $("h2").text();
|
||||
|
||||
novel.novelCover = $("img.img-thumbnail").attr("src");
|
||||
|
||||
novel.novelSummary = $(".fr-view > p").text();
|
||||
|
||||
novel["Author(s)"] = $("div > dt")
|
||||
.filter(function () {
|
||||
return $(this).text().trim() === "Author:";
|
||||
})
|
||||
.next()
|
||||
.text();
|
||||
|
||||
let genres = [];
|
||||
|
||||
$(".genres")
|
||||
.find("div")
|
||||
.each(function (res) {
|
||||
genres.push($(this).find("a").text());
|
||||
});
|
||||
|
||||
novel["Genre(s)"] = genres.join(",");
|
||||
|
||||
novel["Artist(s)"] = null;
|
||||
|
||||
novel.Status = null;
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
$(".chapter-item").each(function (result) {
|
||||
let chapterName = $(this).text();
|
||||
chapterName = chapterName.replace(/[\t\n]/g, "");
|
||||
|
||||
let releaseDate = null;
|
||||
let chapterUrl = $(this).find("a").attr("href");
|
||||
chapterUrl = chapterUrl.replace(`/novel/${novelUrl}/`, "");
|
||||
|
||||
novelChapters.push({
|
||||
chapterName,
|
||||
releaseDate,
|
||||
chapterUrl,
|
||||
});
|
||||
});
|
||||
|
||||
novel.novelChapters = novelChapters;
|
||||
|
||||
res.json(novel);
|
||||
});
|
||||
});
|
||||
|
||||
router.get("/novel/:novelUrl/:chapterUrl", async (req, res) => {
|
||||
let url = `${baseUrl}novel/${req.params.novelUrl}/${req.params.chapterUrl}`;
|
||||
|
||||
const novelUrl = `${req.params.novelUrl}/`;
|
||||
const chapterUrl = `${req.params.chapterUrl}/`;
|
||||
|
||||
request(url, (err, response, body) => {
|
||||
if (err) throw err;
|
||||
const $ = cheerio.load(body);
|
||||
|
||||
let chapterName = $("#sidebar-toggler-container").next().text();
|
||||
chapterName = chapterName.replace(/[\t\n]/g, "");
|
||||
|
||||
let chapterText = $("#chapter-content").text();
|
||||
|
||||
let nextChapter = null;
|
||||
|
||||
if ($("li.next > a").attr("href") !== "#") {
|
||||
nextChapter = $("li.next > a")
|
||||
.attr("href")
|
||||
.replace(`/novel/${novelUrl}`, "");
|
||||
}
|
||||
|
||||
let prevChapter = null;
|
||||
|
||||
if ($("li.prev > a").attr("href") !== "#") {
|
||||
prevChapter = $("li.prev > a")
|
||||
.attr("href")
|
||||
.replace(`/novel/${novelUrl}`, "");
|
||||
}
|
||||
chapter = {
|
||||
extensionId: 7,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
nextChapter,
|
||||
prevChapter,
|
||||
};
|
||||
|
||||
res.json(chapter);
|
||||
});
|
||||
});
|
||||
|
||||
router.get("/search/", (req, res) => {
|
||||
let searchTerm = req.query.s;
|
||||
searchTerm = encodeURI(searchTerm);
|
||||
const searchUrl = `https://www.wuxiaworld.com/api/novels/search?query=`;
|
||||
|
||||
const url = `${searchUrl}${searchTerm}`;
|
||||
|
||||
request(url, (err, response, body) => {
|
||||
if (err) throw err;
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
let data = JSON.parse(body);
|
||||
data = data.items;
|
||||
|
||||
data.map((novel) => {
|
||||
let novelName = novel.name;
|
||||
let novelCover = novel.coverUrl;
|
||||
let novelUrl = novel.slug;
|
||||
novelUrl = novelUrl + "/";
|
||||
|
||||
novels.push({
|
||||
extensionId: 7,
|
||||
novelUrl,
|
||||
novelName,
|
||||
novelCover,
|
||||
});
|
||||
});
|
||||
res.json(novels);
|
||||
});
|
||||
});
|
||||
|
||||
module.exports = router;
|
||||
@@ -1,76 +0,0 @@
|
||||
extensions = [
|
||||
{
|
||||
sourceId: 1,
|
||||
sourceUrl: "https://boxnovel.com/",
|
||||
sourceName: "BoxNovel",
|
||||
sourceCover:
|
||||
"https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/boxnovel/icon.png?raw=true",
|
||||
sourceLanguage: "English",
|
||||
status: 1,
|
||||
},
|
||||
{
|
||||
sourceId: 2,
|
||||
sourceUrl: "https://www.readlightnovel.org/",
|
||||
sourceName: "ReadLightNovel",
|
||||
sourceCover:
|
||||
"https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/readlightnovel/icon.png?raw=true",
|
||||
sourceLanguage: "English",
|
||||
status: 1,
|
||||
},
|
||||
{
|
||||
sourceId: 3,
|
||||
sourceUrl: "https://fastnovel.net/",
|
||||
sourceName: "FastNovel",
|
||||
sourceCover:
|
||||
"https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/fastnovel/icon.png?raw=true",
|
||||
sourceLanguage: "English",
|
||||
status: 0,
|
||||
},
|
||||
{
|
||||
sourceId: 4,
|
||||
sourceUrl: "https://readnovelfull.com/",
|
||||
sourceName: "ReadNovelFull",
|
||||
sourceCover:
|
||||
"https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/readnovelfull/icon.png?raw=true",
|
||||
sourceLanguage: "English",
|
||||
status: 1,
|
||||
},
|
||||
{
|
||||
sourceId: 5,
|
||||
sourceUrl: "https://www.mtlnovel.com/",
|
||||
sourceName: "MTLNovel",
|
||||
sourceCover:
|
||||
"https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/mtlnovel/icon.png?raw=true",
|
||||
sourceLanguage: "English",
|
||||
status: 0,
|
||||
},
|
||||
{
|
||||
sourceId: 6,
|
||||
sourceUrl: "https://www.novelhall.com/",
|
||||
sourceName: "NovelHall",
|
||||
sourceCover:
|
||||
"https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/novelhall/icon.png?raw=true",
|
||||
sourceLanguage: "English",
|
||||
status: 1,
|
||||
},
|
||||
{
|
||||
sourceId: 7,
|
||||
sourceUrl: "https://www.wuxiaworld.com/",
|
||||
sourceName: "WuxiaWorld",
|
||||
sourceCover:
|
||||
"https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/wuxiaworld/icon.png?raw=true",
|
||||
sourceLanguage: "English",
|
||||
status: 1,
|
||||
},
|
||||
{
|
||||
sourceId: 8,
|
||||
sourceUrl: "https://novelfull.com/",
|
||||
sourceName: "NovelFull",
|
||||
sourceCover:
|
||||
"https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/novelfull/icon.png?raw=true",
|
||||
sourceLanguage: "English",
|
||||
status: 1,
|
||||
},
|
||||
];
|
||||
|
||||
module.exports = extensions;
|
||||
+2
-2
@@ -1,7 +1,7 @@
|
||||
const chromium = require("chrome-aws-lambda");
|
||||
const puppeteer = require("puppeteer");
|
||||
|
||||
async function scraper(url) {
|
||||
const scraper = async (url) => {
|
||||
const browser = await puppeteer.launch({
|
||||
args: [...chromium.args, "--hide-scrollbars", "--disable-web-security"],
|
||||
defaultViewport: chromium.defaultViewport,
|
||||
@@ -15,6 +15,6 @@ async function scraper(url) {
|
||||
const body = await page.content();
|
||||
|
||||
return body;
|
||||
}
|
||||
};
|
||||
|
||||
module.exports = { scraper };
|
||||
|
||||
Reference in New Issue
Block a user