use WuxiaWorld api & cheerio instead of puppeteer
This commit is contained in:
Rajarshee Chatterjee
@@ -1,39 +1,37 @@
|
|||||||
const express = require("express");
|
const express = require("express");
|
||||||
const cheerio = require("cheerio");
|
const cheerio = require("cheerio");
|
||||||
const request = require("request");
|
const request = require("request");
|
||||||
const chromium = require("chrome-aws-lambda");
|
|
||||||
const puppeteer = require("puppeteer");
|
|
||||||
const { scraper } = require("../../helper");
|
|
||||||
|
|
||||||
const baseUrl = "https://www.wuxiaworld.com/";
|
const baseUrl = "https://www.wuxiaworld.com/";
|
||||||
|
|
||||||
const router = express.Router();
|
const router = express.Router();
|
||||||
|
|
||||||
router.get("/novels/", async (req, res) => {
|
router.get("/novels/", async (req, res) => {
|
||||||
let url = `https://www.wuxiaworld.com/novels`;
|
let url = `https://www.wuxiaworld.com/api/novels`;
|
||||||
const body = await scraper(url);
|
|
||||||
|
|
||||||
let novels = [];
|
request(url, (err, response, body) => {
|
||||||
|
if (err) throw err;
|
||||||
|
|
||||||
const $ = cheerio.load(body);
|
let novels = [];
|
||||||
|
|
||||||
$(".novel-item").each(function (result) {
|
let data = JSON.parse(body);
|
||||||
let novelName = $(this).find("div.novel-title").text();
|
data = data.items;
|
||||||
let novelCover = $(this).find("img").attr("src");
|
|
||||||
let novelUrl = $(this).find("a").attr("href");
|
|
||||||
novelUrl = novelUrl.replace("/novel/", "").concat("/");
|
|
||||||
|
|
||||||
novel = {
|
data.map((novel) => {
|
||||||
extensionId: 7,
|
let novelName = novel.name;
|
||||||
novelName,
|
let novelCover = novel.coverUrl;
|
||||||
novelCover,
|
let novelUrl = novel.slug;
|
||||||
novelUrl,
|
novelUrl = novelUrl + "/";
|
||||||
};
|
|
||||||
|
|
||||||
novels.push(novel);
|
novels.push({
|
||||||
|
extensionId: 7,
|
||||||
|
novelUrl,
|
||||||
|
novelName,
|
||||||
|
novelCover,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
res.json(novels);
|
||||||
});
|
});
|
||||||
|
|
||||||
res.json(novels);
|
|
||||||
});
|
});
|
||||||
|
|
||||||
// Novel
|
// Novel
|
||||||
@@ -41,67 +39,69 @@ router.get("/novels/", async (req, res) => {
|
|||||||
router.get("/novel/:novelUrl", async (req, res) => {
|
router.get("/novel/:novelUrl", async (req, res) => {
|
||||||
let novelUrl = req.params.novelUrl;
|
let novelUrl = req.params.novelUrl;
|
||||||
let url = `${baseUrl}novel/${novelUrl}/`;
|
let url = `${baseUrl}novel/${novelUrl}/`;
|
||||||
const body = await scraper(url);
|
|
||||||
|
|
||||||
const $ = cheerio.load(body);
|
request(url, (err, response, body) => {
|
||||||
|
if (err) throw err;
|
||||||
|
const $ = cheerio.load(body);
|
||||||
|
|
||||||
let novel = {};
|
let novel = {};
|
||||||
|
|
||||||
novel.extensionId = 7;
|
novel.extensionId = 7;
|
||||||
|
|
||||||
novel.sourceName = "WuxiaWorld";
|
novel.sourceName = "WuxiaWorld";
|
||||||
|
|
||||||
novel.sourceUrl = url;
|
novel.sourceUrl = url;
|
||||||
|
|
||||||
novel.novelUrl = `${novelUrl}/`;
|
novel.novelUrl = `${novelUrl}/`;
|
||||||
|
|
||||||
novel.novelName = $("h2").text();
|
novel.novelName = $("h2").text();
|
||||||
|
|
||||||
novel.novelCover = $("img.img-thumbnail").attr("src");
|
novel.novelCover = $("img.img-thumbnail").attr("src");
|
||||||
|
|
||||||
novel.novelSummary = $(".fr-view > p").text();
|
novel.novelSummary = $(".fr-view > p").text();
|
||||||
|
|
||||||
novel["Author(s)"] = $("div > dt")
|
novel["Author(s)"] = $("div > dt")
|
||||||
.filter(function () {
|
.filter(function () {
|
||||||
return $(this).text().trim() === "Author:";
|
return $(this).text().trim() === "Author:";
|
||||||
})
|
})
|
||||||
.next()
|
.next()
|
||||||
.text();
|
.text();
|
||||||
|
|
||||||
let genres = [];
|
let genres = [];
|
||||||
|
|
||||||
$(".genres")
|
$(".genres")
|
||||||
.find("div")
|
.find("div")
|
||||||
.each(function (res) {
|
.each(function (res) {
|
||||||
genres.push($(this).find("a").text());
|
genres.push($(this).find("a").text());
|
||||||
|
});
|
||||||
|
|
||||||
|
novel["Genre(s)"] = genres.join(",");
|
||||||
|
|
||||||
|
novel["Artist(s)"] = null;
|
||||||
|
|
||||||
|
novel.Status = null;
|
||||||
|
|
||||||
|
let novelChapters = [];
|
||||||
|
|
||||||
|
$(".chapter-item").each(function (result) {
|
||||||
|
let chapterName = $(this).text();
|
||||||
|
chapterName = chapterName.replace(/[\t\n]/g, "");
|
||||||
|
|
||||||
|
let releaseDate = null;
|
||||||
|
let chapterUrl = $(this).find("a").attr("href");
|
||||||
|
chapterUrl = chapterUrl.replace(`/novel/${novelUrl}/`, "");
|
||||||
|
|
||||||
|
novelChapters.push({
|
||||||
|
chapterName,
|
||||||
|
releaseDate,
|
||||||
|
chapterUrl,
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
novel["Genre(s)"] = genres.join(",");
|
novel.novelChapters = novelChapters;
|
||||||
|
|
||||||
novel["Artist(s)"] = null;
|
res.json(novel);
|
||||||
|
|
||||||
novel.Status = null;
|
|
||||||
|
|
||||||
let novelChapters = [];
|
|
||||||
|
|
||||||
$(".chapter-item").each(function (result) {
|
|
||||||
let chapterName = $(this).text();
|
|
||||||
chapterName = chapterName.replace(/[\t\n]/g, "");
|
|
||||||
|
|
||||||
let releaseDate = null;
|
|
||||||
let chapterUrl = $(this).find("a").attr("href");
|
|
||||||
chapterUrl = chapterUrl.replace(`/novel/${novelUrl}/`, "");
|
|
||||||
|
|
||||||
novelChapters.push({
|
|
||||||
chapterName,
|
|
||||||
releaseDate,
|
|
||||||
chapterUrl,
|
|
||||||
});
|
|
||||||
});
|
});
|
||||||
|
|
||||||
novel.novelChapters = novelChapters;
|
|
||||||
|
|
||||||
res.json(novel);
|
|
||||||
});
|
});
|
||||||
|
|
||||||
router.get("/novel/:novelUrl/:chapterUrl", async (req, res) => {
|
router.get("/novel/:novelUrl/:chapterUrl", async (req, res) => {
|
||||||
@@ -109,41 +109,43 @@ router.get("/novel/:novelUrl/:chapterUrl", async (req, res) => {
|
|||||||
|
|
||||||
const novelUrl = `${req.params.novelUrl}/`;
|
const novelUrl = `${req.params.novelUrl}/`;
|
||||||
const chapterUrl = `${req.params.chapterUrl}/`;
|
const chapterUrl = `${req.params.chapterUrl}/`;
|
||||||
const body = await scraper(url);
|
|
||||||
|
|
||||||
const $ = cheerio.load(body);
|
request(url, (err, response, body) => {
|
||||||
|
if (err) throw err;
|
||||||
|
const $ = cheerio.load(body);
|
||||||
|
|
||||||
let chapterName = $("#sidebar-toggler-container").next().text();
|
let chapterName = $("#sidebar-toggler-container").next().text();
|
||||||
chapterName = chapterName.replace(/[\t\n]/g, "");
|
chapterName = chapterName.replace(/[\t\n]/g, "");
|
||||||
|
|
||||||
let chapterText = $("#chapter-content").text();
|
let chapterText = $("#chapter-content").text();
|
||||||
|
|
||||||
let nextChapter = null;
|
let nextChapter = null;
|
||||||
|
|
||||||
if ($("li.next > a").attr("href") !== "#") {
|
if ($("li.next > a").attr("href") !== "#") {
|
||||||
nextChapter = $("li.next > a")
|
nextChapter = $("li.next > a")
|
||||||
.attr("href")
|
.attr("href")
|
||||||
.replace(`/novel/${novelUrl}`, "");
|
.replace(`/novel/${novelUrl}`, "");
|
||||||
}
|
}
|
||||||
|
|
||||||
let prevChapter = null;
|
let prevChapter = null;
|
||||||
|
|
||||||
if ($("li.prev > a").attr("href") !== "#") {
|
if ($("li.prev > a").attr("href") !== "#") {
|
||||||
prevChapter = $("li.prev > a")
|
prevChapter = $("li.prev > a")
|
||||||
.attr("href")
|
.attr("href")
|
||||||
.replace(`/novel/${novelUrl}`, "");
|
.replace(`/novel/${novelUrl}`, "");
|
||||||
}
|
}
|
||||||
chapter = {
|
chapter = {
|
||||||
extensionId: 7,
|
extensionId: 7,
|
||||||
novelUrl,
|
novelUrl,
|
||||||
chapterUrl,
|
chapterUrl,
|
||||||
chapterName,
|
chapterName,
|
||||||
chapterText,
|
chapterText,
|
||||||
nextChapter,
|
nextChapter,
|
||||||
prevChapter,
|
prevChapter,
|
||||||
};
|
};
|
||||||
|
|
||||||
res.json(chapter);
|
res.json(chapter);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
router.get("/search/", (req, res) => {
|
router.get("/search/", (req, res) => {
|
||||||
|
|||||||
Reference in New Issue
Block a user