From b0d966b0d4a3135be89ae538ee597d5eb8529dde Mon Sep 17 00:00:00 2001 From: Bread Genie <63963181+BreadGenie@users.noreply.github.com> Date: Tue, 13 Jul 2021 09:43:44 +0530 Subject: [PATCH] feat(source): Add Yushubo (#45) * feat(source): Add Yushubo * docs(source): Add Yushubo --- README.md | 1 + index.js | 19 ++-- src/ch/yushubo/Yushubo.js | 14 +++ src/ch/yushubo/YushuboScraper.js | 177 +++++++++++++++++++++++++++++++ 4 files changed, 203 insertions(+), 8 deletions(-) create mode 100644 src/ch/yushubo/Yushubo.js create mode 100644 src/ch/yushubo/YushuboScraper.js diff --git a/README.md b/README.md index a3d5eaa..91190a7 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,7 @@ Sources for [LNReader](https://github.com/rajarsheechatterjee/lnreader) | https://hasutl.wordpress.com/ | :heavy_check_mark: | | https://oasistranslations.wordpress.com/ | :heavy_check_mark: | | http://www.novelawuxia.com/ | :heavy_check_mark: | +| https://m.yushubo.com | :heavy_check_mark: | ## Setup locally diff --git a/index.js b/index.js index 8ae6fbc..c113ea1 100755 --- a/index.js +++ b/index.js @@ -4,17 +4,17 @@ const sources = require("./sources"); const app = express(); app.use( - express.json({ - extended: false, - }) + express.json({ + extended: false, + }) ); app.get("/", (req, res) => - res.json({ - Author: "@rajarsheechatterjee", - Version: "1.0.0", - Github: "https://github.com/rajarsheechatterjee/lnreader-extensions", - }) + res.json({ + Author: "@rajarsheechatterjee", + Version: "1.0.0", + Github: "https://github.com/rajarsheechatterjee/lnreader-extensions", + }) ); // English @@ -62,6 +62,9 @@ app.use("/api/31/", require("./src/es/novelawuxia/NovelaWuxia")); // Japanese app.use("/api/36", require("./src/jp/syosetu/Syosetu")); +// Chinese +app.use("/api/50/", require("./src/ch/yushubo/Yushubo")); + // MultiSrc app.use("/api/", require("./multisrc/madara/MadaraGenerator")); diff --git a/src/ch/yushubo/Yushubo.js b/src/ch/yushubo/Yushubo.js new file mode 100644 index 0000000..aa6ce88 --- /dev/null +++ b/src/ch/yushubo/Yushubo.js @@ -0,0 +1,14 @@ +const express = require("express"); +const YushuboScraper = require("./YushuboScraper"); + +const router = express.Router(); + +router.get("/novels/", YushuboScraper.novelsScraper); + +router.get("/novel/:novelUrl", YushuboScraper.novelScraper); + +router.get("/novel/:novelUrl/:chapterUrl", YushuboScraper.chapterScraper); + +router.get("/search/", YushuboScraper.searchScraper); + +module.exports = router; diff --git a/src/ch/yushubo/YushuboScraper.js b/src/ch/yushubo/YushuboScraper.js new file mode 100644 index 0000000..b0ad655 --- /dev/null +++ b/src/ch/yushubo/YushuboScraper.js @@ -0,0 +1,177 @@ +const cheerio = require("cheerio"); +const fetch = require("node-fetch"); +const UserAgent = require("user-agents"); + +const userAgent = new UserAgent(); + +const baseUrl = "https://m.yushubo.com"; + +const novelsScraper = async (req, res) => { + const url = `${baseUrl}/all/order/hits_week+desc.html`; + + const result = await fetch(url, { headers: { "User-Agent": userAgent } }); + const body = await result.text(); + + $ = cheerio.load(body); + + let novels = []; + + $(".clearfix > li").each(function (result) { + const novelUrl = + $(this).find("a").attr("href").replace(".html", "").substring(1) + "/"; + + const novelName = $(this).find("a").text(); + const novelCover = $(this).find("img").attr("src"); + + const novel = { + extensionId: 50, + novelUrl, + novelName, + novelCover, + }; + + novels.push(novel); + }); + + res.json(novels); +}; + +const novelScraper = async (req, res) => { + const novelUrl = req.params.novelUrl; + const url = `${baseUrl}/${novelUrl}.html`; + + const result = await fetch(url, { headers: { "User-Agent": userAgent } }); + const body = await result.text(); + + $ = cheerio.load(body); + + let novel = {}; + + novel.extensionId = 50; + + novel.sourceName = "Yushubo"; + + novel.sourceUrl = url; + + novel.novelUrl = novelUrl + "/"; + + novel.novelName = $("dd > h2").text(); + + novel.novelCover = $("dt > img").attr("src"); + + novel.novelSummary = $("div.content > p").text(); + + let info = []; + $(".info a").each(function (result) { + info.push($(this).text()); + }); + + novel["Author(s)"] = info[0]; + + novel["Genre(s)"] = info[1]; + + novel["Artist(s)"] = null; + + novel.Status = "Unknown"; + + let chapters = []; + + $(".bookshelf-list a").each(function (result) { + let chapterUrl = $(this).attr("href").substring(1); + let chapterName = $(this).attr("title"); + let releaseDate = null; + + chapters.push({ + chapterName, + releaseDate, + chapterUrl, + }); + }); + + novel.novelChapters = chapters; + + res.json(novel); +}; + +const chapterScraper = async (req, res) => { + let novelUrl = req.params.novelUrl.replace(".html/", ""); + let chapterUrl = req.params.chapterUrl; + + const url = `${baseUrl}/${chapterUrl}`; + + const result = await fetch(url, { headers: { "User-Agent": userAgent } }); + const body = await result.text(); + + $ = cheerio.load(body); + + const chapterName = $(".read-section h3").text(); + const chapterText = $(".read-section p").text(); + + let nextChapter = null; + if ($(".btn-next").attr("href")) { + nextChapter = $(".btn-prev").attr("href"); + } + + let prevChapter = null; + if ($(".btn-prev").attr("href")) { + prevChapter = $(".btn-prev").attr("href"); + } + + const chapter = { + extensionId: 50, + novelUrl, + chapterUrl, + chapterName, + chapterText, + nextChapter, + prevChapter, + }; + + res.json(chapter); +}; + +const searchScraper = async (req, res) => { + const searchTerm = req.query.s; + const searchUrl = `${baseUrl}/search.html`; + + const result = await fetch(searchUrl, { + method: "post", + body: JSON.stringify({ keyword: searchTerm }), + headers: { "User-Agent": userAgent, "Content-Type": "application/json" }, + }); + + const body = await result.text(); + + $ = cheerio.load(body); + + let novels = []; + + $(".clearfix > li").each(function (result) { + let novelUrl = + $(this).find("a").attr("href").replace(".html", "").substring(1) + "/"; + + const novelName = $(this).find("a").text(); + let novelCover = $(this).find("img").attr("src"); + novelCover = novelCover.startsWith("https://tva1.sinaimg.cn/") + ? `${novelCover}` + : `${baseUrl}${novelCover}`; + + const novel = { + extensionId: 50, + novelUrl, + novelName, + novelCover, + }; + + novels.push(novel); + }); + + res.json(novels); +}; + +module.exports = readNovelFullScraper = { + novelsScraper, + novelScraper, + chapterScraper, + searchScraper, +};