add NovelFull (closes #1)
This commit is contained in:
Rajarshee Chatterjee
@@ -25,6 +25,7 @@ app.use("/api/4/", require("./src/en/readnovelfull/readnovelfull"));
|
||||
app.use("/api/5/", require("./src/en/mtlnovel/mtlnovel"));
|
||||
app.use("/api/6/", require("./src/en/novelhall/novelhall"));
|
||||
app.use("/api/7/", require("./src/en/wuxiaworld/wuxiaworld"));
|
||||
app.use("/api/8/", require("./src/en/novelfull/novelfull"));
|
||||
|
||||
const PORT = process.env.PORT || 5000;
|
||||
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 63 KiB |
@@ -0,0 +1,210 @@
|
||||
const express = require("express");
|
||||
const cheerio = require("cheerio");
|
||||
const request = require("request");
|
||||
|
||||
const router = express.Router();
|
||||
|
||||
const baseUrl = "https://novelfull.com";
|
||||
|
||||
// Top novels
|
||||
|
||||
router.get("/novels/", (req, res) => {
|
||||
let url = `${baseUrl}/most-popular`;
|
||||
|
||||
console.log(url);
|
||||
|
||||
request(url, (err, response, body) => {
|
||||
if (err) throw err;
|
||||
|
||||
let novels = [];
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
$("div.col-truyen-main > div.list-truyen > .row").each(function (
|
||||
result
|
||||
) {
|
||||
let novelUrl = $(this)
|
||||
.find("h3.truyen-title > a")
|
||||
.attr("href")
|
||||
.replace(".html", "")
|
||||
.substring(1);
|
||||
novelUrl = `${novelUrl}/`;
|
||||
let novelName = $(this).find("h3.truyen-title > a").text();
|
||||
let novelCover = $(this).find("img").attr("src");
|
||||
novelCover = baseUrl + novelCover;
|
||||
|
||||
novel = {
|
||||
extensionId: 8,
|
||||
novelUrl,
|
||||
novelName,
|
||||
novelCover,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
res.json(novels);
|
||||
});
|
||||
});
|
||||
|
||||
router.get("/novel/:novelUrl", (req, res) => {
|
||||
let novelUrl = req.params.novelUrl;
|
||||
url = `${baseUrl}/${novelUrl}.html`;
|
||||
|
||||
request(url, (err, response, body) => {
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.extensionId = 8;
|
||||
|
||||
novel.sourceName = "NovelFull";
|
||||
|
||||
novel.sourceUrl = url;
|
||||
|
||||
novel.novelUrl = `${novelUrl}/`;
|
||||
|
||||
novel.novelName = $("div.book > img").attr("alt");
|
||||
|
||||
novel.novelCover = baseUrl + $("div.book > img").attr("src");
|
||||
|
||||
novel.novelSummary = $("div.desc-text").text();
|
||||
|
||||
novel["Author(s)"] = $("div.info > div > h3")
|
||||
.filter(function () {
|
||||
return $(this).text().trim() === "Author:";
|
||||
})
|
||||
.siblings()
|
||||
.text();
|
||||
|
||||
novel["Genre(s)"] = $("div.info > div")
|
||||
.filter(function () {
|
||||
return $(this).find("h3").text().trim() === "Genre:";
|
||||
})
|
||||
.text()
|
||||
.replace("Genre:", "");
|
||||
|
||||
novel["Artist(s)"] = null;
|
||||
|
||||
novel.Status = $("div.info > div > h3")
|
||||
.filter(function () {
|
||||
return $(this).text().trim() === "Status:";
|
||||
})
|
||||
.next()
|
||||
.text();
|
||||
|
||||
const novelId = $("#rating").attr("data-novel-id");
|
||||
console.log(novelId);
|
||||
|
||||
const chapterListUrl =
|
||||
baseUrl + "/ajax/chapter-option?novelId=" + novelId;
|
||||
|
||||
function callback(res) {
|
||||
console.log(res);
|
||||
novel.novelChapters = res;
|
||||
}
|
||||
|
||||
request(chapterListUrl, (err, response, body) => {
|
||||
$ = cheerio.load(body);
|
||||
let novelChapters = [];
|
||||
|
||||
$("select > option").each(function (result) {
|
||||
let chapterName = $(this).text();
|
||||
let releaseDate = null;
|
||||
let chapterUrl = $(this).attr("value");
|
||||
chapterUrl = chapterUrl.replace(`/${novelUrl}/`, "");
|
||||
|
||||
novelChapters.push({
|
||||
chapterName,
|
||||
releaseDate,
|
||||
chapterUrl,
|
||||
});
|
||||
});
|
||||
callback(novelChapters);
|
||||
|
||||
res.json(novel);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
router.get("/novel/:novelUrl/:chapterUrl", (req, res) => {
|
||||
let novelUrl = req.params.novelUrl.replace(".html/", "");
|
||||
url = `${baseUrl}/${novelUrl}/${req.params.chapterUrl}`;
|
||||
|
||||
request(url, (err, response, body) => {
|
||||
if (err) throw err;
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let chapterName = $(".chapter-title").attr("title");
|
||||
|
||||
let chapterText = $("#chapter-content").text();
|
||||
|
||||
let nextChapter = null;
|
||||
if ($("a#next_chap").attr("href")) {
|
||||
nextChapter = $("a#next_chap")
|
||||
.attr("href")
|
||||
.replace(`/${novelUrl}/`, "");
|
||||
}
|
||||
|
||||
let prevChapter = null;
|
||||
if ($("a#prev_chap").attr("href")) {
|
||||
prevChapter = $("a#prev_chap")
|
||||
.attr("href")
|
||||
.replace(`/${novelUrl}/`, "");
|
||||
}
|
||||
|
||||
chapter = {
|
||||
extensionId: 8,
|
||||
novelUrl: `${req.params.novelUrl}`,
|
||||
chapterUrl: `${req.params.chapterUrl}`,
|
||||
chapterName,
|
||||
chapterText,
|
||||
nextChapter,
|
||||
prevChapter,
|
||||
};
|
||||
|
||||
res.json(chapter);
|
||||
});
|
||||
});
|
||||
|
||||
router.get("/search/", (req, res) => {
|
||||
let searchTerm = req.query.s;
|
||||
searchUrl = `https://novelfull.com/search?keyword=`;
|
||||
|
||||
url = `${searchUrl}${searchTerm}`;
|
||||
|
||||
request(url, (err, response, body) => {
|
||||
if (err) throw err;
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$("div.col-truyen-main > div.list-truyen > .row").each(function (
|
||||
result
|
||||
) {
|
||||
let novelUrl = $(this)
|
||||
.find("h3.truyen-title > a")
|
||||
.attr("href")
|
||||
.replace(".html", "")
|
||||
.substring(1);
|
||||
novelUrl = `${novelUrl}/`;
|
||||
let novelName = $(this).find("h3.truyen-title > a").text();
|
||||
let novelCover = $(this).find("img").attr("src");
|
||||
|
||||
novel = {
|
||||
extensionId: 8,
|
||||
novelUrl,
|
||||
novelName,
|
||||
novelCover,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
res.json(novels);
|
||||
});
|
||||
});
|
||||
|
||||
module.exports = router;
|
||||
@@ -88,28 +88,38 @@ router.get("/novel/:novelUrl", (req, res) => {
|
||||
.siblings()
|
||||
.text();
|
||||
|
||||
let novelChapters = [];
|
||||
const novelId = $("#rating").attr("data-novel-id");
|
||||
|
||||
$(".panel-body")
|
||||
.find("li")
|
||||
.each(function (result) {
|
||||
let chapterName = $(this).find("a").attr("title");
|
||||
const chapterListUrl =
|
||||
"https://readnovelfull.com/ajax/chapter-archive?novelId=" + novelId;
|
||||
|
||||
let releaseDate = null;
|
||||
function callback(res) {
|
||||
console.log(res);
|
||||
novel.novelChapters = res;
|
||||
}
|
||||
|
||||
let chapterUrl = $(this).find("a").attr("href");
|
||||
chapterUrl = chapterUrl.replace(`/${novelUrl}/`, "");
|
||||
request(chapterListUrl, (err, response, body) => {
|
||||
$ = cheerio.load(body);
|
||||
let novelChapters = [];
|
||||
|
||||
novelChapters.push({
|
||||
chapterName,
|
||||
releaseDate,
|
||||
chapterUrl,
|
||||
$(".panel-body")
|
||||
.find("li")
|
||||
.each(function (result) {
|
||||
let chapterName = $(this).find("a").attr("title");
|
||||
let releaseDate = null;
|
||||
let chapterUrl = $(this).find("a").attr("href");
|
||||
chapterUrl = chapterUrl.replace(`/${novelUrl}/`, "");
|
||||
|
||||
novelChapters.push({
|
||||
chapterName,
|
||||
releaseDate,
|
||||
chapterUrl,
|
||||
});
|
||||
});
|
||||
});
|
||||
callback(novelChapters);
|
||||
|
||||
novel.novelChapters = novelChapters;
|
||||
|
||||
res.json(novel);
|
||||
res.json(novel);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@ const cheerio = require("cheerio");
|
||||
const request = require("request");
|
||||
const chromium = require("chrome-aws-lambda");
|
||||
const puppeteer = require("puppeteer");
|
||||
const { scraper } = require("../../helper");
|
||||
|
||||
const baseUrl = "https://www.wuxiaworld.com/";
|
||||
|
||||
@@ -10,18 +11,7 @@ const router = express.Router();
|
||||
|
||||
router.get("/novels/", async (req, res) => {
|
||||
let url = `https://www.wuxiaworld.com/novels`;
|
||||
const browser = await puppeteer.launch({
|
||||
args: [...chromium.args, "--hide-scrollbars", "--disable-web-security"],
|
||||
defaultViewport: chromium.defaultViewport,
|
||||
executablePath: await chromium.executablePath,
|
||||
headless: true,
|
||||
ignoreHTTPSErrors: true,
|
||||
});
|
||||
const page = await browser.newPage();
|
||||
|
||||
await page.goto(url);
|
||||
|
||||
const body = await page.content();
|
||||
const body = await scraper(url);
|
||||
|
||||
let novels = [];
|
||||
|
||||
@@ -51,18 +41,7 @@ router.get("/novels/", async (req, res) => {
|
||||
router.get("/novel/:novelUrl", async (req, res) => {
|
||||
let novelUrl = req.params.novelUrl;
|
||||
let url = `${baseUrl}novel/${novelUrl}/`;
|
||||
const browser = await puppeteer.launch({
|
||||
args: [...chromium.args, "--hide-scrollbars", "--disable-web-security"],
|
||||
defaultViewport: chromium.defaultViewport,
|
||||
executablePath: await chromium.executablePath,
|
||||
headless: true,
|
||||
ignoreHTTPSErrors: true,
|
||||
});
|
||||
const page = await browser.newPage();
|
||||
|
||||
await page.goto(url);
|
||||
|
||||
const body = await page.content();
|
||||
const body = await scraper(url);
|
||||
|
||||
const $ = cheerio.load(body);
|
||||
|
||||
@@ -130,18 +109,7 @@ router.get("/novel/:novelUrl/:chapterUrl", async (req, res) => {
|
||||
|
||||
const novelUrl = `${req.params.novelUrl}/`;
|
||||
const chapterUrl = `${req.params.chapterUrl}/`;
|
||||
const browser = await puppeteer.launch({
|
||||
args: [...chromium.args, "--hide-scrollbars", "--disable-web-security"],
|
||||
defaultViewport: chromium.defaultViewport,
|
||||
executablePath: await chromium.executablePath,
|
||||
headless: true,
|
||||
ignoreHTTPSErrors: true,
|
||||
});
|
||||
const page = await browser.newPage();
|
||||
|
||||
await page.goto(url);
|
||||
|
||||
const body = await page.content();
|
||||
const body = await scraper(url);
|
||||
|
||||
const $ = cheerio.load(body);
|
||||
|
||||
|
||||
@@ -62,6 +62,15 @@ extensions = [
|
||||
sourceLanguage: "English",
|
||||
status: 1,
|
||||
},
|
||||
{
|
||||
sourceId: 8,
|
||||
sourceUrl: "https://novelfull.com/",
|
||||
sourceName: "NovelFull",
|
||||
sourceCover:
|
||||
"https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/novelfull/icon.png?raw=true",
|
||||
sourceLanguage: "English",
|
||||
status: 1,
|
||||
},
|
||||
];
|
||||
|
||||
module.exports = extensions;
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
const chromium = require("chrome-aws-lambda");
|
||||
const puppeteer = require("puppeteer");
|
||||
|
||||
async function scraper(url) {
|
||||
const browser = await puppeteer.launch({
|
||||
args: [...chromium.args, "--hide-scrollbars", "--disable-web-security"],
|
||||
defaultViewport: chromium.defaultViewport,
|
||||
executablePath: await chromium.executablePath,
|
||||
headless: true,
|
||||
ignoreHTTPSErrors: true,
|
||||
});
|
||||
|
||||
const page = await browser.newPage();
|
||||
await page.goto(url);
|
||||
const body = await page.content();
|
||||
|
||||
return body;
|
||||
}
|
||||
|
||||
module.exports = { scraper };
|
||||
Reference in New Issue
Block a user