Remove src
This commit is contained in:
Rajarshee Chatterjee
@@ -1,186 +0,0 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
|
||||
const sourceId = 59;
|
||||
const sourceName = "ArNovel";
|
||||
|
||||
const baseUrl = "https://arnovel.me/";
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
let totalPages = 21;
|
||||
let url = baseUrl + "novels/page/" + page;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".page-item-detail").each(function () {
|
||||
const novelName = $(this).find(".h5 > a").text();
|
||||
const novelCover =
|
||||
"https://github.com/LNReader/lnreader-sources/blob/main/src/coverNotAvailable.jpg?raw=true";
|
||||
|
||||
let novelUrl = $(this).find(".h5 > a").attr("href").split("/")[4];
|
||||
|
||||
const novel = {
|
||||
sourceId,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return { totalPages, novels };
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = `${baseUrl}novel/${novelUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = { sourceId, sourceName, url, novelUrl };
|
||||
|
||||
novel.novelName = $(".post-title > h1")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
novel.novelCover =
|
||||
"https://github.com/LNReader/lnreader-sources/blob/main/src/coverNotAvailable.jpg?raw=true";
|
||||
|
||||
$(".post-content_item").each(function () {
|
||||
detailName = $(this)
|
||||
.find(".summary-heading > h5")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
detail = $(this)
|
||||
.find(".summary-content")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
switch (detailName) {
|
||||
case "التصنيفات":
|
||||
novel.genre = detail.trim().replace(/[\t\n]/g, ",");
|
||||
break;
|
||||
case "المؤلف (ين)":
|
||||
novel.author = detail.trim();
|
||||
break;
|
||||
case "الحالة":
|
||||
novel.status = detail.trim();
|
||||
break;
|
||||
}
|
||||
});
|
||||
|
||||
$(".description-summary > div.summary__content").find("em").remove();
|
||||
|
||||
novel.summary = $(".description-summary > div.summary__content")
|
||||
.text()
|
||||
.trim();
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
const novelId = $("#manga-chapters-holder").attr("data-id");
|
||||
|
||||
let formData = new FormData();
|
||||
formData.append("action", "manga_get_chapters");
|
||||
formData.append("manga", novelId);
|
||||
|
||||
const data = await fetch("https://www.arnovel.me/wp-admin/admin-ajax.php", {
|
||||
method: "POST",
|
||||
body: formData,
|
||||
});
|
||||
const text = await data.text();
|
||||
|
||||
$ = cheerio.load(text);
|
||||
|
||||
$(".wp-manga-chapter").each(function () {
|
||||
chapterName = $(this)
|
||||
.find("a")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
releaseDate = $(this).find("span").text().trim();
|
||||
|
||||
chapterUrl = $(this).find("a").attr("href").split("/");
|
||||
|
||||
chapterUrl[6]
|
||||
? (chapterUrl = chapterUrl[5] + "/" + chapterUrl[6])
|
||||
: (chapterUrl = chapterUrl[5]);
|
||||
|
||||
novelChapters.push({ chapterName, releaseDate, chapterUrl });
|
||||
});
|
||||
|
||||
novel.chapters = novelChapters.reverse();
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = `${baseUrl}novel/${novelUrl}/${chapterUrl}/`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
const chapterName = $("h1#chapter-heading").text();
|
||||
const chapterText = $(".reading-content").html();
|
||||
|
||||
const chapter = {
|
||||
sourceId,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const url = `${baseUrl}?s=${searchTerm}&post_type=wp-manga`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".c-tabs-item__content").each(function () {
|
||||
const novelName = $(this).find(".h4 > a").text();
|
||||
const novelCover =
|
||||
"https://github.com/LNReader/lnreader-sources/blob/main/src/coverNotAvailable.jpg?raw=true";
|
||||
|
||||
let novelUrl = $(this).find(".h4 > a").attr("href").split("/")[4];
|
||||
|
||||
const novel = {
|
||||
sourceId,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const ArNovelScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default ArNovelScraper;
|
||||
@@ -1,183 +0,0 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
|
||||
const baseUrl = "https://m.yushubo.com";
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
const url = `${baseUrl}/all/order/hits_week+desc.html?page=${page}`;
|
||||
const totalPages = 5846;
|
||||
|
||||
const result = await fetch(url, {
|
||||
headers: {
|
||||
"User-Agent":
|
||||
"'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36",
|
||||
},
|
||||
});
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".clearfix > li").each(function (result) {
|
||||
const novelUrl =
|
||||
$(this).find("a").attr("href").replace(".html", "").substring(1) +
|
||||
"/";
|
||||
|
||||
const novelName = $(this).find("a").text();
|
||||
let novelCover = $(this).find("img").attr("src");
|
||||
novelCover = novelCover.includes("https://tva1.sinaimg.cn/")
|
||||
? `${novelCover}`
|
||||
: `${baseUrl}${novelCover}`;
|
||||
|
||||
const novel = {
|
||||
sourceId: 52,
|
||||
novelUrl,
|
||||
novelName,
|
||||
novelCover,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return { totalPages, novels };
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = `${baseUrl}/${novelUrl.slice(0, -1)}.html`;
|
||||
|
||||
const result = await fetch(url, {
|
||||
headers: {
|
||||
"User-Agent":
|
||||
"'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36",
|
||||
},
|
||||
});
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.sourceId = 52;
|
||||
|
||||
novel.sourceName = "Yushubo";
|
||||
|
||||
novel.url = url;
|
||||
|
||||
novel.novelUrl = novelUrl;
|
||||
|
||||
novel.novelName = $("dd > h2").text();
|
||||
|
||||
novel.novelCover = $("dt > img").attr("src");
|
||||
|
||||
novel.summary = $("div.content > p").text();
|
||||
|
||||
let info = [];
|
||||
$(".info a").each(function (result) {
|
||||
info.push($(this).text());
|
||||
});
|
||||
|
||||
novel.genre = info[0];
|
||||
|
||||
novel.author = info[1];
|
||||
|
||||
novel.artist = null;
|
||||
|
||||
novel.status = "Unknown";
|
||||
|
||||
let chapters = [];
|
||||
|
||||
$(".bookshelf-list a").each(function (result) {
|
||||
let chapterUrl = $(this).attr("href").substring(1);
|
||||
let chapterName = $(this).attr("title");
|
||||
let releaseDate = null;
|
||||
|
||||
chapters.push({
|
||||
chapterName,
|
||||
releaseDate,
|
||||
chapterUrl,
|
||||
});
|
||||
});
|
||||
|
||||
novel.chapters = chapters;
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = `${baseUrl}/${chapterUrl}`;
|
||||
|
||||
const result = await fetch(url, {
|
||||
headers: {
|
||||
"User-Agent":
|
||||
"'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36",
|
||||
},
|
||||
});
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
const chapterName = $(".read-section h3").text();
|
||||
let chapterText = $(".read-section").html();
|
||||
|
||||
const chapter = {
|
||||
sourceId: 52,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const searchUrl = `${baseUrl}/search.html`;
|
||||
|
||||
const result = await fetch(searchUrl, {
|
||||
method: "post",
|
||||
body: JSON.stringify({ keyword: searchTerm }),
|
||||
headers: {
|
||||
"User-Agent":
|
||||
"'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36",
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
});
|
||||
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".clearfix > li").each(function (result) {
|
||||
let novelUrl =
|
||||
$(this).find("a").attr("href").replace(".html", "").substring(1) +
|
||||
"/";
|
||||
|
||||
const novelName = $(this).find("a").text();
|
||||
let novelCover = $(this).find("img").attr("src");
|
||||
novelCover = novelCover.includes("https://tva1.sinaimg.cn/")
|
||||
? `${novelCover}`
|
||||
: `${baseUrl}${novelCover}`;
|
||||
|
||||
const novel = {
|
||||
sourceId: 52,
|
||||
novelUrl,
|
||||
novelName,
|
||||
novelCover,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const YushuboScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default YushuboScraper;
|
||||
@@ -1,184 +0,0 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
|
||||
const baseUrl = "https://boxnovel.com/novel";
|
||||
const searchUrl = "https://boxnovel.com/";
|
||||
|
||||
const sourceId = 1;
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
let url = `${baseUrl}/page/${page}/?m_orderby=rating`;
|
||||
let totalPages = 73;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".page-item-detail").each(function (result) {
|
||||
const novelName = $(this).find("h5 > a").text();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
|
||||
let novelUrl = $(this).find("h5 > a").attr("href");
|
||||
novelUrl = novelUrl.replace(`${baseUrl}/`, "");
|
||||
|
||||
const novel = {
|
||||
sourceId,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return { totalPages, novels };
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = `${baseUrl}/${novelUrl}`;
|
||||
|
||||
console.log(url);
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
$(".post-title > h3 > span").remove();
|
||||
|
||||
novel.sourceId = 1;
|
||||
|
||||
novel.sourceName = "BoxNovel";
|
||||
|
||||
novel.url = url;
|
||||
|
||||
novel.novelUrl = novelUrl;
|
||||
|
||||
novel.novelName = $(".post-title > h3")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
novel.novelCover = $(".summary_image > a > img").attr("src");
|
||||
|
||||
$(".post-content_item").each(function (result) {
|
||||
detailName = $(this)
|
||||
.find(".summary-heading > h5")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
detail = $(this)
|
||||
.find(".summary-content")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
switch (detailName) {
|
||||
case "Genre(s)":
|
||||
novel.genre = detail.trim().replace(/[\t\n]/g, ",");
|
||||
break;
|
||||
case "Author(s)":
|
||||
novel.author = detail.trim();
|
||||
break;
|
||||
case "Artist(s)":
|
||||
novel.status = detail.trim();
|
||||
break;
|
||||
}
|
||||
});
|
||||
|
||||
$(".description-summary > div.summary__content").find("em").remove();
|
||||
|
||||
novel.summary = $(".description-summary > div.summary__content")
|
||||
.text()
|
||||
.replace(/[\t\n]|Description|Summary/g, "");
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
$(".wp-manga-chapter").each(function (result) {
|
||||
const chapterName = $(this)
|
||||
.find("a")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
const releaseDate = $(this)
|
||||
.find("span")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
const chapterUrl = $(this).find("a").attr("href").replace(url, "");
|
||||
|
||||
const chapter = { chapterName, releaseDate, chapterUrl };
|
||||
|
||||
novelChapters.push(chapter);
|
||||
});
|
||||
|
||||
novel.chapters = novelChapters.reverse();
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = `${baseUrl}/${novelUrl}/${chapterUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
const chapterName = $("div.text-left > h3").text();
|
||||
let chapterText = $(".reading-content").html();
|
||||
const chapter = {
|
||||
sourceId,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const url = `${searchUrl}?s=${searchTerm}&post_type=wp-manga&m_orderby=rating`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".c-tabs-item__content").each(function (result) {
|
||||
const novelName = $(this).find("h4 > a").text();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
|
||||
let novelUrl = $(this).find("h4 > a").attr("href");
|
||||
novelUrl = novelUrl.replace(`${baseUrl}/`, "");
|
||||
|
||||
const novel = {
|
||||
sourceId,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const BoxNovelScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default BoxNovelScraper;
|
||||
@@ -1,204 +0,0 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
|
||||
const sourceId = 27;
|
||||
|
||||
const sourceName = "Comrade Mao";
|
||||
|
||||
const baseUrl = "https://comrademao.com/";
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
let url = baseUrl + "novel/page/" + page;
|
||||
const totalPages = 111;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$("div.container")
|
||||
.find("section")
|
||||
.each(function () {
|
||||
const novelName = $(this)
|
||||
.find("div.columns > div:nth-child(2) > a")
|
||||
.text();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
|
||||
let novelUrl = $(this)
|
||||
.find("div.columns > div:nth-child(2) > a")
|
||||
.attr("href");
|
||||
novelUrl = novelUrl.replace(baseUrl + "novel/", "");
|
||||
|
||||
const novel = {
|
||||
sourceId,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return { novels, totalPages };
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = baseUrl + "novel/" + novelUrl;
|
||||
console.log(url);
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = { sourceId, sourceName, url, novelUrl };
|
||||
|
||||
novel.novelName = $(
|
||||
"#NovelInfo > div > div.column.is-one-third.has-text-centered > p"
|
||||
)
|
||||
.text()
|
||||
.trim();
|
||||
|
||||
novel.novelCover = $("img.attachment-post-thumbnail").attr("src");
|
||||
|
||||
novel.summary = $("#NovelInfo > div > div:nth-child(2) > p").text().trim();
|
||||
|
||||
novel.genre = $("#NovelInfo > p:nth-child(2)")
|
||||
.text()
|
||||
.replace(/Genre(s):|\s/g, "");
|
||||
|
||||
novel.status = $("#NovelInfo > p:nth-child(5)")
|
||||
.text()
|
||||
.replace(/Status:|\s/g, "")
|
||||
.trim();
|
||||
|
||||
novel.author = $("#NovelInfo > p:nth-child(4)")
|
||||
.text()
|
||||
.replace(/Publisher:|\s/g, "")
|
||||
.trim();
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
// $("table > tbody")
|
||||
// .find("tr")
|
||||
// .each(function () {
|
||||
// const releaseDate = $(this).find("span").text();
|
||||
|
||||
// const chapterName = $(this).find("a").text();
|
||||
// const chapterUrl = $(this).find("a").attr("href").split("/")[5];
|
||||
|
||||
// novelChapters.push({
|
||||
// chapterName,
|
||||
// chapterUrl,
|
||||
// releaseDate,
|
||||
// });
|
||||
// });
|
||||
|
||||
// $("small").remove();
|
||||
|
||||
let chapterUrlPrefix = $("table > tbody")
|
||||
.find("tr")
|
||||
.first()
|
||||
.find("a")
|
||||
.attr("href")
|
||||
.slice(0, -1)
|
||||
.split("/")
|
||||
.pop()
|
||||
.split("-");
|
||||
|
||||
chapterUrlPrefix.pop();
|
||||
|
||||
chapterUrlPrefix = chapterUrlPrefix.join("-").toLowerCase();
|
||||
|
||||
let latestChapter = $("table > tbody")
|
||||
.find("tr")
|
||||
.first()
|
||||
.find("a")
|
||||
.text()
|
||||
.replace(/^\D+/g, "");
|
||||
|
||||
for (let i = 1; i <= latestChapter; i++) {
|
||||
const chapterName = "Chapter " + i;
|
||||
const releaseDate = null;
|
||||
const chapterUrl = chapterUrlPrefix + "-" + i + "/";
|
||||
|
||||
const chapter = {
|
||||
chapterName,
|
||||
releaseDate,
|
||||
chapterUrl,
|
||||
};
|
||||
|
||||
novelChapters.push(chapter);
|
||||
}
|
||||
|
||||
novel.chapters = novelChapters;
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = `${baseUrl}mtl/${novelUrl}/${chapterUrl}`;
|
||||
|
||||
console.log(url);
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let chapterName = "";
|
||||
|
||||
let chapterText = $("#content").html();
|
||||
|
||||
const chapter = {
|
||||
sourceId,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const url = baseUrl + "?s=" + searchTerm + "&post_type=novel";
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".newbox")
|
||||
.find("li")
|
||||
.each(function () {
|
||||
const novelName = $(this).find("h3").text();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
|
||||
let novelUrl = $(this).find("a").attr("href");
|
||||
novelUrl = novelUrl.replace(baseUrl + "novel/", "");
|
||||
|
||||
const novel = {
|
||||
sourceId,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const ComradeMaoScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default ComradeMaoScraper;
|
||||
@@ -1,152 +0,0 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
|
||||
const sourceId = 70;
|
||||
|
||||
const sourceName = "Divine Dao Library";
|
||||
|
||||
const baseUrl = "https://www.divinedaolibrary.com/";
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
let url = baseUrl + "novels";
|
||||
const totalPages = 1;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$("#main")
|
||||
.find("li")
|
||||
.each(function () {
|
||||
const novelName = $(this).find("a").text();
|
||||
const novelCover = null;
|
||||
const novelUrl = $(this).find(" a").attr("href");
|
||||
|
||||
const novel = {
|
||||
sourceId,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return { novels, totalPages };
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = novelUrl;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = { sourceId, sourceName, url, novelUrl };
|
||||
|
||||
novel.novelName = $("h1.entry-title").text().trim();
|
||||
|
||||
novel.novelCover =
|
||||
$(".entry-content").find("img").attr("src") ||
|
||||
"https://github.com/LNReader/lnreader-sources/blob/main/src/coverNotAvailable.jpg?raw=true";
|
||||
|
||||
novel.summary = $("#main > article > div > p:nth-child(6)").text().trim();
|
||||
|
||||
novel.genre = null;
|
||||
|
||||
novel.status = null;
|
||||
|
||||
novel.author = $("#main > article > div > h3:nth-child(2)")
|
||||
.text()
|
||||
.replace(/Author:/g, "")
|
||||
.trim();
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
$("#main")
|
||||
.find("li > span > a")
|
||||
.each(function () {
|
||||
const chapterName = $(this).text().trim();
|
||||
const releaseDate = null;
|
||||
const chapterUrl = $(this).attr("href");
|
||||
|
||||
novelChapters.push({ chapterName, releaseDate, chapterUrl });
|
||||
});
|
||||
|
||||
novel.chapters = novelChapters;
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = chapterUrl;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let chapterName = $(".entry-title").text().trim();
|
||||
|
||||
let chapterText = $(".entry-content").html();
|
||||
|
||||
if (!chapterText) {
|
||||
chapterText = $(".page-header").html();
|
||||
}
|
||||
|
||||
const chapter = {
|
||||
sourceId,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
let url = baseUrl + "novels";
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$("#main")
|
||||
.find("li")
|
||||
.each(function () {
|
||||
const novelName = $(this).find("a").text();
|
||||
const novelCover = null;
|
||||
const novelUrl = $(this).find(" a").attr("href");
|
||||
|
||||
const novel = {
|
||||
sourceId,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
novels = novels.filter((novel) =>
|
||||
novel.novelName.toLowerCase().includes(searchTerm.toLowerCase())
|
||||
);
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const DivineDaoLibraryScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default DivineDaoLibraryScraper;
|
||||
@@ -1,165 +0,0 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
|
||||
const baseUrl = "https://fastnovel.net";
|
||||
const searchUrl = `https://fastnovel.net/search/`;
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
let totalPages = 39;
|
||||
const url = `${baseUrl}/list/most-popular.html?page=${page}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".film-item").each(function (result) {
|
||||
const novelName = $(this).find(".name").text();
|
||||
const novelCover = $(this).find(".img").attr("data-original");
|
||||
const novelUrl = $(this).find("a").attr("href").substring(1);
|
||||
|
||||
const novel = {
|
||||
sourceId: 3,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return { totalPages, novels };
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = `${baseUrl}/${novelUrl}/`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.sourceId = 3;
|
||||
|
||||
novel.sourceName = "FastNovel";
|
||||
|
||||
novel.url = url;
|
||||
|
||||
novel.novelUrl = novelUrl;
|
||||
|
||||
novel.novelName = $("h1").text();
|
||||
|
||||
novel.novelCover = $(".book-cover").attr("data-original");
|
||||
|
||||
novel.summary = $("div.film-content > p").text();
|
||||
|
||||
novel.author = $("li > label")
|
||||
.filter(function () {
|
||||
return $(this).text().trim() === "Author:";
|
||||
})
|
||||
.next()
|
||||
.text();
|
||||
|
||||
novel.genre = $("li")
|
||||
.filter(function () {
|
||||
return $(this).find("label").text().trim() === "Genre:";
|
||||
})
|
||||
.text()
|
||||
.replace("Genre:", "");
|
||||
|
||||
novel.artist = null;
|
||||
|
||||
novel.status = null;
|
||||
|
||||
novel.status = $("li")
|
||||
.filter(function () {
|
||||
return $(this).find("label").text().trim() === "Status:";
|
||||
})
|
||||
.text()
|
||||
.includes("Completed")
|
||||
? "Completed"
|
||||
: "Ongoing";
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
$(".chapter").each(function (result) {
|
||||
const chapterName = $(this).text();
|
||||
const releaseDate = null;
|
||||
let chapterUrl = $(this).attr("href");
|
||||
chapterUrl = chapterUrl.replace(`/${novelUrl}/`, "");
|
||||
|
||||
const novel = {
|
||||
chapterName,
|
||||
releaseDate,
|
||||
chapterUrl,
|
||||
};
|
||||
|
||||
novelChapters.push(novel);
|
||||
});
|
||||
|
||||
novel.chapters = novelChapters;
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = `${baseUrl}/${novelUrl}/${chapterUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
const chapterName = $(".episode-name").text();
|
||||
|
||||
let chapterText = $("#chapter-body").html();
|
||||
const chapter = {
|
||||
sourceId: 3,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const url = `${searchUrl}${searchTerm}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".film-item").each(function (result) {
|
||||
const novelName = $(this).find("div.title > p.name").text();
|
||||
const novelCover = $(this).find(".img").attr("data-original");
|
||||
const novelUrl = $(this).find("a").attr("href").substring(1);
|
||||
|
||||
const novel = {
|
||||
sourceId: 3,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const fastNovelScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default fastNovelScraper;
|
||||
@@ -1,191 +0,0 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
|
||||
const baseUrl = "https://foxaholic.com/";
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
let totalPages = 5;
|
||||
let url = baseUrl + "novel/page/" + page + "/?m_orderby=rating";
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".page-item-detail").each(function (result) {
|
||||
const novelName = $(this).find(".h5 > a").text();
|
||||
const novelCover = $(this).find("img").attr("data-src");
|
||||
|
||||
let novelUrl = $(this).find(".h5 > a").attr("href").split("/")[4];
|
||||
novelUrl += "/";
|
||||
|
||||
const novel = {
|
||||
sourceId: 22,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return { totalPages, novels };
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = `${baseUrl}novel/${novelUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.sourceId = 22;
|
||||
|
||||
novel.sourceName = "Foxaholic";
|
||||
|
||||
novel.url = url;
|
||||
|
||||
novel.novelUrl = novelUrl;
|
||||
|
||||
novel.novelName = $(".post-title > h1")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
novel.novelCover = $(".summary_image > a > img").attr("data-src");
|
||||
|
||||
$(".post-content_item").each(function (result) {
|
||||
detailName = $(this)
|
||||
.find(".summary-heading > h5")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
detail = $(this)
|
||||
.find(".summary-content")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
switch (detailName) {
|
||||
case "Genre":
|
||||
novel.genre = detail.trim().replace(/[\t\n]/g, ",");
|
||||
break;
|
||||
case "Author":
|
||||
novel.author = detail.trim();
|
||||
break;
|
||||
case "Novel":
|
||||
novel.status = detail.trim();
|
||||
break;
|
||||
}
|
||||
});
|
||||
|
||||
$(".description-summary > div.summary__content").find("em").remove();
|
||||
|
||||
novel.summary = $(".description-summary > div.summary__content")
|
||||
.text()
|
||||
.trim();
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
const novelId = $("#manga-chapters-holder").attr("data-id");
|
||||
|
||||
let formData = new FormData();
|
||||
formData.append("action", "manga_get_chapters");
|
||||
formData.append("manga", novelId);
|
||||
|
||||
const data = await fetch(
|
||||
"https://www.foxaholic.com/wp-admin/admin-ajax.php",
|
||||
{
|
||||
method: "POST",
|
||||
body: formData,
|
||||
}
|
||||
);
|
||||
const text = await data.text();
|
||||
|
||||
$ = cheerio.load(text);
|
||||
|
||||
$(".wp-manga-chapter").each(function (result) {
|
||||
chapterName = $(this)
|
||||
.find("a")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
releaseDate = $(this).find("span").text().trim();
|
||||
|
||||
chapterUrl = $(this).find("a").attr("href").split("/");
|
||||
|
||||
chapterUrl[6]
|
||||
? (chapterUrl = chapterUrl[5] + "/" + chapterUrl[6])
|
||||
: (chapterUrl = chapterUrl[5]);
|
||||
|
||||
novelChapters.push({ chapterName, releaseDate, chapterUrl });
|
||||
});
|
||||
|
||||
novel.chapters = novelChapters.reverse();
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = `${baseUrl}novel/${novelUrl}${chapterUrl}/`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
const chapterName = $("h1#chapter-heading").text();
|
||||
let chapterText = $(".reading-content").html();
|
||||
const chapter = {
|
||||
sourceId: 22,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const url = `${baseUrl}?s=${searchTerm}&post_type=wp-manga`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".c-tabs-item__content").each(function (result) {
|
||||
const novelName = $(this).find(".h4 > a").text();
|
||||
const novelCover = $(this).find("img").attr("data-src");
|
||||
|
||||
let novelUrl = $(this).find(".h4 > a").attr("href").split("/")[4];
|
||||
novelUrl += "/";
|
||||
const novel = {
|
||||
sourceId: 22,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const FoxaHolicScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default FoxaHolicScraper;
|
||||
@@ -1,183 +0,0 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
|
||||
const baseUrl = "https://freewebnovel.com/";
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
let totalPages = 30;
|
||||
let url = baseUrl + "completed-novel/" + page;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".li-row").each(function (result) {
|
||||
const novelName = $(this).find(".tit").text();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
|
||||
let novelUrl = $(this)
|
||||
.find("h3 > a")
|
||||
.attr("href")
|
||||
.replace(".html", "")
|
||||
.slice(1);
|
||||
|
||||
const novel = {
|
||||
sourceId: 13,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return { totalPages, novels };
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = `${baseUrl}${novelUrl.replace("/", "")}.html`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.sourceId = 13;
|
||||
|
||||
novel.sourceName = "FreeWebNovel";
|
||||
|
||||
novel.url = url;
|
||||
|
||||
novel.novelUrl = novelUrl;
|
||||
|
||||
novel.novelName = $("h1.tit").text();
|
||||
|
||||
novel.novelCover = $(".pic > img").attr("src");
|
||||
|
||||
novel.genre = $("[title=Genre]")
|
||||
.next()
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "");
|
||||
|
||||
novel.author = $("[title=Author]")
|
||||
.next()
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "");
|
||||
|
||||
novel.artist = null;
|
||||
|
||||
novel.status = $("[title=Status]")
|
||||
.next()
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "");
|
||||
|
||||
let novelSummary = $(".inner").text().trim();
|
||||
novel.summary = novelSummary;
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
let latestChapter;
|
||||
|
||||
$("h3.tit").each(function (res) {
|
||||
if ($(this).find("a").text() === novel.novelName) {
|
||||
latestChapter = $(this).next().find("span.s3").text().match(/\d+/);
|
||||
}
|
||||
});
|
||||
|
||||
latestChapter = latestChapter[0];
|
||||
|
||||
for (let i = 1; i <= parseInt(latestChapter); i++) {
|
||||
const chapterName = "Chapter " + i;
|
||||
|
||||
const releaseDate = null;
|
||||
|
||||
const chapterUrl = "chapter-" + i;
|
||||
|
||||
const chapter = { chapterName, releaseDate, chapterUrl };
|
||||
|
||||
novelChapters.push(chapter);
|
||||
}
|
||||
|
||||
novel.chapters = novelChapters;
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
let novelId = novelUrl.replace("/", "");
|
||||
|
||||
const url = `${baseUrl}${novelId}/${chapterUrl}.html`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let chapterName = $("h1.tit").text();
|
||||
|
||||
let chapterText = $("div.txt").html();
|
||||
|
||||
const chapter = {
|
||||
sourceId: 13,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const url = baseUrl + "search/";
|
||||
|
||||
const formData = new FormData();
|
||||
formData.append("searchkey", searchTerm);
|
||||
|
||||
const result = await fetch(url, {
|
||||
method: "POST",
|
||||
body: formData,
|
||||
});
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".li-row > .li > .con").each(function (result) {
|
||||
const novelName = $(this).find(".tit").text();
|
||||
const novelCover = $(this).find(".pic > a > img").attr("data-cfsrc");
|
||||
|
||||
let novelUrl = $(this)
|
||||
.find("h3 > a")
|
||||
.attr("href")
|
||||
.replace(".html", "")
|
||||
.slice(1);
|
||||
|
||||
novelUrl += "/";
|
||||
|
||||
const novel = {
|
||||
sourceId: 13,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const FreeWebNovelScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default FreeWebNovelScraper;
|
||||
-170
@@ -1,170 +0,0 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
|
||||
const baseUrl = "https://jpmtl.com/";
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
let totalPages = 157;
|
||||
let url =
|
||||
"https://jpmtl.com/v2/book/show/browse?query=&categories=&content_type=0&direction=0&page=" +
|
||||
page +
|
||||
"&limit=20&type=5&status=all&language=3&exclude_categories=";
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.json();
|
||||
|
||||
let novels = [];
|
||||
|
||||
body.map((item) => {
|
||||
const novelName = item.title;
|
||||
const novelCover = item.cover;
|
||||
const novelUrl = item.id + "/";
|
||||
|
||||
const novel = { sourceId: 14, novelName, novelCover, novelUrl };
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return { totalPages, novels };
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = `${baseUrl}books/${novelUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.sourceId = 14;
|
||||
|
||||
novel.sourceName = "JPMTL";
|
||||
|
||||
novel.url = url;
|
||||
|
||||
novel.novelUrl = novelUrl;
|
||||
|
||||
novel.novelName = $("h1.book-sidebar__title").text();
|
||||
|
||||
novel.novelCover = $(".book-sidebar").find("img").attr("src");
|
||||
|
||||
$(".post-content_item").each(function (result) {
|
||||
detailName = $(this)
|
||||
.find(".summary-heading > h5")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
detail = $(this)
|
||||
.find(".summary-content")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
novel[detailName] = detail;
|
||||
});
|
||||
|
||||
novel.summary = $(".main-book__synopsis").text();
|
||||
|
||||
novel.genre = "";
|
||||
|
||||
$("a.main-book__category").each(function (result) {
|
||||
novel.genre += $(this).text();
|
||||
});
|
||||
|
||||
function capitalizeFirstLetter(string) {
|
||||
return string.charAt(0).toUpperCase() + string.slice(1);
|
||||
}
|
||||
|
||||
const info = $(".book-sidebar__author > .book-sidebar__info")
|
||||
.text()
|
||||
.trim()
|
||||
.split("・");
|
||||
|
||||
if (info) {
|
||||
novel.author = info[0];
|
||||
|
||||
novel.status = capitalizeFirstLetter(info[1]);
|
||||
}
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
const chapterListUrl = `https://jpmtl.com/v2/chapter/${novelUrl}/list?state=published&structured=true&d`;
|
||||
|
||||
const chapterResult = await fetch(chapterListUrl);
|
||||
const volumes = await chapterResult.json();
|
||||
|
||||
volumes.map((volume) => {
|
||||
volume.chapters.map((chapter) => {
|
||||
const chapterName = chapter.title;
|
||||
const releaseDate = chapter.created_at;
|
||||
const chapterUrl = chapter.id;
|
||||
|
||||
const obj = {
|
||||
sourceId: 14,
|
||||
chapterName,
|
||||
releaseDate,
|
||||
chapterUrl,
|
||||
};
|
||||
|
||||
novelChapters.push(obj);
|
||||
});
|
||||
});
|
||||
|
||||
novel.chapters = novelChapters;
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = `${baseUrl}books/${novelUrl}${chapterUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
const chapterName = $(".chapter-content__title").text();
|
||||
let chapterText = $(".chapter-content__content").html();
|
||||
novelUrl += "/";
|
||||
|
||||
const chapter = {
|
||||
sourceId: 14,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const url = `https://jpmtl.com/v2/book/show/browse?query=${searchTerm}&categories=&content_type=2&direction=0&page=1&limit=20&type=5&status=all&language=3&exclude_categories=`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.json();
|
||||
|
||||
const novels = [];
|
||||
|
||||
body.map((item) => {
|
||||
const novelName = item.title;
|
||||
const novelCover = item.cover;
|
||||
const novelUrl = item.id + "/";
|
||||
|
||||
const novel = { sourceId: 14, novelName, novelCover, novelUrl };
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const JPMTLScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default JPMTLScraper;
|
||||
@@ -1,178 +0,0 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
|
||||
const baseUrl = "https://kisslightnovels.info/";
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
let totalPages = 26;
|
||||
const url = `${baseUrl}light-novel/page/${page}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".page-item-detail").each(function (result) {
|
||||
const novelName = $(this).find("h5 > a").text();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
|
||||
let novelUrl = $(this).find("h5 > a").attr("href");
|
||||
novelUrl = novelUrl.replace(`${baseUrl}novel/`, "");
|
||||
|
||||
const novel = {
|
||||
sourceId: 11,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return { totalPages, novels };
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = `${baseUrl}novel/${novelUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.sourceId = 11;
|
||||
|
||||
novel.sourceName = "KissLightNovels";
|
||||
|
||||
novel.url = url;
|
||||
|
||||
novel.novelUrl = novelUrl;
|
||||
|
||||
novel.novelName = $(".post-title > h1")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
novel.novelCover = $(".summary_image > a > img").attr("src");
|
||||
|
||||
$(".post-content_item").each(function (result) {
|
||||
detailName = $(this)
|
||||
.find(".summary-heading > h5")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
detail = $(this)
|
||||
.find(".summary-content")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
switch (detailName) {
|
||||
case "Genre(s)":
|
||||
novel.genre = detail.trim().replace(/[\t\n]/g, ",");
|
||||
break;
|
||||
case "Author(s)":
|
||||
novel.author = detail.trim();
|
||||
break;
|
||||
case "Artist(s)":
|
||||
novel.status = detail.trim();
|
||||
break;
|
||||
}
|
||||
});
|
||||
|
||||
$(".description-summary > div.summary__content").find("em").remove();
|
||||
|
||||
novel.summary = $(".description-summary > div.summary__content")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "");
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
$(".wp-manga-chapter").each(function (result) {
|
||||
chapterName = $(this)
|
||||
.find("a")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
releaseDate = $(this)
|
||||
.find("span")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
chapterUrl = $(this).find("a").attr("href").replace(url, "");
|
||||
|
||||
novelChapters.push({ chapterName, releaseDate, chapterUrl });
|
||||
});
|
||||
|
||||
novel.chapters = novelChapters.reverse();
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = `${baseUrl}novel/${novelUrl}/${chapterUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
$("#snippet-box").remove();
|
||||
|
||||
const chapterName = $("div.text-left > h4").text();
|
||||
|
||||
let chapterText = $(".reading-content").html();
|
||||
const chapter = {
|
||||
sourceId: 11,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const url = `${baseUrl}?s=${searchTerm}&post_type=wp-manga&m_orderby=rating`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".c-tabs-item__content").each(function (result) {
|
||||
const novelName = $(this).find("h4 > a").text();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
|
||||
let novelUrl = $(this).find("h4 > a").attr("href");
|
||||
novelUrl = novelUrl.replace(`${baseUrl}novel/`, "");
|
||||
|
||||
const novel = {
|
||||
sourceId: 11,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const kissLightNovelScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default kissLightNovelScraper;
|
||||
@@ -1,215 +0,0 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
|
||||
const baseUrl = "https://www.lightnovelpub.com/";
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
let totalPages = 33;
|
||||
let url = baseUrl + "browse/all/popular/all/" + page;
|
||||
|
||||
let headers = new Headers({
|
||||
Accept: "application/json",
|
||||
"Content-Type": "application/json",
|
||||
"User-Agent":
|
||||
"'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36",
|
||||
});
|
||||
|
||||
const result = await fetch(url, {
|
||||
method: "GET",
|
||||
headers: headers,
|
||||
});
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".novel-item.ads").remove();
|
||||
|
||||
$(".novel-item").each(function () {
|
||||
const novelName = $(this)
|
||||
.find(".novel-title")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "");
|
||||
|
||||
const novelCover = $(this).find("img").attr("data-src");
|
||||
|
||||
let novelUrl = $(this)
|
||||
.find(".novel-title > a")
|
||||
.attr("href")
|
||||
.replace(`/novel/`, "");
|
||||
novelUrl += "/";
|
||||
|
||||
const novel = {
|
||||
sourceId: 15,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return { totalPages, novels };
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = `${baseUrl}novel/${novelUrl}/`;
|
||||
|
||||
let headers = new Headers({
|
||||
Accept: "application/json",
|
||||
"Content-Type": "application/json",
|
||||
"User-Agent":
|
||||
"'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36",
|
||||
});
|
||||
|
||||
const result = await fetch(url, {
|
||||
method: "GET",
|
||||
headers: headers,
|
||||
});
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.sourceId = 15;
|
||||
|
||||
novel.sourceName = "LightNovelPub";
|
||||
|
||||
novel.url = url;
|
||||
|
||||
novel.novelUrl = novelUrl;
|
||||
|
||||
novel.novelName = $("h1.novel-title")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
novel.novelCover = $("figure.cover > img").attr("data-src");
|
||||
|
||||
novel.genre = "";
|
||||
|
||||
$("div.categories > ul > li").each(function () {
|
||||
novel.genre +=
|
||||
$(this)
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "") + ",";
|
||||
});
|
||||
|
||||
$("div.header-stats > span").each(function () {
|
||||
if ($(this).find("small").text() === "Status") {
|
||||
novel.status = $(this).find("strong").text();
|
||||
}
|
||||
});
|
||||
|
||||
novel.genre = novel.genre.slice(0, -1);
|
||||
|
||||
novel.author = $(".author > a > span").text();
|
||||
|
||||
novel.summary = $(".summary > .content").text().trim();
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
let totalChapters;
|
||||
|
||||
totalChapters = $(".header-stats > span").first().text().match(/\d+/)[0];
|
||||
|
||||
for (let i = 1; i <= totalChapters; i++) {
|
||||
const chapterName = "Chapter " + i;
|
||||
|
||||
const releaseDate = null;
|
||||
|
||||
const chapterUrl = "chapter-" + i;
|
||||
|
||||
const chapter = { chapterName, releaseDate, chapterUrl };
|
||||
|
||||
novelChapters.push(chapter);
|
||||
}
|
||||
|
||||
novel.chapters = novelChapters;
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = `${baseUrl}novel/${novelUrl}/${chapterUrl}`;
|
||||
|
||||
let headers = new Headers({
|
||||
Accept: "application/json",
|
||||
"Content-Type": "application/json",
|
||||
"User-Agent":
|
||||
"'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36",
|
||||
});
|
||||
|
||||
const result = await fetch(url, {
|
||||
method: "GET",
|
||||
headers: headers,
|
||||
});
|
||||
const body = await result.text();
|
||||
$ = cheerio.load(body);
|
||||
|
||||
const chapterName = $("h2").text();
|
||||
let chapterText = $("#chapter-container").html();
|
||||
const chapter = {
|
||||
sourceId: 15,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const url = `${baseUrl}lnwsearchlive?inputContent=${searchTerm}`;
|
||||
|
||||
let headers = new Headers({
|
||||
Accept: "application/json",
|
||||
"Content-Type": "application/json",
|
||||
"User-Agent":
|
||||
"'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36",
|
||||
});
|
||||
|
||||
const result = await fetch(url, {
|
||||
method: "GET",
|
||||
headers: headers,
|
||||
});
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
let results = JSON.parse($("body").text());
|
||||
|
||||
$ = cheerio.load(results.resultview);
|
||||
|
||||
$(".novel-item").each(function () {
|
||||
const novelName = $(this).find("h4.novel-title").text();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
|
||||
let novelUrl = $(this).find("a").attr("href").replace(`/novel/`, "");
|
||||
novelUrl += "/";
|
||||
|
||||
const novel = {
|
||||
sourceId: 15,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const lightNovelPubScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default lightNovelPubScraper;
|
||||
@@ -1,159 +0,0 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
import NovelUpdatesScraper from "./novelupdates";
|
||||
|
||||
const sourceId = 72;
|
||||
const sourceName = "LightNovelUpdates";
|
||||
const baseUrl = "https://www.lightnovelupdates.com/";
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
let url = `${baseUrl}novel/page/${page}/?m_orderby=rating`;
|
||||
let totalPages = 500;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".page-item-detail").each(function () {
|
||||
const novelName = $(this).find(".h5 > a").text();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
const novelUrl = $(this).find(".h5 > a").attr("href");
|
||||
|
||||
const novel = {
|
||||
sourceId,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return { totalPages, novels };
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = novelUrl;
|
||||
|
||||
console.log(url);
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = { sourceId, sourceName, novelUrl, url };
|
||||
|
||||
novel.novelName = $(".post-title > h1").text().trim();
|
||||
novel.novelCover =
|
||||
$(".summary_image > a > img").attr("src") ||
|
||||
"https://github.com/LNReader/lnreader-sources/blob/main/src/coverNotAvailable.jpg?raw=true";
|
||||
|
||||
$(".post-content_item").each(function (result) {
|
||||
detailName = $(this).find(".summary-heading > h5").text().trim();
|
||||
detail = $(this).find(".summary-content").text().trim();
|
||||
|
||||
switch (detailName) {
|
||||
case "Genre(s)":
|
||||
novel.genre = detail.trim().replace(/[\t\n]/g, ",");
|
||||
break;
|
||||
case "Author(s)":
|
||||
novel.author = detail.trim();
|
||||
break;
|
||||
case "Artist(s)":
|
||||
novel.status = detail.trim();
|
||||
break;
|
||||
}
|
||||
});
|
||||
|
||||
novel.summary = $("div.summary__content").text();
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
const novelId = $(".rating-post-id").attr("value");
|
||||
|
||||
let formData = new FormData();
|
||||
formData.append("action", "manga_get_chapters");
|
||||
formData.append("manga", novelId);
|
||||
|
||||
const data = await fetch(
|
||||
"https://www.lightnovelupdates.com/wp-admin/admin-ajax.php",
|
||||
{
|
||||
method: "POST",
|
||||
body: formData,
|
||||
}
|
||||
);
|
||||
const text = await data.text();
|
||||
|
||||
$ = cheerio.load(text);
|
||||
|
||||
$(".wp-manga-chapter").each(function () {
|
||||
const chapterName = $(this).find("a").text().trim();
|
||||
const releaseDate = $(this).find("span").text().trim();
|
||||
const chapterUrl = $(this).find("a").attr("href");
|
||||
|
||||
const chapter = { chapterName, releaseDate, chapterUrl };
|
||||
|
||||
novelChapters.push(chapter);
|
||||
});
|
||||
|
||||
novel.chapters = novelChapters.reverse();
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const chapterName = "";
|
||||
const chapterText = (
|
||||
await NovelUpdatesScraper.parseChapter(novelUrl, chapterUrl)
|
||||
).chapterText;
|
||||
|
||||
const chapter = {
|
||||
sourceId,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const url = `${baseUrl}?s=${searchTerm}&post_type=wp-manga&m_orderby=rating`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".c-tabs-item__content").each(function (result) {
|
||||
const novelName = $(this).find(".h4 > a").text();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
const novelUrl = $(this).find(".h4 > a").attr("href");
|
||||
|
||||
const novel = {
|
||||
sourceId,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const LightNovelUpdatesScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default LightNovelUpdatesScraper;
|
||||
-194
@@ -1,194 +0,0 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
import { showToast } from "../../hooks/showToast";
|
||||
import { htmlToText } from "../helpers/htmlToText";
|
||||
|
||||
const baseUrl = "https://lnmtl.com/";
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
const totalPages = 63;
|
||||
let url = baseUrl + "novel?page=" + page;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".media").each(function (result) {
|
||||
const novelName = $(this).find("h4").text();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
|
||||
let novelUrl = $(this).find("h4 > a").attr("href");
|
||||
novelUrl = novelUrl.replace(baseUrl + "novel/", "");
|
||||
|
||||
const novel = {
|
||||
sourceId: 37,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return { totalPages, novels };
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
showToast("LNMTL might take around 20-30 seconds.");
|
||||
|
||||
const url = baseUrl + "novel/" + novelUrl;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.sourceId = 37;
|
||||
|
||||
novel.sourceName = "LNMTL";
|
||||
|
||||
novel.url = url;
|
||||
|
||||
novel.novelUrl = novelUrl;
|
||||
|
||||
novel.novelName = $(".novel-name").text();
|
||||
|
||||
novel.novelCover = $("div.novel").find("img").attr("src");
|
||||
|
||||
novel.summary = $("div.description").text().trim();
|
||||
|
||||
novel.author = $(
|
||||
"main > div:nth-child(3) > div > div.col-lg-3.col-md-4 > div:nth-child(2) > div.panel-body > dl:nth-child(1) > dd > span"
|
||||
).text();
|
||||
|
||||
novel.status = $(
|
||||
"main > div:nth-child(3) > div > div.col-lg-3.col-md-4 > div:nth-child(2) > div.panel-body > dl:nth-child(2) > dd"
|
||||
)
|
||||
.text()
|
||||
.trim();
|
||||
|
||||
novel.genre = $(
|
||||
"main > div.container > div > div.col-lg-3.col-md-4 > div:nth-child(4) > div.panel-body > ul"
|
||||
)
|
||||
.text()
|
||||
.trim()
|
||||
.replace(/\s\s/g, ",");
|
||||
|
||||
let volumes = JSON.parse(
|
||||
$("main")
|
||||
.next()
|
||||
.html()
|
||||
.match(/lnmtl.volumes = \[(.*?)\]/)[0]
|
||||
.replace("lnmtl.volumes = ", "")
|
||||
);
|
||||
|
||||
let chapters = [];
|
||||
|
||||
volumes = volumes.map((volume) => volume.id);
|
||||
|
||||
for (const volume of volumes) {
|
||||
let volumeData = await fetch(
|
||||
`https://lnmtl.com/chapter?page=1&volumeId=${volume}`
|
||||
);
|
||||
volumeData = await volumeData.json();
|
||||
|
||||
// volumeData = volumeData.data.map((volume) => volume.slug);
|
||||
|
||||
for (let i = 1; i <= volumeData.last_page; i++) {
|
||||
let chapterData = await fetch(
|
||||
`https://lnmtl.com/chapter?page=${i}&volumeId=${volume}`
|
||||
);
|
||||
chapterData = await chapterData.json();
|
||||
|
||||
chapterData = chapterData.data.map((chapter) => ({
|
||||
chapterName: `#${chapter.number} ${chapter.title}`,
|
||||
chapterUrl: chapter.slug,
|
||||
releaseDate: chapter.created_at,
|
||||
}));
|
||||
|
||||
chapters.push(...chapterData);
|
||||
}
|
||||
}
|
||||
|
||||
novel.chapters = chapters;
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = `${baseUrl}chapter/${chapterUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let chapterName = $("h3 > span.chapter-title").text().trim();
|
||||
|
||||
$(".original").remove();
|
||||
|
||||
let chapterText = $(".chapter-body").html();
|
||||
|
||||
if (!chapterText) {
|
||||
chapterText = $(".alert.alert-warning").text();
|
||||
}
|
||||
|
||||
chapterText =
|
||||
chapterName +
|
||||
"\n\n" +
|
||||
htmlToText(chapterText, { removeLineBreaks: false });
|
||||
|
||||
const chapter = {
|
||||
sourceId: 37,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const url = "https://lnmtl.com/term";
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = $("footer")
|
||||
.next()
|
||||
.next()
|
||||
.html()
|
||||
.match(/local: \[(.*?)\]/)[0]
|
||||
.replace("local: ", "");
|
||||
|
||||
novels = JSON.parse(novels);
|
||||
|
||||
novels = novels.filter((novel) =>
|
||||
novel.name.toLowerCase().includes(searchTerm.toLowerCase())
|
||||
);
|
||||
|
||||
novels = novels.map((novel) => ({
|
||||
sourceId: 37,
|
||||
novelName: novel.name,
|
||||
novelUrl: novel.slug,
|
||||
novelCover: novel.image,
|
||||
}));
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const LNMTLScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default LNMTLScraper;
|
||||
@@ -1,202 +0,0 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
|
||||
const baseUrl = "https://www.mtlnovel.com";
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
let totalPages = 10;
|
||||
const url = `${baseUrl}/alltime-rank/page/${page}`;
|
||||
|
||||
console.log(url);
|
||||
|
||||
let headers = new Headers({
|
||||
referer: "https://www.mtlnovel.com/",
|
||||
"User-Agent":
|
||||
"'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36",
|
||||
});
|
||||
|
||||
const result = await fetch(url, {
|
||||
method: "GET",
|
||||
headers: headers,
|
||||
});
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$("div.box.wide").each(function (result) {
|
||||
const novelName = $(this).find("a.list-title").text().slice(4);
|
||||
const novelCover = $(this).find("amp-img").attr("src");
|
||||
|
||||
let novelUrl = $(this).find("a.list-title").attr("href");
|
||||
novelUrl = novelUrl.replace("https://www.mtlnovel.com/", "");
|
||||
|
||||
const novel = {
|
||||
sourceId: 5,
|
||||
novelUrl,
|
||||
novelName,
|
||||
novelCover,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return { totalPages, novels };
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = `${baseUrl}/${novelUrl}`;
|
||||
|
||||
let headers = new Headers({
|
||||
referer: "https://www.mtlnovel.com/alltime-rank/",
|
||||
"User-Agent":
|
||||
"'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36",
|
||||
});
|
||||
|
||||
const result = await fetch(url, {
|
||||
method: "GET",
|
||||
headers: headers,
|
||||
});
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.sourceId = 5;
|
||||
|
||||
novel.sourceName = "MTLNovel";
|
||||
|
||||
novel.url = url;
|
||||
|
||||
novel.novelUrl = novelUrl;
|
||||
|
||||
novel.novelName = $("h1.entry-title").text();
|
||||
|
||||
novel.novelCover = $(".nov-head > amp-img").attr("src");
|
||||
|
||||
novel.summary = $("div.desc > h2").next().text();
|
||||
|
||||
novel.author = $("tr > td")
|
||||
.filter(function () {
|
||||
return $(this).prev().text().trim() === "Author";
|
||||
})
|
||||
.next()
|
||||
.text()
|
||||
.replace("Auhtor:", "");
|
||||
|
||||
novel.status = $("tr > td")
|
||||
.filter(function () {
|
||||
return $(this).prev().text().trim() === "Status";
|
||||
})
|
||||
.next()
|
||||
.text()
|
||||
.replace("Status:", "");
|
||||
|
||||
novel.genre = $("td")
|
||||
.filter(function () {
|
||||
return $(this).prev().text().trim() === "Genre";
|
||||
})
|
||||
.next()
|
||||
.text()
|
||||
.replace("Genre:", "");
|
||||
|
||||
novel.artist = null;
|
||||
|
||||
const chapterListUrl = url + "/chapter-list/";
|
||||
|
||||
const getChapters = async () => {
|
||||
const result = await fetch(chapterListUrl);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
$("div.ch-list")
|
||||
.find("a.ch-link")
|
||||
.each(function (result) {
|
||||
const chapterName = $(this).text().replace("~ ", "");
|
||||
const releaseDate = null;
|
||||
|
||||
let chapterUrl = $(this).attr("href");
|
||||
chapterUrl = chapterUrl.replace(
|
||||
`https://www.mtlnovel.com/${novelUrl}`,
|
||||
""
|
||||
);
|
||||
|
||||
novelChapters.push({
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
releaseDate,
|
||||
});
|
||||
});
|
||||
return novelChapters.reverse();
|
||||
};
|
||||
|
||||
novel.chapters = await getChapters();
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = `${baseUrl}/${novelUrl}${chapterUrl}`;
|
||||
console.log(url);
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
const chapterName = $("h1.main-title").text();
|
||||
let chapterText = $("div.par").html();
|
||||
const chapter = {
|
||||
sourceId: 5,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const searchUrl =
|
||||
"https://www.mtlnovel.com/wp-admin/admin-ajax.php?action=autosuggest&q=" +
|
||||
searchTerm +
|
||||
"&__amp_source_origin=https%3A%2F%2Fwww.mtlnovel.com";
|
||||
|
||||
const res = await fetch(searchUrl);
|
||||
const result = await res.text();
|
||||
|
||||
$ = cheerio.load(result);
|
||||
|
||||
let body = JSON.parse($("body").text());
|
||||
|
||||
let novels = [];
|
||||
|
||||
body.items[0].results.map((item) => {
|
||||
const novelName = item.title.replace(/<\/?strong>/g, "");
|
||||
const novelCover = item.thumbnail;
|
||||
const novelUrl = item.permalink.replace(
|
||||
"https://www.mtlnovel.com/",
|
||||
""
|
||||
);
|
||||
|
||||
const novel = { sourceId: 5, novelName, novelCover, novelUrl };
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const mtlNovelScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default mtlNovelScraper;
|
||||
@@ -1,186 +0,0 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
|
||||
const baseUrl = "https://novelfull.com";
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
let totalPages = 42;
|
||||
const url = `${baseUrl}/most-popular?page=${page}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$("div.col-truyen-main > div.list-truyen > .row").each(function (result) {
|
||||
const novelName = $(this).find("h3.truyen-title > a").text();
|
||||
|
||||
let novelCover = $(this).find("img").attr("src");
|
||||
novelCover = baseUrl + novelCover;
|
||||
|
||||
let novelUrl = $(this)
|
||||
.find("h3.truyen-title > a")
|
||||
.attr("href")
|
||||
.replace(".html", "")
|
||||
.substring(1);
|
||||
novelUrl = `${novelUrl}/`;
|
||||
|
||||
const novel = {
|
||||
sourceId: 8,
|
||||
novelUrl,
|
||||
novelName,
|
||||
novelCover,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return { totalPages, novels };
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = `${baseUrl}/${novelUrl.slice(0, -1)}.html`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.sourceId = 8;
|
||||
|
||||
novel.sourceName = "NovelFull";
|
||||
|
||||
novel.url = url;
|
||||
|
||||
novel.novelUrl = novelUrl;
|
||||
|
||||
novel.novelName = $("div.book > img").attr("alt");
|
||||
|
||||
novel.novelCover = baseUrl + $("div.book > img").attr("src");
|
||||
|
||||
novel.summary = $("div.desc-text").text().trim();
|
||||
|
||||
novel.author = $("div.info > div > h3")
|
||||
.filter(function () {
|
||||
return $(this).text().trim() === "Author:";
|
||||
})
|
||||
.siblings()
|
||||
.text();
|
||||
|
||||
novel.genre = $("div.info > div")
|
||||
.filter(function () {
|
||||
return $(this).find("h3").text().trim() === "Genre:";
|
||||
})
|
||||
.text()
|
||||
.replace("Genre:", "");
|
||||
|
||||
novel.artist = null;
|
||||
|
||||
novel.status = $("div.info > div > h3")
|
||||
.filter(function () {
|
||||
return $(this).text().trim() === "Status:";
|
||||
})
|
||||
.next()
|
||||
.text();
|
||||
|
||||
const novelId = $("#rating").attr("data-novel-id");
|
||||
|
||||
const getChapters = async (novelId) => {
|
||||
const chapterListUrl =
|
||||
baseUrl + "/ajax/chapter-option?novelId=" + novelId;
|
||||
|
||||
const data = await fetch(chapterListUrl);
|
||||
const chapters = await data.text();
|
||||
|
||||
$ = cheerio.load(chapters);
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
$("select > option").each(function (result) {
|
||||
let chapterName = $(this).text();
|
||||
let releaseDate = null;
|
||||
let chapterUrl = $(this).attr("value");
|
||||
chapterUrl = chapterUrl.replace(`/${novelUrl}`, "");
|
||||
|
||||
novelChapters.push({
|
||||
chapterName,
|
||||
releaseDate,
|
||||
chapterUrl,
|
||||
});
|
||||
});
|
||||
return novelChapters;
|
||||
};
|
||||
|
||||
novel.chapters = await getChapters(novelId);
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = `${baseUrl}/${novelUrl}${chapterUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
const chapterName = $(".chapter-title").attr("title");
|
||||
let chapterText = $("#chapter-content").html();
|
||||
const chapter = {
|
||||
sourceId: 8,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const searchUrl = `https://novelfull.com/search?keyword=`;
|
||||
|
||||
const url = `${searchUrl}${searchTerm}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$("div.col-truyen-main > div.list-truyen > .row").each(function (result) {
|
||||
let novelUrl = $(this)
|
||||
.find("h3.truyen-title > a")
|
||||
.attr("href")
|
||||
.replace(".html", "")
|
||||
.substring(1);
|
||||
novelUrl = `${novelUrl}/`;
|
||||
|
||||
const novelName = $(this).find("h3.truyen-title > a").text();
|
||||
const novelCover = baseUrl + $(this).find("img").attr("src");
|
||||
|
||||
const novel = {
|
||||
sourceId: 8,
|
||||
novelUrl,
|
||||
novelName,
|
||||
novelCover,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const novelFullScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default novelFullScraper;
|
||||
@@ -1,161 +0,0 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
|
||||
const baseUrl = "https://www.novelhall.com/";
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
let totalPages = 1;
|
||||
const result = await fetch(baseUrl);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$("div.section1")
|
||||
.find("li")
|
||||
.each(function (result) {
|
||||
const novelName = $(this).find(".book-info > h2 > a").text();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
const novelUrl = $(this).find("a").attr("href").substring(1);
|
||||
|
||||
const novel = {
|
||||
sourceId: 6,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return { totalPages, novels };
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = `${baseUrl}${novelUrl}/`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.sourceId = 6;
|
||||
|
||||
novel.sourceName = "NovelHall";
|
||||
|
||||
novel.url = url;
|
||||
|
||||
novel.novelUrl = novelUrl;
|
||||
|
||||
novel.novelName = $("h1").text();
|
||||
|
||||
novel.novelCover = $("div.book-img > img").attr("src");
|
||||
|
||||
novel.summary = $("div.intro")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "");
|
||||
|
||||
novel.author = $("span.blue").first().text().replace("Author：", "");
|
||||
|
||||
novel.genre = $("a.red").text();
|
||||
|
||||
novel.artist = null;
|
||||
|
||||
novel.status = $("span.blue").first().next().text().replace("Status：", "");
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
$("div.book-catalog.hidden-xs#morelist")
|
||||
.find("li.post-11")
|
||||
.each(function (result) {
|
||||
let chapterName = $(this).find("a").text();
|
||||
|
||||
let releaseDate = null;
|
||||
|
||||
let chapterUrl = $(this).find("a").attr("href");
|
||||
|
||||
if (chapterUrl) {
|
||||
chapterUrl = chapterUrl.replace(`/${novelUrl}/`, "");
|
||||
}
|
||||
|
||||
const chapter = {
|
||||
chapterName,
|
||||
releaseDate,
|
||||
chapterUrl,
|
||||
};
|
||||
|
||||
novelChapters.push(chapter);
|
||||
});
|
||||
|
||||
novel.chapters = novelChapters;
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = `${baseUrl}${novelUrl}/${chapterUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
const chapterName = $("h1").text();
|
||||
let chapterText = $("div.entry-content").html();
|
||||
chapter = {
|
||||
sourceId: 6,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const searchUrl = `${baseUrl}index.php?s=so&module=book&keyword=`;
|
||||
|
||||
const url = `${searchUrl}${searchTerm}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$("tr").each(function (result) {
|
||||
let novelName = $(this).find("td:nth-child(2)").text();
|
||||
novelName = novelName.replace(/[\t\n]/g, "");
|
||||
|
||||
const novelCover = "https://cdn.novelupdates.com/imgmid/noimagemid.jpg";
|
||||
|
||||
let novelUrl = $(this).find("td:nth-child(2) >").attr("href");
|
||||
if (novelUrl) {
|
||||
novelUrl = novelUrl.slice(1);
|
||||
}
|
||||
|
||||
novel = {
|
||||
sourceId: 6,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const novelhallScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default novelhallScraper;
|
||||
@@ -1,158 +0,0 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
import { Status } from "../helpers/constants";
|
||||
|
||||
const sourceId = 71;
|
||||
const sourceName = "NovelOnlineFull.com";
|
||||
const baseUrl = "https://novelonlinefull.com/";
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
let url =
|
||||
baseUrl + "novel_list?type=topview&category=all&state=all&page=" + page;
|
||||
const totalPages = 2155;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".update_item.list_category").each(function () {
|
||||
const novelName = $(this).find("h3").text().trim();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
|
||||
let novelUrl = $(this).find("h3 > a").attr("href");
|
||||
novelUrl = novelUrl.replace(baseUrl + "novel/", "");
|
||||
|
||||
const novel = {
|
||||
sourceId,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return { novels, totalPages };
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = baseUrl + "novel/" + novelUrl;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = { sourceId, sourceName, url, novelUrl };
|
||||
|
||||
novel.novelName = $("h1").text().trim();
|
||||
|
||||
novel.novelCover = $(".info_image > img").attr("src");
|
||||
|
||||
novel.summary = $("#noidungm").text().trim();
|
||||
|
||||
novel.genre = $("div.truyen_if_wrap > ul > li:nth-child(3)")
|
||||
.text()
|
||||
.replace("GENRES: ", "")
|
||||
.replace(/ - /g, ",")
|
||||
.trim();
|
||||
|
||||
console.log(novel.genre);
|
||||
|
||||
novel.status =
|
||||
$(" div.truyen_if_wrap > ul > li:nth-child(4) > a")
|
||||
.text()
|
||||
.replace("STATUS", "")
|
||||
.trim() === "ONGOING"
|
||||
? Status.ONGOING
|
||||
: Status.COMPLETED;
|
||||
|
||||
novel.author = $("div.truyen_if_wrap > ul > li:nth-child(2)")
|
||||
.text()
|
||||
.replace("Author(s):", "")
|
||||
.trim();
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
$(".chapter-list")
|
||||
.find(".row")
|
||||
.each(function () {
|
||||
const releaseDate = $(this).find("span:nth-child(2)").text();
|
||||
|
||||
const chapterName = $(this).find("span:nth-child(1) > a").text();
|
||||
const chapterUrl = $(this).find("a").attr("href");
|
||||
|
||||
novelChapters.push({
|
||||
chapterName,
|
||||
chapterUrl,
|
||||
releaseDate,
|
||||
});
|
||||
});
|
||||
|
||||
novel.chapters = novelChapters.reverse();
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = chapterUrl;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let chapterName = $(".name_chapter").text();
|
||||
let chapterText = $("#vung_doc").html();
|
||||
|
||||
const chapter = {
|
||||
sourceId,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const url = baseUrl + "search_novels/" + searchTerm;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".update_item.list_category").each(function () {
|
||||
const novelName = $(this).find("h3").text();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
|
||||
let novelUrl = $(this).find("h3 > a").attr("href");
|
||||
novelUrl = novelUrl.replace(baseUrl + "novel/", "");
|
||||
|
||||
const novel = {
|
||||
sourceId,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const NovelOnlineFullScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default NovelOnlineFullScraper;
|
||||
@@ -1,177 +0,0 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
|
||||
import moment from "moment";
|
||||
|
||||
const baseUrl = "https://www.novelpassion.com/";
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
const totalPages = 42;
|
||||
let url = baseUrl + "category/all?p=" + page + "&s=1&f=4";
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$("div.j_bookList.lis-mn.ddc")
|
||||
.find("li")
|
||||
.each(function (result) {
|
||||
const novelName = $(this).find("h3").text();
|
||||
const novelCover =
|
||||
"https://www.novelpassion.com" +
|
||||
$(this).find("img").attr("src");
|
||||
|
||||
let novelUrl = $(this).find("div.pr > a").attr("href");
|
||||
novelUrl = novelUrl.replace("/novel/", "");
|
||||
|
||||
const novel = {
|
||||
sourceId: 33,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return { totalPages, novels };
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = baseUrl + "novel/" + novelUrl;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.sourceId = 33;
|
||||
|
||||
novel.sourceName = "NovelPassion";
|
||||
|
||||
novel.url = url;
|
||||
|
||||
novel.novelUrl = novelUrl;
|
||||
|
||||
novelName = $("h2.mh").text();
|
||||
novel.novelName = novelName;
|
||||
|
||||
novel.novelCover = baseUrl + $("i.g_thumb > img").attr("src");
|
||||
|
||||
let novelSummary = $("div.j_synopsis").text().trim();
|
||||
novel.summary = novelSummary;
|
||||
|
||||
$("address.lh20.m14.pr").each(function (result) {
|
||||
detailName = $(this).find("p.ell.dib.vam").text().trim();
|
||||
|
||||
detail = $(this).find("div.dns").text().trim();
|
||||
|
||||
switch (detailName) {
|
||||
case "Categories:":
|
||||
novel.genre = detail.trim().replace(/ - /g, ",");
|
||||
break;
|
||||
case "Author:":
|
||||
novel.author = detail.trim().replace(/ - /g, ", ");
|
||||
break;
|
||||
}
|
||||
|
||||
if (detailName.includes("Status:")) {
|
||||
novel.status = detailName.replace("Status:", "").trim();
|
||||
}
|
||||
});
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
$(".content-list")
|
||||
.find("li")
|
||||
.each(function (result) {
|
||||
const chapterName = $(this).find("span.sp1").text();
|
||||
|
||||
let releaseDate = $(this).find("i.sp2").text();
|
||||
releaseDate = moment(new Date(releaseDate)).format("MM/DD/YY");
|
||||
|
||||
const chapterUrl = $(this)
|
||||
.find("a")
|
||||
.attr("href")
|
||||
.replace("/novel/" + novelUrl + "/", "");
|
||||
|
||||
novelChapters.push({
|
||||
chapterName,
|
||||
releaseDate,
|
||||
chapterUrl,
|
||||
});
|
||||
});
|
||||
|
||||
novel.chapters = novelChapters.reverse();
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = `${baseUrl}novel/${novelUrl}/${chapterUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let chapterName = $("div.cha-content").find("strong").text();
|
||||
|
||||
let chapterText = $("div.cha-content").html();
|
||||
const chapter = {
|
||||
sourceId: 33,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const url = baseUrl + "search?keyword=" + searchTerm;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$("div.j_bookList.lis-mn.ddc")
|
||||
.find("li")
|
||||
.each(function (result) {
|
||||
const novelName = $(this).find("h3").text();
|
||||
const novelCover =
|
||||
"https://www.novelpassion.com" +
|
||||
$(this).find("img").attr("src");
|
||||
|
||||
let novelUrl = $(this).find("div.pr > a").attr("href");
|
||||
novelUrl = novelUrl.replace("/novel/", "");
|
||||
|
||||
const novel = {
|
||||
sourceId: 33,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const NovelPassionScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default NovelPassionScraper;
|
||||
@@ -1,174 +0,0 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
|
||||
const baseUrl = "https://noveltrench.com/";
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
let totalPages = 270;
|
||||
let url = baseUrl + "manga/page/" + page;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".page-item-detail").each(function (result) {
|
||||
const novelName = $(this).find(".h5 > a").text();
|
||||
const novelCover = $(this).find("img").attr("data-src");
|
||||
|
||||
let novelUrl = $(this).find(".h5 > a").attr("href");
|
||||
novelUrl = novelUrl.replace(`${baseUrl}manga/`, "");
|
||||
|
||||
const novel = {
|
||||
sourceId: 9,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return { totalPages, novels };
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = `${baseUrl}manga/${novelUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.sourceId = 9;
|
||||
|
||||
novel.sourceName = "NovelTrench";
|
||||
|
||||
novel.url = url;
|
||||
|
||||
novel.novelUrl = novelUrl;
|
||||
|
||||
novel.novelName = $(".post-title > h1")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
novel.novelCover = $(".summary_image > a > img").attr("data-src");
|
||||
|
||||
$(".post-content_item").each(function (result) {
|
||||
detailName = $(this)
|
||||
.find(".summary-heading > h5")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
detail = $(this)
|
||||
.find(".summary-content")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
switch (detailName) {
|
||||
case "Genre(s)":
|
||||
novel.genre = detail.trim().replace(/[\t\n]/g, ",");
|
||||
break;
|
||||
case "Author(s)":
|
||||
novel.author = detail.trim();
|
||||
break;
|
||||
case "Artist(s)":
|
||||
novel.status = detail.trim();
|
||||
break;
|
||||
}
|
||||
});
|
||||
|
||||
$(".description-summary > div.summary__content").find("em").remove();
|
||||
|
||||
novel.summary = $(".description-summary > div.summary__content")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "");
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
$(".wp-manga-chapter").each(function (result) {
|
||||
chapterName = $(this)
|
||||
.find("a")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
releaseDate = $(this)
|
||||
.find("span")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
chapterUrl = $(this).find("a").attr("href").replace(url, "");
|
||||
novelChapters.push({ chapterName, releaseDate, chapterUrl });
|
||||
});
|
||||
|
||||
novel.chapters = novelChapters.reverse();
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = `${baseUrl}manga/${novelUrl}${chapterUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
const chapterName = $("h1#chapter-heading").text();
|
||||
let chapterText = $(".reading-content").html();
|
||||
const chapter = {
|
||||
sourceId: 9,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const url = `${baseUrl}?s=${searchTerm}&post_type=wp-manga`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".c-tabs-item__content").each(function (result) {
|
||||
const novelName = $(this).find(".h4 > a").text();
|
||||
const novelCover = $(this).find("img").attr("data-src");
|
||||
|
||||
let novelUrl = $(this).find(".h4 > a").attr("href");
|
||||
novelUrl = novelUrl.replace(`${baseUrl}manga/`, "");
|
||||
|
||||
const novel = {
|
||||
sourceId: 9,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const novelTrenchScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default novelTrenchScraper;
|
||||
@@ -1,310 +0,0 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
|
||||
const sourceId = 50;
|
||||
|
||||
const sourceName = "Novel Updates";
|
||||
|
||||
const baseUrl = "https://www.novelupdates.com/";
|
||||
|
||||
let headers = new Headers({
|
||||
"User-Agent":
|
||||
"'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36",
|
||||
});
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
const totalPages = 100;
|
||||
let url = `${baseUrl}series-ranking/?rank=week&pg=` + page;
|
||||
|
||||
const result = await fetch(url, {
|
||||
method: "GET",
|
||||
headers: headers,
|
||||
});
|
||||
const body = await result.text();
|
||||
|
||||
const $ = cheerio.load(body);
|
||||
|
||||
const novels = [];
|
||||
|
||||
$("div.search_main_box_nu").each(function (res) {
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
const novelName = $(this).find(".search_title > a").text();
|
||||
const novelUrl = $(this)
|
||||
.find(".search_title > a")
|
||||
.attr("href")
|
||||
.split("/")[4];
|
||||
|
||||
const novel = {
|
||||
sourceId,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return { totalPages, novels };
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = `${baseUrl}series/${novelUrl}`;
|
||||
|
||||
const result = await fetch(url, {
|
||||
method: "GET",
|
||||
headers: headers,
|
||||
});
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {
|
||||
sourceId,
|
||||
sourceName,
|
||||
url,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novel.novelName = $(".seriestitlenu").text();
|
||||
|
||||
novel.novelCover = $(".seriesimg > img").attr("src");
|
||||
|
||||
novel.author = $("#showauthors").text().trim();
|
||||
|
||||
novel.genre = $("#seriesgenre").text().trim().replace(/\s/g, ",");
|
||||
|
||||
novel.status = $("#editstatus").text().includes("Ongoing")
|
||||
? "Ongoing"
|
||||
: "Completed";
|
||||
|
||||
let type = $("#showtype").text().trim();
|
||||
|
||||
let summary = $("#editdescription").text().trim();
|
||||
|
||||
novel.summary = summary + `\n\nType: ${type}`;
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
const novelId = $("input#mypostid").attr("value");
|
||||
|
||||
let formData = new FormData();
|
||||
formData.append("action", "nd_getchapters");
|
||||
formData.append("mygrr", 0);
|
||||
formData.append("mypostid", parseInt(novelId));
|
||||
|
||||
const data = await fetch(
|
||||
"https://www.novelupdates.com/wp-admin/admin-ajax.php",
|
||||
{
|
||||
method: "POST",
|
||||
headers,
|
||||
body: formData,
|
||||
}
|
||||
);
|
||||
const text = await data.text();
|
||||
|
||||
$ = cheerio.load(text);
|
||||
|
||||
$("li.sp_li_chp").each(function () {
|
||||
const chapterName = $(this).text().trim();
|
||||
|
||||
const releaseDate = null;
|
||||
|
||||
const chapterUrl =
|
||||
"https:" + $(this).find("a").first().next().attr("href");
|
||||
|
||||
novelChapters.push({ chapterName, releaseDate, chapterUrl });
|
||||
});
|
||||
|
||||
novel.chapters = novelChapters.reverse();
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
function getLocation(href) {
|
||||
var match = href.match(
|
||||
/^(https?\:)\/\/(([^:\/?#]*)(?:\:([0-9]+))?)([\/]{0,1}[^?#]*)(\?[^#]*|)(#.*|)$/
|
||||
);
|
||||
return match && `${match[1]}//${match[3]}`;
|
||||
}
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = chapterUrl;
|
||||
|
||||
// console.log("Original URL: ", url);
|
||||
|
||||
let result, body;
|
||||
|
||||
let chapterName = "";
|
||||
|
||||
try {
|
||||
result = await fetch(url, {
|
||||
method: "GET",
|
||||
headers: headers,
|
||||
});
|
||||
body = await result.text();
|
||||
|
||||
// console.log(result.url);
|
||||
|
||||
// console.log("Redirected URL: ", result.url);
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let isWuxiaWorld = result.url.toLowerCase().includes("wuxiaworld");
|
||||
|
||||
let isBlogspot = result.url.toLowerCase().includes("blogspot");
|
||||
|
||||
let isTumblr = result.url.toLowerCase().includes("tumblr");
|
||||
|
||||
/**
|
||||
* Checks if its a wwordpress site
|
||||
*/
|
||||
let isWordPress =
|
||||
$('meta[name="generator"]').attr("content") || $("footer").text();
|
||||
|
||||
if (isWordPress) {
|
||||
isWordPress =
|
||||
isWordPress.toLowerCase().includes("wordpress") ||
|
||||
isWordPress.includes("Site Kit by Google") ||
|
||||
$(".powered-by").text().toLowerCase().includes("wordpress");
|
||||
}
|
||||
|
||||
let isRainOfSnow = result.url.toLowerCase().includes("rainofsnow");
|
||||
|
||||
let isWebNovel = result.url.toLowerCase().includes("webnovel");
|
||||
|
||||
let isHostedNovel = result.url.toLowerCase().includes("hostednovel");
|
||||
|
||||
let isScribbleHub = result.url.toLowerCase().includes("scribblehub");
|
||||
|
||||
if (isWuxiaWorld) {
|
||||
chapterText = $("#chapter-content").html();
|
||||
} else if (isRainOfSnow) {
|
||||
chapterText = $("div.content").html();
|
||||
} else if (isTumblr) {
|
||||
chapterText = $(".post").html();
|
||||
} else if (isBlogspot) {
|
||||
$(".post-share-buttons").remove();
|
||||
|
||||
chapterText = $(".entry-content").html();
|
||||
} else if (isHostedNovel) {
|
||||
chapterText = $(".chapter").html();
|
||||
} else if (isScribbleHub) {
|
||||
chapterText = $("div.chp_raw").html();
|
||||
} else if (isWordPress) {
|
||||
/**
|
||||
* Remove wordpress bloat tags
|
||||
*/
|
||||
|
||||
const bloatClasses = [
|
||||
".c-ads",
|
||||
"#madara-comments",
|
||||
"#comments",
|
||||
".content-comments",
|
||||
".sharedaddy",
|
||||
".wp-dark-mode-switcher",
|
||||
".wp-next-post-navi",
|
||||
".wp-block-buttons",
|
||||
".wp-block-columns",
|
||||
".post-cats",
|
||||
".sidebar",
|
||||
".author-avatar",
|
||||
".ezoic-ad",
|
||||
];
|
||||
|
||||
bloatClasses.map((tag) => $(tag).remove());
|
||||
|
||||
chapterText =
|
||||
$(".entry-content").html() ||
|
||||
$(".single_post").html() ||
|
||||
$(".post-entry").html() ||
|
||||
$(".main-content").html() ||
|
||||
$("article.post").html() ||
|
||||
$(".content").html() ||
|
||||
$("#content").html();
|
||||
} else if (isWebNovel) {
|
||||
chapterText = $(".cha-words").html();
|
||||
|
||||
if (!chapterText) {
|
||||
chapterText = $("._content").html();
|
||||
}
|
||||
} else {
|
||||
/**
|
||||
* Remove unnecessary tags
|
||||
*/
|
||||
const tags = ["nav", "header", "footer", ".hidden"];
|
||||
|
||||
tags.map((tag) => $(tag).remove());
|
||||
|
||||
chapterText = $("body").html();
|
||||
}
|
||||
|
||||
if (!chapterText) {
|
||||
chapterText =
|
||||
"Chapter not available.\n\nReport if it's available in webview.";
|
||||
} else {
|
||||
/**
|
||||
* Convert relative urls to absolute
|
||||
*/
|
||||
chapterText = chapterText.replace(
|
||||
/href="\//g,
|
||||
`href="${getLocation(result.url)}/`
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
chapterText = `Chapter not available (Error: ${error.message}).\n\nReport if it's available in webview.`;
|
||||
}
|
||||
|
||||
const chapter = {
|
||||
sourceId,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const url =
|
||||
"https://www.novelupdates.com/?s=" +
|
||||
searchTerm +
|
||||
"&post_type=seriesplans";
|
||||
|
||||
const res = await fetch(url, {
|
||||
method: "GET",
|
||||
headers: headers,
|
||||
});
|
||||
const body = await res.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$("div.search_main_box_nu").each(function (res) {
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
const novelName = $(this).find(".search_title > a").text();
|
||||
const novelUrl = $(this)
|
||||
.find(".search_title > a")
|
||||
.attr("href")
|
||||
.split("/")[4];
|
||||
|
||||
const novel = {
|
||||
sourceId,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
return novels;
|
||||
};
|
||||
|
||||
const NovelUpdatesScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default NovelUpdatesScraper;
|
||||
@@ -1,142 +0,0 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
|
||||
const baseUrl = "https://www.novelupdates.cc/";
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
let totalPages = 1;
|
||||
const result = await fetch(baseUrl);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".section-item").each(function (result) {
|
||||
const novelName = $(this).find(".book-name").text();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
|
||||
let novelUrl = $(this).find("a").attr("href").slice(1);
|
||||
|
||||
const novel = {
|
||||
sourceId: 18,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return { totalPages, novels };
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = `${baseUrl}${novelUrl}/`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.sourceId = 18;
|
||||
|
||||
novel.sourceName = "NovelUpdates.cc";
|
||||
|
||||
novel.url = url;
|
||||
|
||||
novel.novelUrl = novelUrl;
|
||||
|
||||
novel.novelName = $(".book-name").text();
|
||||
|
||||
novel.novelCover = $("div.book-img > img").attr("src");
|
||||
|
||||
novel.genre = $("div.book-catalog > span.txt").text();
|
||||
|
||||
novel.status = $("div.book-state > span.txt").text();
|
||||
|
||||
novel.author = $("div.author > span.name").text();
|
||||
|
||||
novelSummary = $("div.content > p.desc").text();
|
||||
novel.summary = novelSummary.trim();
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
$(".chapter-item").each(function (result) {
|
||||
const chapterName = $(this).find(".chapter-name").text();
|
||||
const releaseDate = null;
|
||||
const chapterUrl = $(this).attr("href").slice(1);
|
||||
|
||||
novelChapters.push({
|
||||
chapterName,
|
||||
releaseDate,
|
||||
chapterUrl,
|
||||
});
|
||||
});
|
||||
|
||||
novel.chapters = novelChapters;
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = `${baseUrl}${chapterUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
const chapterName = $("h1.chapter-title").text();
|
||||
let chapterText = $("div.chapter-entity").html();
|
||||
const chapter = {
|
||||
sourceId: 18,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const url = `${baseUrl}search/${searchTerm}/1`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".list-item").each(function (result) {
|
||||
$(this).find("font").remove();
|
||||
|
||||
const novelName = $(this).find(".book-name").text();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
|
||||
let novelUrl = $(this).find("a").attr("href").slice(1);
|
||||
|
||||
const novel = {
|
||||
sourceId: 18,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const novelUpdatesCcScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default novelUpdatesCcScraper;
|
||||
@@ -1,140 +0,0 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
import { Status } from "../helpers/constants";
|
||||
|
||||
const sourceId = 78;
|
||||
|
||||
const sourceName = "RainOfSnow";
|
||||
|
||||
const baseUrl = "https://rainofsnow.com/";
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
let url = baseUrl + "novels/page/" + page;
|
||||
|
||||
const totalPages = 5;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".minbox").each(function () {
|
||||
const novelName = $(this).find("h3").text();
|
||||
const novelCover = $(this).find("img").attr("data-src");
|
||||
const novelUrl = $(this).find("h3 > a").attr("href");
|
||||
|
||||
const novel = {
|
||||
sourceId,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return { novels, totalPages };
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = novelUrl;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = { sourceId, sourceName, url, novelUrl };
|
||||
|
||||
novel.novelName = $("h2").text().trim();
|
||||
|
||||
novel.novelCover = $(".imagboca1 > img").attr("data-src");
|
||||
|
||||
novel.summary = $("#synop").text().trim();
|
||||
|
||||
novel.genre = $(
|
||||
"body > div.queen > div > div > div.row > div.col-md-12.col-lg-7 > div > div.backcolor1 > ul > li:nth-child(5) > small"
|
||||
)
|
||||
.text()
|
||||
.trim();
|
||||
|
||||
novel.author = $(
|
||||
"body > div.queen > div > div > div.row > div.col-md-12.col-lg-7 > div > div.backcolor1 > ul > li:nth-child(2) > small"
|
||||
).text();
|
||||
|
||||
novel.status = Status.UNKNOWN;
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
$("#chapter")
|
||||
.find("li")
|
||||
.each(function () {
|
||||
const chapterName = $(this).find(".chapter").first().text().trim();
|
||||
const releaseDate = $(this).find("small").text();
|
||||
const chapterUrl = $(this).find("a").attr("href");
|
||||
|
||||
novelChapters.push({ chapterName, releaseDate, chapterUrl });
|
||||
});
|
||||
|
||||
novel.chapters = novelChapters;
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const result = await fetch(chapterUrl);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let chapterName = $(".content > h2").text();
|
||||
let chapterText = $(".content").html();
|
||||
|
||||
const chapter = {
|
||||
sourceId,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
let url = baseUrl + "?s=" + searchTerm;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".minbox").each(function () {
|
||||
const novelName = $(this).find("h3").text();
|
||||
const novelCover = $(this).find("img").attr("data-src");
|
||||
const novelUrl = $(this).find("h3 > a").attr("href");
|
||||
|
||||
const novel = {
|
||||
sourceId,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const RainOfSnowScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default RainOfSnowScraper;
|
||||
@@ -1,228 +0,0 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
|
||||
import moment from "moment";
|
||||
import { showToast } from "../../hooks/showToast";
|
||||
|
||||
const sourceId = 51;
|
||||
|
||||
const sourceName = "Ranobes";
|
||||
|
||||
const baseUrl = "https://ranobes.net/";
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
const totalPages = 152;
|
||||
let url = `${baseUrl}novels/page/${page}`;
|
||||
|
||||
let headers = new Headers({
|
||||
referer: "https://ranobes.net/",
|
||||
"user-agent":
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36",
|
||||
});
|
||||
|
||||
const result = await fetch(url, { method: "GET", headers: headers });
|
||||
const body = await result.text();
|
||||
|
||||
let $ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$("article.block.story.shortstory.mod-poster").each(function () {
|
||||
const novelName = $(this).find("h2.title").text();
|
||||
let novelCover = $(this).find("figure").attr("style");
|
||||
|
||||
let novelUrl = $(this)
|
||||
.find("a")
|
||||
.attr("href")
|
||||
.replace(baseUrl + "novels/", "");
|
||||
|
||||
novelCover = novelCover
|
||||
.match(/background-image: url(.*);/)[1]
|
||||
.slice(1, -1);
|
||||
|
||||
const novel = {
|
||||
sourceId,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return { totalPages, novels };
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
showToast("Ranobes: Only has latest 25 chapters.");
|
||||
|
||||
const url = `${baseUrl}novels/${novelUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
let $ = cheerio.load(body);
|
||||
|
||||
let novel = {
|
||||
sourceId,
|
||||
sourceName,
|
||||
url,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novel.novelName = $('meta[property="og:title"]').attr("content");
|
||||
|
||||
novel.novelCover = baseUrl + $(".poster > a > img").attr("src");
|
||||
|
||||
novel.author = $("dl.author > dd").text();
|
||||
|
||||
novel.genre = $(".book-generes").text().trim();
|
||||
|
||||
novel.summary = $('div[itemprop="description"]').text().trim();
|
||||
|
||||
novel.author = $('span[itemprop="author"]').text();
|
||||
novel.genre = $('span[itemprop="genre"]').text().replace(", ", ",");
|
||||
|
||||
let tempNovelUrl = $(".r-fullstory-chapters-foot")
|
||||
.find("a")
|
||||
.next()
|
||||
.next()
|
||||
.attr("href")
|
||||
.split("/")[2];
|
||||
|
||||
const chapterListUrl = `${baseUrl}up/${tempNovelUrl}/page/1`;
|
||||
|
||||
const chaptersHtml = await fetch(chapterListUrl);
|
||||
const chapterHtmlToString = await chaptersHtml.text();
|
||||
|
||||
$ = cheerio.load(chapterHtmlToString);
|
||||
|
||||
// let lastPage = $("div.pages > a").last().text();
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
$("#dle-content > div.cat_block.cat_line").each(function () {
|
||||
const chapterName = $(this).find(".title").text();
|
||||
|
||||
let releaseDate = new Date();
|
||||
|
||||
let timeAgo = $(this).find("small").text();
|
||||
|
||||
timeAgo = timeAgo.match(/\d+/)[0];
|
||||
|
||||
if (timeAgo.includes("hours ago")) {
|
||||
releaseDate.setTime(releaseDate.getTime() - timeAgo);
|
||||
}
|
||||
|
||||
if (timeAgo.includes("months ago")) {
|
||||
releaseDate.setMonth(releaseDate.getMonth() - timeAgo);
|
||||
}
|
||||
|
||||
if (timeAgo.includes("days ago")) {
|
||||
releaseDate.setDate(releaseDate.getDate() - timeAgo);
|
||||
}
|
||||
|
||||
releaseDate = moment(releaseDate).format("MM/DD/YY");
|
||||
|
||||
const chapterUrl = $(this)
|
||||
.find("a")
|
||||
.attr("href")
|
||||
.replace(baseUrl + "up/", "");
|
||||
|
||||
const chapter = {
|
||||
chapterName,
|
||||
releaseDate,
|
||||
chapterUrl,
|
||||
};
|
||||
|
||||
novelChapters.push(chapter);
|
||||
});
|
||||
|
||||
// for (i = 1; i <= lastPage; i++) {
|
||||
// const chapterPageUrl = `${baseUrl}up/${tempNovelUrl}/page/${i}`;
|
||||
|
||||
// const chaptersHtml = await fetch(chapterPageUrl);
|
||||
// const chapterHtmlToString = await chaptersHtml.text();
|
||||
// $ = cheerio.load(chapterHtmlToString);
|
||||
|
||||
// $("#dle-content > div.cat_block.cat_line").each(function () {
|
||||
// const chapterName = $(this).find(".title").text();
|
||||
|
||||
// let releaseDate = new Date();
|
||||
|
||||
// let timeAgo = $(this).find("small").text();
|
||||
|
||||
// timeAgo = timeAgo.match(/\d+/)[0];
|
||||
|
||||
// if (timeAgo.includes("hours ago")) {
|
||||
// releaseDate.setTime(releaseDate.getTime() - timeAgo);
|
||||
// }
|
||||
|
||||
// if (timeAgo.includes("months ago")) {
|
||||
// releaseDate.setMonth(releaseDate.getMonth() - timeAgo);
|
||||
// }
|
||||
|
||||
// if (timeAgo.includes("days ago")) {
|
||||
// releaseDate.setDate(releaseDate.getDate() - timeAgo);
|
||||
// }
|
||||
|
||||
// releaseDate = moment(releaseDate).format("MM/DD/YY");
|
||||
|
||||
// const chapterUrl = $(this)
|
||||
// .find("a")
|
||||
// .attr("href")
|
||||
// .replace(baseUrl + "up/", "");
|
||||
|
||||
// const chapter = {
|
||||
// chapterName,
|
||||
// releaseDate,
|
||||
// chapterUrl,
|
||||
// };
|
||||
|
||||
// novelChapters.push(chapter);
|
||||
// });
|
||||
// }
|
||||
|
||||
novel.chapters = novelChapters.reverse();
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = `${baseUrl}up/${chapterUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let chapterName = $("h1.title").text();
|
||||
|
||||
let chapterText = $("#arrticle").html();
|
||||
const chapter = {
|
||||
sourceId,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
/**
|
||||
* TODO
|
||||
*/
|
||||
|
||||
showToast("Ranobes: Search does not work");
|
||||
return [];
|
||||
};
|
||||
|
||||
const RanobesScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default RanobesScraper;
|
||||
@@ -1,214 +0,0 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
|
||||
const sourceId = 2;
|
||||
|
||||
const sourceName = "ReadLightNovel";
|
||||
|
||||
const lang = "en";
|
||||
|
||||
const baseUrl = "https://www.readlightnovel.me";
|
||||
|
||||
const searchUrl = "https://www.readlightnovel.me/detailed-search";
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
let totalPages = 1751;
|
||||
const url = `${baseUrl}/top-novel/${page}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
const $ = cheerio.load(body);
|
||||
|
||||
const novels = [];
|
||||
|
||||
$(".top-novel-block").each(function (result) {
|
||||
const novelName = $(this).find("h2 > a").text();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
const novelUrl = $(this)
|
||||
.find("h2 > a")
|
||||
.attr("href")
|
||||
.replace(`${baseUrl}/`, "");
|
||||
|
||||
const novel = {
|
||||
sourceId,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return { totalPages, novels };
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = `${baseUrl}/${novelUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel;
|
||||
|
||||
const novelName = $(".block-title > h1").text();
|
||||
|
||||
const novelCover = $(".novel-cover > a > img").attr("src");
|
||||
|
||||
let author, artist, genre, summary, status;
|
||||
|
||||
$(".novel-detail-item").each(function (result) {
|
||||
const detailName = $(this).find(".novel-detail-header > h6").text();
|
||||
const detail = $(this).find(".novel-detail-body").text().trim();
|
||||
|
||||
switch (detailName) {
|
||||
case "Genre":
|
||||
genre = detail.trim().replace(/\s{2,}/g, ",");
|
||||
|
||||
break;
|
||||
case "Author(s)":
|
||||
author = detail;
|
||||
break;
|
||||
case "Artist(s)":
|
||||
artist = detail;
|
||||
break;
|
||||
case "Description":
|
||||
summary = detail;
|
||||
break;
|
||||
case "Status":
|
||||
status = detail;
|
||||
break;
|
||||
}
|
||||
});
|
||||
|
||||
let chapters = [];
|
||||
|
||||
$(".panel").each(function () {
|
||||
let volumeName = $(this).find("h4.panel-title").text();
|
||||
|
||||
$(this)
|
||||
.find("ul.chapter-chs > li")
|
||||
.each(function () {
|
||||
let chapterName = $(this).find("a").text();
|
||||
|
||||
const releaseDate = null;
|
||||
|
||||
const chapterUrl = $(this)
|
||||
.find("a")
|
||||
.attr("href")
|
||||
.replace(`${baseUrl}/${novelUrl}/`, "");
|
||||
|
||||
if (volumeName.includes("Volume")) {
|
||||
chapterName = volumeName + " " + chapterName;
|
||||
}
|
||||
|
||||
const chapter = {
|
||||
chapterName,
|
||||
releaseDate,
|
||||
chapterUrl,
|
||||
};
|
||||
|
||||
chapters.push(chapter);
|
||||
});
|
||||
});
|
||||
|
||||
novel = {
|
||||
sourceId,
|
||||
sourceName,
|
||||
url,
|
||||
novelUrl,
|
||||
novelName,
|
||||
novelCover,
|
||||
genre,
|
||||
author,
|
||||
status,
|
||||
artist,
|
||||
summary,
|
||||
chapters,
|
||||
};
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = `${baseUrl}/${novelUrl}/${chapterUrl}/`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
$(".block-title > h1").find("a").remove();
|
||||
|
||||
const chapterName = $(".block-title > h1").text().replace(" - ", "");
|
||||
|
||||
$(".alert").remove();
|
||||
$(".hidden").remove();
|
||||
$("iframe").remove();
|
||||
$("button").remove();
|
||||
$(".hid").remove();
|
||||
$("center").remove();
|
||||
$(
|
||||
'div[style="float: left; margin-top: 20px; font-style: italic;margin-left: 50px; font-size: 14px;"]'
|
||||
).remove();
|
||||
$('div[style="float:left;margin-top:15px;margin-bottom:15px;"]').remove();
|
||||
|
||||
let chapterText = $(".desc").html();
|
||||
|
||||
const chapter = {
|
||||
sourceId: 2,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const formData = new FormData();
|
||||
formData.append("keyword", searchTerm);
|
||||
formData.append("search", 1);
|
||||
|
||||
const result = await fetch(searchUrl, {
|
||||
method: "POST",
|
||||
body: formData,
|
||||
});
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".top-novel-block").each(function (result) {
|
||||
const novelName = $(this).find(".top-novel-header > h2 > a").text();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
|
||||
const novelUrl = $(this)
|
||||
.find(".top-novel-header > h2 > a")
|
||||
.attr("href")
|
||||
.replace(`${baseUrl}/`, "");
|
||||
|
||||
const novel = {
|
||||
sourceId,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const ReadLightNovelScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default ReadLightNovelScraper;
|
||||
@@ -1,142 +0,0 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
|
||||
const baseUrl = "https://m.readlightnovel.cc/";
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
let totalPages = 1;
|
||||
const result = await fetch(baseUrl);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".section-item").each(function (result) {
|
||||
const novelName = $(this).find(".book-name").text();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
|
||||
let novelUrl = $(this).find("a").attr("href").slice(1);
|
||||
|
||||
const novel = {
|
||||
sourceId: 19,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return { totalPages, novels };
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = `${baseUrl}${novelUrl}/`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.sourceId = 19;
|
||||
|
||||
novel.sourceName = "ReadLightNovel.cc";
|
||||
|
||||
novel.url = url;
|
||||
|
||||
novel.novelUrl = novelUrl;
|
||||
|
||||
novel.novelName = $(".book-name").text();
|
||||
|
||||
novel.novelCover = $("div.book-img > img").attr("src");
|
||||
|
||||
novel.genre = $("div.book-catalog > span.txt").text();
|
||||
|
||||
novel.status = $("div.book-state > span.txt").text();
|
||||
|
||||
novel.author = $("div.author > span.name").text();
|
||||
|
||||
novelSummary = $("div.content > p.desc").text();
|
||||
novel.summary = novelSummary.trim();
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
$(".chapter-item").each(function (result) {
|
||||
const chapterName = $(this).find(".chapter-name").text();
|
||||
const releaseDate = null;
|
||||
const chapterUrl = $(this).attr("href").slice(1);
|
||||
|
||||
novelChapters.push({
|
||||
chapterName,
|
||||
releaseDate,
|
||||
chapterUrl,
|
||||
});
|
||||
});
|
||||
|
||||
novel.chapters = novelChapters;
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = `${baseUrl}${chapterUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
const chapterName = $("h1.chapter-title").text();
|
||||
let chapterText = $("div.chapter-entity").html();
|
||||
const chapter = {
|
||||
sourceId: 19,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const url = `${baseUrl}search/${searchTerm}/1`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".list-item").each(function (result) {
|
||||
$(this).find("font").remove();
|
||||
|
||||
const novelName = $(this).find(".book-name").text();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
|
||||
let novelUrl = $(this).find("a").attr("href").slice(1);
|
||||
|
||||
const novel = {
|
||||
sourceId: 19,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const readLightNovelCcScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default readLightNovelCcScraper;
|
||||
@@ -1,186 +0,0 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
|
||||
const baseUrl = "https://readnovelfull.com";
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
let totalPages = 61;
|
||||
const url = `${baseUrl}/most-popular-novel?page=${page}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$("div.col-novel-main > div.list-novel > .row").each(function (result) {
|
||||
let novelUrl = $(this)
|
||||
.find("h3.novel-title > a")
|
||||
.attr("href")
|
||||
.replace(".html", "")
|
||||
.substring(1);
|
||||
novelUrl = `${novelUrl}/`;
|
||||
|
||||
const novelName = $(this).find("h3.novel-title > a").text();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
|
||||
const novel = {
|
||||
sourceId: 4,
|
||||
novelUrl,
|
||||
novelName,
|
||||
novelCover,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return { totalPages, novels };
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = `${baseUrl}/${novelUrl.slice(0, -1)}.html`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.sourceId = 4;
|
||||
|
||||
novel.sourceName = "ReadNovelFull";
|
||||
|
||||
novel.url = url;
|
||||
|
||||
novel.novelUrl = novelUrl;
|
||||
|
||||
novel.novelName = $("div.book > img").attr("alt");
|
||||
|
||||
novel.novelCover = $("div.book > img").attr("src");
|
||||
|
||||
novel.summary = $("div.desc-text").text().trim();
|
||||
|
||||
novel.author = $("li > h3")
|
||||
.filter(function () {
|
||||
return $(this).text().trim() === "Author:";
|
||||
})
|
||||
.siblings()
|
||||
.text();
|
||||
|
||||
novel.genre = $("li")
|
||||
.filter(function () {
|
||||
return $(this).find("h3").text().trim() === "Genre:";
|
||||
})
|
||||
.text()
|
||||
.replace("Genre:", "");
|
||||
|
||||
novel.artist = null;
|
||||
|
||||
novel.status = $("li > h3")
|
||||
.filter(function () {
|
||||
return $(this).text().trim() === "Status:";
|
||||
})
|
||||
.siblings()
|
||||
.text();
|
||||
|
||||
const novelId = $("#rating").attr("data-novel-id");
|
||||
|
||||
const getChapters = async (novelId) => {
|
||||
const chapterListUrl =
|
||||
"https://readnovelfull.com/ajax/chapter-archive?novelId=" + novelId;
|
||||
|
||||
const data = await fetch(chapterListUrl);
|
||||
const chapters = await data.text();
|
||||
|
||||
$ = cheerio.load(chapters);
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
$(".panel-body")
|
||||
.find("li")
|
||||
.each(function (result) {
|
||||
let chapterName = $(this).find("a").attr("title");
|
||||
let releaseDate = null;
|
||||
let chapterUrl = $(this).find("a").attr("href");
|
||||
chapterUrl = chapterUrl.replace(`/${novelUrl}`, "");
|
||||
|
||||
novelChapters.push({
|
||||
chapterName,
|
||||
releaseDate,
|
||||
chapterUrl,
|
||||
});
|
||||
});
|
||||
return novelChapters;
|
||||
};
|
||||
|
||||
novel.chapters = await getChapters(novelId);
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = `${baseUrl}/${novelUrl}${chapterUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
const chapterName = $(".chr-title").attr("title");
|
||||
let chapterText = $("#chr-content").html();
|
||||
const chapter = {
|
||||
sourceId: 4,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const searchUrl = `https://readnovelfull.com/search?keyword=`;
|
||||
|
||||
const url = `${searchUrl}${searchTerm}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$("div.col-novel-main > div.list-novel > .row").each(function (result) {
|
||||
let novelUrl = $(this)
|
||||
.find("h3.novel-title > a")
|
||||
.attr("href")
|
||||
.replace(".html", "")
|
||||
.substring(1);
|
||||
novelUrl = `${novelUrl}/`;
|
||||
|
||||
const novelName = $(this).find("h3.novel-title > a").text();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
|
||||
const novel = {
|
||||
sourceId: 4,
|
||||
novelUrl,
|
||||
novelName,
|
||||
novelCover,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const readNovelFullScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default readNovelFullScraper;
|
||||
@@ -1,193 +0,0 @@
|
||||
import QueryString from "qs";
|
||||
import cheerio from "react-native-cheerio";
|
||||
|
||||
const sourceId = 68;
|
||||
|
||||
const sourceName = "Readwn.com";
|
||||
|
||||
const baseUrl = "https://www.readwn.com/";
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
let url = `${baseUrl}list/all/all-onclick-${page - 1}.html`;
|
||||
const totalPages = 281;
|
||||
|
||||
const headers = new Headers({
|
||||
"user-agent":
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.131 Safari/537.36",
|
||||
});
|
||||
|
||||
const result = await fetch(url, { method: "GET", headers });
|
||||
const body = await result.text();
|
||||
|
||||
console.log(body);
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$("li.novel-item").each(function () {
|
||||
const novelName = $(this).find("h4").text();
|
||||
const novelUrl = $(this).find("a").attr("href").split("/").pop();
|
||||
|
||||
const novelCover =
|
||||
baseUrl + $(this).find(".novel-cover > img").attr("data-src");
|
||||
|
||||
const novel = {
|
||||
sourceId,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
return { novels, totalPages };
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = `${baseUrl}novel/${novelUrl}`;
|
||||
|
||||
const headers = new Headers({
|
||||
"user-agent":
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.131 Safari/537.36",
|
||||
});
|
||||
|
||||
const result = await fetch(url, { method: "GET", headers });
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {
|
||||
sourceId,
|
||||
sourceName,
|
||||
url,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novel.novelUrl = novelUrl;
|
||||
|
||||
novel.novelName = $("h1.novel-title").text();
|
||||
|
||||
novel.novelCover = baseUrl + $("figure.cover > img").attr("data-src");
|
||||
|
||||
novel.summary = $("p.description").text().trim();
|
||||
|
||||
novel.genre = "";
|
||||
|
||||
$("div.categories > ul > li").each(function () {
|
||||
novel.genre += $(this).text().trim() + ",";
|
||||
});
|
||||
|
||||
$("div.header-stats > span").each(function () {
|
||||
if ($(this).find("small").text() === "Status") {
|
||||
novel.status = $(this).find("strong").text();
|
||||
}
|
||||
});
|
||||
|
||||
novel.genre = novel.genre.slice(0, -1);
|
||||
|
||||
novel.author = $("span[itemprop=author]").text();
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
const novelId = novelUrl.split("/").pop().replace(".html", "");
|
||||
|
||||
const latestChapterNumber = $(".header-stats")
|
||||
.find("span > strong")
|
||||
.first()
|
||||
.text()
|
||||
.trim();
|
||||
|
||||
for (let i = 1; i <= latestChapterNumber; i++) {
|
||||
const chapterName = `Chapter ${i}`;
|
||||
const chapterUrl = `${novelId}_${i}`;
|
||||
const releaseDate = null;
|
||||
|
||||
const chapter = { chapterName, releaseDate, chapterUrl };
|
||||
|
||||
novelChapters.push(chapter);
|
||||
}
|
||||
|
||||
novel.chapters = novelChapters;
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = `${baseUrl}novel/${chapterUrl}.html`;
|
||||
|
||||
const headers = new Headers({
|
||||
"user-agent":
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.131 Safari/537.36",
|
||||
});
|
||||
|
||||
const result = await fetch(url, { method: "GET", headers });
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
const chapterName = $(".titles > h2").text();
|
||||
let chapterText = $(".chapter-content").html();
|
||||
|
||||
const chapter = {
|
||||
sourceId,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const searchUrl = "https://www.readwn.com/e/search/index.php";
|
||||
|
||||
const result = await fetch(searchUrl, {
|
||||
headers: {
|
||||
"Content-Type": "application/x-www-form-urlencoded",
|
||||
Referer: "https://www.readwn.com/search.html",
|
||||
Origin: "https://www.readwn.com",
|
||||
"user-agent":
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.131 Safari/537.36",
|
||||
},
|
||||
method: "POST",
|
||||
body: QueryString.stringify({
|
||||
show: "title",
|
||||
tempid: 1,
|
||||
tbname: "news",
|
||||
keyboard: searchTerm,
|
||||
}),
|
||||
});
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$("li.novel-item").each(function () {
|
||||
const novelName = $(this).find("h4").text();
|
||||
const novelCover = baseUrl + $(this).find("img").attr("src");
|
||||
const novelUrl = $(this).find("a").attr("href").split("/").pop();
|
||||
|
||||
const novel = {
|
||||
sourceId,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const ReadwnScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default ReadwnScraper;
|
||||
@@ -1,160 +0,0 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
import { Status } from "../helpers/constants";
|
||||
|
||||
const baseUrl = "https://reaperscans.com";
|
||||
|
||||
const sourceId = 67;
|
||||
const sourceName = "ReaperScans";
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
let url = `${baseUrl}/all-series/novels/`;
|
||||
let totalPages = 1;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".page-item-detail").each(function (result) {
|
||||
const novelName = $(this).find(".item-summary h3").text().trim();
|
||||
const novelCover = $(this).find(".img-responsive").attr("data-src");
|
||||
|
||||
let novelUrl = $(this).find("div > a").attr("href");
|
||||
novelUrl = novelUrl.replace(`${baseUrl}/`, "");
|
||||
|
||||
const novel = {
|
||||
sourceId,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return { totalPages, novels };
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = `${baseUrl}/${novelUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = { sourceId, url, sourceName };
|
||||
|
||||
$(".post-title > h3 > span").remove();
|
||||
|
||||
novel.novelUrl = novelUrl;
|
||||
|
||||
novel.novelName = $(".post-title > h1").text().trim();
|
||||
|
||||
novel.novelCover = $(".summary_image > a > img").attr("data-src");
|
||||
|
||||
$(".post-content_item").each(function () {
|
||||
const detailName = $(this).find(".summary-heading > h5").text().trim();
|
||||
const detail = $(this).find(".summary-content").text().trim();
|
||||
|
||||
switch (detailName) {
|
||||
case "Genre(s)":
|
||||
novel.genre = detail.replace(/[\t\n]/g, ",");
|
||||
break;
|
||||
case "Author(s)":
|
||||
novel.author = detail;
|
||||
break;
|
||||
case "Status":
|
||||
novel.status = detail.includes("OnGoing")
|
||||
? Status.ONGOING
|
||||
: Status.COMPLETED;
|
||||
break;
|
||||
}
|
||||
});
|
||||
|
||||
$(".description-summary > div.summary__content").find("em").remove();
|
||||
$(".premium-block").remove();
|
||||
|
||||
novel.summary = $("div.summary__content").text().trim();
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
$(".wp-manga-chapter").each(function (result) {
|
||||
$("i").remove();
|
||||
|
||||
const chapterName = $(this).find("a").text().trim();
|
||||
const releaseDate = null;
|
||||
|
||||
const chapterUrl = $(this).find("a").attr("href").replace(url, "");
|
||||
|
||||
const chapter = { chapterName, releaseDate, chapterUrl };
|
||||
|
||||
novelChapters.push(chapter);
|
||||
});
|
||||
|
||||
novel.chapters = novelChapters.reverse();
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = `${baseUrl}/${novelUrl}/${chapterUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
const chapterName = $("#chapter-heading").text();
|
||||
let chapterText = $(".reading-content").html();
|
||||
const chapter = {
|
||||
sourceId,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const url = `${baseUrl}?s=${searchTerm}&post_type=wp-manga`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".c-tabs-item__content").each(function (result) {
|
||||
const novelName = $(this).find(".post-title > h3").text().trim();
|
||||
const novelCover = $(this).find("div > div > a > img").attr("data-src");
|
||||
|
||||
let novelUrl = $(this).find("div > div > a").attr("href");
|
||||
novelUrl = novelUrl.replace(`${baseUrl}/`, "");
|
||||
|
||||
const novel = {
|
||||
sourceId,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const ReaperScansScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default ReaperScansScraper;
|
||||
@@ -1,184 +0,0 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
|
||||
import { parseMadaraDate } from "../helpers/parseDate";
|
||||
import { Status } from "../helpers/constants";
|
||||
|
||||
const baseUrl = "https://www.royalroad.com/";
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
const totalPages = 1846;
|
||||
let url = baseUrl + "fictions/best-rated?page=" + page;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$("div.fiction-list-item.row").each(function () {
|
||||
const novelName = $(this).find("h2.fiction-title").text().trim();
|
||||
let novelCover = $(this).find("img").attr("src");
|
||||
|
||||
if (novelCover === "/Content/Images/nocover-new-min.png") {
|
||||
novelCover =
|
||||
"https://github.com/LNReader/lnreader-sources/blob/main/src/coverNotAvailable.jpg?raw=true";
|
||||
}
|
||||
let novelUrl = $(this)
|
||||
.find("h2.fiction-title > a")
|
||||
.attr("href")
|
||||
.replace("/fiction/", "");
|
||||
|
||||
const novel = {
|
||||
sourceId: 34,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return { totalPages, novels };
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = baseUrl + "fiction/" + novelUrl;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.sourceId = 34;
|
||||
|
||||
novel.sourceName = "Royal Road";
|
||||
|
||||
novel.url = url;
|
||||
|
||||
novel.novelUrl = novelUrl;
|
||||
|
||||
novelName = $("h1").text();
|
||||
novel.novelName = novelName;
|
||||
|
||||
let novelCover = $("img.thumbnail").attr("src");
|
||||
|
||||
if (novelCover === "/Content/Images/nocover-new-min.png") {
|
||||
novelCover =
|
||||
"https://github.com/LNReader/lnreader-sources/blob/main/src/coverNotAvailable.jpg?raw=true";
|
||||
}
|
||||
|
||||
novel.novelCover = novelCover;
|
||||
|
||||
novel.summary = $("div.description").text().trim();
|
||||
|
||||
novel.author = $("h1").next().text().replace("by ", "").trim();
|
||||
novel.genre = $("span.tags").text().trim().replace(/\n\s+/g, ",");
|
||||
novel.status =
|
||||
$(
|
||||
"div.fiction-info > div.portlet > .col-md-8 > .margin-bottom-10 > span"
|
||||
)
|
||||
.first()
|
||||
.next()
|
||||
.text()
|
||||
.trim() === "ONGOING"
|
||||
? Status.ONGOING
|
||||
: Status.COMPLETED;
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
$("table#chapters > tbody")
|
||||
.find("tr")
|
||||
.each(function () {
|
||||
const chapterName = $(this).find("td").first().text().trim();
|
||||
let releaseDate = $(this).find("td").first().next().text().trim();
|
||||
releaseDate = releaseDate && parseMadaraDate(releaseDate);
|
||||
|
||||
const chapterUrl = $(this)
|
||||
.find("td")
|
||||
.first()
|
||||
.find("a")
|
||||
.attr("href")
|
||||
.replace("/fiction/" + novelUrl + "/chapter/", "");
|
||||
|
||||
novelChapters.push({
|
||||
chapterName,
|
||||
releaseDate,
|
||||
chapterUrl,
|
||||
});
|
||||
});
|
||||
|
||||
novel.chapters = novelChapters;
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = `${baseUrl}fiction/${novelUrl}/chapter/${chapterUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let chapterName = $("div.chapter-content").find("strong").text();
|
||||
|
||||
let chapterText = $("div.chapter-content").html();
|
||||
|
||||
const chapter = {
|
||||
sourceId: 34,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const url = baseUrl + "fictions/search?title=" + searchTerm;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$("div.fiction-list-item.row").each(function () {
|
||||
const novelName = $(this).find("h2.fiction-title").text().trim();
|
||||
let novelCover = $(this).find("img").attr("src");
|
||||
|
||||
if (novelCover.includes("Content")) {
|
||||
novelCover = baseUrl + novelCover;
|
||||
}
|
||||
|
||||
let novelUrl = $(this)
|
||||
.find("h2.fiction-title > a")
|
||||
.attr("href")
|
||||
.replace("/fiction/", "");
|
||||
|
||||
const novel = {
|
||||
sourceId: 34,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const RoyalRoadScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default RoyalRoadScraper;
|
||||
@@ -1,176 +0,0 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
|
||||
const baseUrl = "https://www.scribblehub.com/";
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
const totalPages = 326;
|
||||
let url = baseUrl + "series-ranking/?sort=1&order=4&pg=" + page;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$("div.search_main_box").each(function (result) {
|
||||
const novelName = $(this).find("div.search_title > a").text();
|
||||
const novelCover = $(this).find("div.search_img > img").attr("src");
|
||||
|
||||
let novelUrl = $(this).find("div.search_title > a").attr("href");
|
||||
novelUrl = novelUrl.split("/");
|
||||
novelUrl = novelUrl[4] + "-" + novelUrl[5];
|
||||
|
||||
const novel = {
|
||||
sourceId: 35,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return { totalPages, novels };
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = baseUrl + "read/" + novelUrl;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.sourceId = 35;
|
||||
|
||||
novel.sourceName = "Scribble Hub";
|
||||
|
||||
novel.url = url;
|
||||
|
||||
novel.novelUrl = novelUrl;
|
||||
|
||||
novel.novelName = $("div.fic_title").text();
|
||||
|
||||
novel.novelCover = $("div.fic_image > img").attr("src");
|
||||
|
||||
novel.summary = $("div.wi_fic_desc").text();
|
||||
|
||||
novel.genre = "";
|
||||
$("span.wi_fic_genre")
|
||||
.find("span")
|
||||
.each(function (res) {
|
||||
novel.genre += $(this).text() + ",";
|
||||
});
|
||||
if (novel.genre) {
|
||||
novel.genre = novel.genre.slice(0, -1);
|
||||
}
|
||||
|
||||
novel.status = $("span.rnd_stats").next().text().includes("Ongoing")
|
||||
? "Ongoing"
|
||||
: "Completed";
|
||||
|
||||
novel.author = $("span.auth_name_fic").text();
|
||||
|
||||
let formData = new FormData();
|
||||
formData.append("action", "wi_getreleases_pagination");
|
||||
formData.append("pagenum", "-1");
|
||||
formData.append("mypostid", novelUrl.split("-")[0]);
|
||||
|
||||
const data = await fetch(
|
||||
"https://www.scribblehub.com/wp-admin/admin-ajax.php",
|
||||
{
|
||||
method: "POST",
|
||||
body: formData,
|
||||
}
|
||||
);
|
||||
const text = await data.text();
|
||||
|
||||
$ = cheerio.load(text);
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
$(".toc_w").each(function (result) {
|
||||
const chapterName = $(this).find(".toc_a").text();
|
||||
const releaseDate = $(this).find(".fic_date_pub").text();
|
||||
|
||||
const chapterUrl = $(this).find("a").attr("href").split("/")[6];
|
||||
// .replace("/novel/" + novelUrl + "/", "");
|
||||
|
||||
novelChapters.push({
|
||||
chapterName,
|
||||
releaseDate,
|
||||
chapterUrl,
|
||||
});
|
||||
});
|
||||
|
||||
novel.chapters = novelChapters.reverse();
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = `${baseUrl}read/${novelUrl}/chapter/${chapterUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let chapterName = $("div.chapter-title").text();
|
||||
|
||||
let chapterText = $("div.chp_raw").html();
|
||||
const chapter = {
|
||||
sourceId: 35,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const url =
|
||||
"https://www.scribblehub.com/?s=" +
|
||||
searchTerm +
|
||||
"&post_type=fictionposts";
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$("div.search_main_box").each(function (result) {
|
||||
const novelName = $(this).find("div.search_title > a").text();
|
||||
const novelCover = $(this).find("div.search_img > img").attr("src");
|
||||
|
||||
let novelUrl = $(this).find("div.search_title > a").attr("href");
|
||||
novelUrl = novelUrl.split("/");
|
||||
novelUrl = novelUrl[4] + "-" + novelUrl[5];
|
||||
|
||||
const novel = {
|
||||
sourceId: 35,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
return novels;
|
||||
};
|
||||
|
||||
const ScribbleHubScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default ScribbleHubScraper;
|
||||
@@ -1,174 +0,0 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
|
||||
const baseUrl = "http://www.tapread.com";
|
||||
|
||||
let headers = new Headers({
|
||||
"User-Agent":
|
||||
"'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36",
|
||||
});
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
let totalPages = 0;
|
||||
|
||||
const result = await fetch(baseUrl);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$('.section-item[data-classify="1"]').each(function (result) {
|
||||
const novelName = $(this).find(".book-name").text();
|
||||
const novelCover = "http:" + $(this).find("img").attr("src");
|
||||
let novelUrl = $(this).attr("data-id");
|
||||
|
||||
const novel = {
|
||||
sourceId: 17,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return { totalPages, novels };
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = `${baseUrl}/book/detail/${novelUrl}`;
|
||||
|
||||
console.log(url);
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.sourceId = 17;
|
||||
|
||||
novel.sourceName = "TapRead";
|
||||
|
||||
novel.url = url;
|
||||
|
||||
novel.novelUrl = novelUrl;
|
||||
|
||||
novel.novelName = $(".book-name").text();
|
||||
|
||||
novel.novelCover = "http:" + $("div.book-img > img").attr("src");
|
||||
|
||||
novel.genre = $("div.book-catalog > span.txt").text();
|
||||
|
||||
novel.status = $("div.book-state > span.txt").text();
|
||||
|
||||
novel.author = $("div.author > span.name").text();
|
||||
|
||||
novel.summary = $("div.content > p.desc").text();
|
||||
|
||||
const getChapters = async (novelId) => {
|
||||
const chapterListUrl = "http://www.tapread.com/ajax/book/contents";
|
||||
|
||||
let formData = new FormData();
|
||||
formData.append("bookId", novelId);
|
||||
|
||||
const data = await fetch(chapterListUrl, {
|
||||
method: "POST",
|
||||
body: formData,
|
||||
});
|
||||
|
||||
const chapters = await data.json();
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
chapters.result.chapterList.map((chapter) => {
|
||||
let chapterName = chapter.chapterName;
|
||||
const releaseDate = chapter.pubTime;
|
||||
const chapterUrl = chapter.chapterId.toString();
|
||||
|
||||
if (chapter.lock) {
|
||||
chapterName = "🔒 " + chapterName;
|
||||
}
|
||||
|
||||
novelChapters.push({
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
releaseDate,
|
||||
});
|
||||
});
|
||||
|
||||
return novelChapters;
|
||||
};
|
||||
|
||||
novel.chapters = await getChapters(novelUrl);
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
let formData = new FormData();
|
||||
formData.append("bookId", novelUrl);
|
||||
formData.append("chapterId", chapterUrl);
|
||||
|
||||
const url = "http://www.tapread.com/ajax/book/chapter";
|
||||
|
||||
const result = await fetch(url, {
|
||||
method: "POST",
|
||||
body: formData,
|
||||
});
|
||||
const body = await result.json();
|
||||
|
||||
const chapterName = body.result.chapterName;
|
||||
const chapterText = body.result.content;
|
||||
|
||||
const chapter = {
|
||||
sourceId: 17,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
let formData = new FormData();
|
||||
formData.append("storyType", 1);
|
||||
formData.append("pageNo", 1);
|
||||
formData.append("searchText", searchTerm);
|
||||
|
||||
const url = "http://www.tapread.com/ajax/search/story";
|
||||
|
||||
const data = await fetch(url, {
|
||||
method: "POST",
|
||||
body: formData,
|
||||
});
|
||||
|
||||
const body = await data.json();
|
||||
|
||||
const novels = [];
|
||||
|
||||
body.result.storyList.map((novel) => {
|
||||
const novelName = novel.storyName.replace(
|
||||
/<font color="#FFCE2E">|<\/font>/g,
|
||||
""
|
||||
);
|
||||
const novelUrl = novel.storyId;
|
||||
const novelCover = "http://static.tapread.com" + novel.coverUrl;
|
||||
|
||||
novels.push({ sourceId: 17, novelName, novelCover, novelUrl });
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const tapReadScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default tapReadScraper;
|
||||
@@ -1,156 +0,0 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
|
||||
const sourceId = 76;
|
||||
|
||||
const sourceName = "TravisTranslations";
|
||||
|
||||
const baseUrl = "https://travistranslations.com/";
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
let url = baseUrl + "all-series/";
|
||||
|
||||
if (page > 1) {
|
||||
url += `page/${page}/`;
|
||||
}
|
||||
|
||||
const totalPages = 19;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".group").each(function () {
|
||||
const novelName = $(this).find("h3").text();
|
||||
const novelCover = "https://" + $(this).find("img").attr("data-src");
|
||||
const novelUrl = $(this).find(" a").attr("href");
|
||||
|
||||
const novel = {
|
||||
sourceId,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return { novels, totalPages };
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = novelUrl;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = { sourceId, sourceName, url, novelUrl };
|
||||
|
||||
novel.novelName = $("h1").text().trim();
|
||||
|
||||
novel.novelCover = "https://" + $("img.object-cover").attr("data-src");
|
||||
|
||||
novel.summary = $("div[property=description]").text().trim();
|
||||
|
||||
novel.genre = "";
|
||||
|
||||
$("div.flex.flex-col.mt-3.text-center > ul > li").each(function () {
|
||||
novel.genre +=
|
||||
$(this)
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "") + ",";
|
||||
});
|
||||
$("div.header-stats > span").each(function () {
|
||||
if ($(this).find("small").text() === "Status") {
|
||||
novel.status = $(this).find("strong").text();
|
||||
}
|
||||
});
|
||||
|
||||
novel.genre = novel.genre.slice(0, -1);
|
||||
|
||||
novel.author = $(".author > a > span").text();
|
||||
let novelChapters = [];
|
||||
|
||||
$(".tab_content > ul.grid")
|
||||
.find("li")
|
||||
.each(function () {
|
||||
const chapterName = $(this).find("span").first().text().trim();
|
||||
const releaseDate = null;
|
||||
const chapterUrl = $(this).find("a").attr("href");
|
||||
|
||||
novelChapters.push({ chapterName, releaseDate, chapterUrl });
|
||||
});
|
||||
|
||||
novel.chapters = novelChapters;
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const result = await fetch(chapterUrl);
|
||||
let body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
const chapterId = $("input[name=id]").attr("value");
|
||||
|
||||
const url =
|
||||
"https://travistranslations.com/wp-json/wp/v2/posts/" + chapterId;
|
||||
|
||||
const data = await fetch(url);
|
||||
body = await data.json();
|
||||
|
||||
let chapterName = body.title.rendered;
|
||||
let chapterText = body.content.rendered;
|
||||
|
||||
const chapter = {
|
||||
sourceId,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
let url = baseUrl + "all-series/?search=" + searchTerm;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".group").each(function () {
|
||||
const novelName = $(this).find("h3").text();
|
||||
const novelCover = "https://" + $(this).find("img").attr("data-src");
|
||||
const novelUrl = $(this).find(" a").attr("href");
|
||||
|
||||
const novel = {
|
||||
sourceId,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const TravisTranslationsScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default TravisTranslationsScraper;
|
||||
@@ -1,189 +0,0 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
|
||||
const baseUrl = "https://vipnovel.com/";
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
let totalPages = 89;
|
||||
let url = `${baseUrl}vipnovel/page/${page}/?m_orderby=rating`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".page-item-detail").each(function (result) {
|
||||
const novelName = $(this).find("h5 > a").text();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
|
||||
let novelUrl = $(this).find("h5 > a").attr("href");
|
||||
novelUrl = novelUrl.replace(`${baseUrl}vipnovel/`, "");
|
||||
|
||||
const novel = {
|
||||
sourceId: 10,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return { totalPages, novels };
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = `${baseUrl}vipnovel/${novelUrl}`;
|
||||
|
||||
console.log(url);
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.sourceId = 10;
|
||||
|
||||
novel.sourceName = "VipNovel";
|
||||
|
||||
novel.url = url;
|
||||
|
||||
novel.novelUrl = novelUrl;
|
||||
|
||||
novel.novelName = $(".post-title > h3")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
novel.novelCover = $(".summary_image > a > img").attr("src");
|
||||
$(".post-content_item").each(function (result) {
|
||||
detailName = $(this)
|
||||
.find(".summary-heading > h5")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
detail = $(this)
|
||||
.find(".summary-content")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
switch (detailName) {
|
||||
case "Genre(s)":
|
||||
novel.genre = detail.trim().replace(/[\t\n]/g, ",");
|
||||
break;
|
||||
case "Author(s)":
|
||||
novel.author = detail.trim();
|
||||
break;
|
||||
case "Artist(s)":
|
||||
novel.status = detail.trim();
|
||||
break;
|
||||
}
|
||||
});
|
||||
|
||||
$(".summary__content > p").first().remove();
|
||||
|
||||
let novelSummary = $(".description-summary > div.summary__content")
|
||||
.text()
|
||||
.replace("Synopsis", "");
|
||||
|
||||
novel.summary = novelSummary.replace(/[\t\n]/g, "");
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
$(".wp-manga-chapter").each(function (result) {
|
||||
const chapterName = $(this)
|
||||
.find("a")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
const releaseDate = $(this)
|
||||
.find("span")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
const chapterUrl = $(this).find("a").attr("href").replace(url, "");
|
||||
|
||||
const chapter = { chapterName, releaseDate, chapterUrl };
|
||||
|
||||
console.log(chapterUrl);
|
||||
|
||||
novelChapters.push(chapter);
|
||||
});
|
||||
|
||||
novel.chapters = novelChapters.reverse();
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = baseUrl + "vipnovel/" + novelUrl + chapterUrl;
|
||||
|
||||
console.log(url);
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let chapterName = $(".text-center").text();
|
||||
|
||||
if (!chapterName) {
|
||||
chapterName = $(".text-left > div> h3").text();
|
||||
}
|
||||
|
||||
let chapterText = $(".text-left").html();
|
||||
const chapter = {
|
||||
sourceId: 10,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const url = `${baseUrl}?s=${searchTerm}&post_type=wp-manga`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".c-tabs-item__content").each(function (result) {
|
||||
const novelName = $(this).find("h4 > a").text();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
|
||||
let novelUrl = $(this).find("h4 > a").attr("href");
|
||||
novelUrl = novelUrl.replace(`${baseUrl}vipnovel/`, "");
|
||||
|
||||
const novel = {
|
||||
sourceId: 10,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const vipNovelScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default vipNovelScraper;
|
||||
@@ -1,177 +0,0 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
import { Status } from "../helpers/constants";
|
||||
import NovelUpdatesScraper from "./novelupdates";
|
||||
|
||||
const sourceId = 62;
|
||||
|
||||
const sourceName = "WLNUpdates";
|
||||
|
||||
const baseUrl = "https://www.wlnupdates.com/";
|
||||
|
||||
let headers = new Headers({
|
||||
"User-Agent":
|
||||
"'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36",
|
||||
});
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
const totalPages = 100;
|
||||
let url = `${baseUrl}highest-rated/` + page;
|
||||
|
||||
const result = await fetch(url, {
|
||||
method: "GET",
|
||||
headers: headers,
|
||||
});
|
||||
const body = await result.text();
|
||||
|
||||
const $ = cheerio.load(body);
|
||||
|
||||
const novels = [];
|
||||
|
||||
$("table").find("tr").first().remove();
|
||||
|
||||
$("tr").each(function (res) {
|
||||
const novelCover =
|
||||
"https://github.com/LNReader/lnreader-sources/blob/main/src/coverNotAvailable.jpg?raw=true";
|
||||
|
||||
const novelName = $(this).find("td > a").text();
|
||||
const novelUrl =
|
||||
"https://www.wlnupdates.com" + $(this).find("td > a").attr("href");
|
||||
|
||||
const novel = {
|
||||
sourceId,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return { totalPages, novels };
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = novelUrl;
|
||||
|
||||
const result = await fetch(url, {
|
||||
method: "GET",
|
||||
headers: headers,
|
||||
});
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {
|
||||
sourceId,
|
||||
sourceName,
|
||||
url,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novel.novelName = $("h2").text();
|
||||
|
||||
novel.novelCover =
|
||||
"https://github.com/LNReader/lnreader-sources/blob/main/src/coverNotAvailable.jpg?raw=true";
|
||||
|
||||
novel.author = $(".multiitem#author")
|
||||
.text()
|
||||
.trim()
|
||||
.replace(/[\t\n]/g, "");
|
||||
|
||||
novel.genre = $(".multiitem#genre")
|
||||
.text()
|
||||
.trim()
|
||||
.replace(/[\t\n]/g, "");
|
||||
|
||||
novel.status = Status.UNKNOWN;
|
||||
|
||||
let summary = $("#description").text().trim();
|
||||
|
||||
novel.summary = summary;
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
$("#release-entry").each(function () {
|
||||
let chapterName;
|
||||
|
||||
if ($(this).find("td.numeric").length > 1) {
|
||||
chapterName =
|
||||
"Volume " +
|
||||
$(this).find(" td:nth-child(3)").text().trim() +
|
||||
" Chapter " +
|
||||
$(this).find(" td:nth-child(4)").text().trim();
|
||||
} else if ($(this).find("td.numeric").length > 0) {
|
||||
chapterName = "Chapter " + $(this).find("td.numeric").text().trim();
|
||||
} else {
|
||||
chapterName = $(this).find("td.postfix").text().trim();
|
||||
}
|
||||
|
||||
const releaseDate = $(this).find(".release-entry-cell").text().trim();
|
||||
|
||||
const chapterUrl = $(this).find("td:nth-child(1) > a").attr("href");
|
||||
|
||||
novelChapters.push({ chapterName, releaseDate, chapterUrl });
|
||||
});
|
||||
|
||||
novel.chapters = novelChapters.reverse();
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const chapterName = "";
|
||||
const chapterText = (
|
||||
await NovelUpdatesScraper.parseChapter(novelUrl, chapterUrl)
|
||||
).chapterText;
|
||||
|
||||
const chapter = {
|
||||
sourceId,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const url = baseUrl + "search?title=" + searchTerm;
|
||||
|
||||
const res = await fetch(url, {
|
||||
method: "GET",
|
||||
headers: headers,
|
||||
});
|
||||
const body = await res.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$("tr").each(function (res) {
|
||||
const novelCover =
|
||||
"https://github.com/LNReader/lnreader-sources/blob/main/src/coverNotAvailable.jpg?raw=true";
|
||||
const novelName = $(this).find("td > a").text();
|
||||
const novelUrl =
|
||||
"https://www.wlnupdates.com" + $(this).find("td > a").attr("href");
|
||||
|
||||
const novel = {
|
||||
sourceId,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
return novels;
|
||||
};
|
||||
|
||||
const WLNUpdatesScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default WLNUpdatesScraper;
|
||||
@@ -1,187 +0,0 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
|
||||
const baseUrl = "https://woopread.com/";
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
let totalPages = 6;
|
||||
let url = baseUrl + "novellist/page/" + page + "/?m_orderby=rating";
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".page-item-detail").each(function (result) {
|
||||
const novelName = $(this).find(".h5 > a").text();
|
||||
const novelCover = $(this).find("img").attr("data-src");
|
||||
|
||||
let novelUrl = $(this).find(".h5 > a").attr("href");
|
||||
novelUrl = novelUrl.replace(baseUrl + "series/", "");
|
||||
|
||||
const novel = {
|
||||
sourceId: 21,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return { totalPages, novels };
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = `${baseUrl}series/${novelUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.sourceId = 21;
|
||||
|
||||
novel.sourceName = "WoopRead";
|
||||
|
||||
novel.url = url;
|
||||
|
||||
novel.novelUrl = novelUrl;
|
||||
|
||||
novel.novelName = $(".post-title > h1")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
novel.novelCover = $(".summary_image > a > img").attr("src");
|
||||
|
||||
$(".post-content_item").each(function (result) {
|
||||
detailName = $(this)
|
||||
.find(".summary-heading > h5")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
detail = $(this)
|
||||
.find(".summary-content")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
switch (detailName) {
|
||||
case "Genre(s)":
|
||||
novel.genre = detail.trim().replace(/[\t\n]/g, ",");
|
||||
break;
|
||||
case "Author(s)":
|
||||
novel.author = detail.trim();
|
||||
break;
|
||||
case "Artist(s)":
|
||||
novel.status = detail.trim();
|
||||
break;
|
||||
}
|
||||
});
|
||||
$(".description-summary > div.summary__content").find("em").remove();
|
||||
|
||||
novel.summary = $(".description-summary > div.summary__content")
|
||||
.text()
|
||||
.trim();
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
const novelId = $(".rating-post-id").attr("value");
|
||||
|
||||
let formData = new FormData();
|
||||
formData.append("action", "manga_get_chapters");
|
||||
formData.append("manga", novelId);
|
||||
|
||||
const data = await fetch("https://woopread.com/wp-admin/admin-ajax.php", {
|
||||
method: "POST",
|
||||
body: formData,
|
||||
});
|
||||
const text = await data.text();
|
||||
|
||||
$ = cheerio.load(text);
|
||||
|
||||
$(".wp-manga-chapter.free-chap").each(function (result) {
|
||||
chapterName = $(this)
|
||||
.find("a")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
releaseDate = $(this).find("span").text().replace("Free", "").trim();
|
||||
|
||||
chapterUrl = $(this).find("a").attr("href").replace(url, "");
|
||||
|
||||
novelChapters.push({ chapterName, releaseDate, chapterUrl });
|
||||
});
|
||||
|
||||
novel.chapters = novelChapters.reverse();
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = `${baseUrl}series/${novelUrl}/${chapterUrl}/`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
const chapterName = $("h1#chapter-heading").text();
|
||||
|
||||
let chapterText = $(".reading-content").html();
|
||||
chapterText = chapterText.replace(/(?<=[[])[\n](?=[h])/g, "");
|
||||
|
||||
const chapter = {
|
||||
sourceId: 21,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const url = `${baseUrl}?s=${searchTerm}&post_type=wp-manga&op=&author=&artist=&release=&adult=`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".c-tabs-item__content").each(function (result) {
|
||||
const novelName = $(this).find(".h4 > a").text();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
|
||||
let novelUrl = $(this).find(".h4 > a").attr("href");
|
||||
novelUrl = novelUrl.replace(`${baseUrl}series/`, "");
|
||||
|
||||
const novel = {
|
||||
sourceId: 21,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const WoopReadScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default WoopReadScraper;
|
||||
@@ -1,204 +0,0 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
|
||||
const sourceId = 48;
|
||||
|
||||
const sourceName = "Wuxia.Blog";
|
||||
|
||||
const baseUrl = "https://www.wuxia.blog/";
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
const totalPages = 0;
|
||||
let url = `${baseUrl}listNovels`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$("div.panel-body > table")
|
||||
.find("tr")
|
||||
.each(function () {
|
||||
const novelName = $(this).find("td.novel").text();
|
||||
|
||||
if (novelName) {
|
||||
const novelId = $(this).find("td:nth-child(1)").text();
|
||||
|
||||
const novelCover = baseUrl + "/data/image/" + novelId + ".jpg";
|
||||
|
||||
let novelUrl = $(this)
|
||||
.find("td.novel > a")
|
||||
.attr("href")
|
||||
.replace("/novel/", "");
|
||||
|
||||
const novel = {
|
||||
sourceId,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
}
|
||||
});
|
||||
|
||||
return { totalPages, novels };
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = `${baseUrl}novel/${novelUrl}/`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {
|
||||
sourceId,
|
||||
sourceName,
|
||||
url,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novel.novelName = $(
|
||||
"body > div.container-fluid.text-center > div.row.content > div.col-sm-8.text-left > div:nth-child(6) > div.panel-heading.clearfix > h4"
|
||||
).text();
|
||||
|
||||
novel.novelCover = $('img[itemprop="image"]').attr("src");
|
||||
|
||||
novel.author = $(
|
||||
"div.col-md-6 > div > div:nth-child(2) > div > div:nth-child(2) > a"
|
||||
).text();
|
||||
|
||||
novel.genre = $("div.row > div.col-md-6 > div > a:nth-child(4)").text();
|
||||
|
||||
novel.summary = $('div[itemprop="description"]').find("p").text().trim();
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
$("table#chplist > #chapters > tr").each(function () {
|
||||
const chapterName = $(this).find("a").text();
|
||||
|
||||
const releaseDate = $(this)
|
||||
.find("td:nth-child(2)")
|
||||
.text()
|
||||
.replace(/-/g, "/");
|
||||
|
||||
const chapterUrl = $(this).find("a").attr("href").replace(url, "");
|
||||
|
||||
const chapter = {
|
||||
chapterName,
|
||||
releaseDate,
|
||||
chapterUrl,
|
||||
};
|
||||
|
||||
novelChapters.push(chapter);
|
||||
});
|
||||
|
||||
let moreChaptersUrl =
|
||||
baseUrl +
|
||||
"temphtml/_tempChapterList_all_" +
|
||||
$("div#more").attr("data-nid") +
|
||||
".html";
|
||||
|
||||
let moreChapters = await fetch(moreChaptersUrl);
|
||||
let moreChaptersString = await moreChapters.text();
|
||||
|
||||
$ = cheerio.load(moreChaptersString);
|
||||
|
||||
$("a").each(function () {
|
||||
const chapterName = $(this).text();
|
||||
|
||||
const releaseDate = $(this)[0]
|
||||
.nextSibling.nodeValue.replace(/-/g, "/")
|
||||
.trim();
|
||||
|
||||
const chapterUrl = $(this).attr("href").replace(url, "");
|
||||
|
||||
const chapter = {
|
||||
chapterName,
|
||||
releaseDate,
|
||||
chapterUrl,
|
||||
};
|
||||
|
||||
novelChapters.push(chapter);
|
||||
});
|
||||
|
||||
novel.chapters = novelChapters.reverse();
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = `${baseUrl}novel/${novelUrl}/${chapterUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let chapterName = $("div.panel-body.article").find("h4").text();
|
||||
|
||||
$("ul.pager").remove();
|
||||
|
||||
let chapterText = $("div.panel-body.article")
|
||||
.html()
|
||||
.replace(/<span(.*?)>(.*?)<\/span>/g, "");
|
||||
|
||||
const chapter = {
|
||||
sourceId,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const url = `${baseUrl}?search=${searchTerm}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$("#table")
|
||||
.find("tr")
|
||||
.each(function () {
|
||||
const novelName = $(this).find("a").text().trim();
|
||||
|
||||
if (novelName) {
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
|
||||
let novelUrl = $(this)
|
||||
.find("a")
|
||||
.attr("href")
|
||||
.replace(baseUrl + "novel/", "");
|
||||
|
||||
const novel = {
|
||||
sourceId,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
}
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const WuxiaBlogScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default WuxiaBlogScraper;
|
||||
@@ -1,173 +0,0 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
|
||||
import moment from "moment";
|
||||
|
||||
const sourceId = 49;
|
||||
|
||||
const sourceName = "wuxia.city";
|
||||
|
||||
const baseUrl = "https://wuxia.city/";
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
const totalPages = 133;
|
||||
let url = `${baseUrl}genre/all?page=` + page;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".section-item.card").each(function () {
|
||||
const novelName = $(this).find(".book-name").text();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
const novelUrl = $(this).find("a").attr("href").replace("/book/", "");
|
||||
|
||||
const novel = {
|
||||
sourceId,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return { totalPages, novels };
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = `${baseUrl}book/${novelUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
let $ = cheerio.load(body);
|
||||
|
||||
let novel = {
|
||||
sourceId,
|
||||
sourceName,
|
||||
url,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novel.novelName = $("h1.book-name").text();
|
||||
|
||||
novel.novelCover = $("div.book-img > img").attr("src");
|
||||
|
||||
novel.author = $("dl.author > dd").text();
|
||||
|
||||
novel.genre = $(".book-generes").text().trim();
|
||||
|
||||
novel.summary = $(".book-synopsis").text().replace("Synopsis", "").trim();
|
||||
|
||||
const chapterListUrl = `${baseUrl}book/${novelUrl}/chapter-list`;
|
||||
|
||||
const chaptersHtml = await fetch(chapterListUrl);
|
||||
const chapterHtmlToString = await chaptersHtml.text();
|
||||
|
||||
$ = cheerio.load(chapterHtmlToString);
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
$("ul.chapters > li").each(function () {
|
||||
const chapterName = $(this)
|
||||
.find(".chapter-name")
|
||||
.text()
|
||||
.trim()
|
||||
.replace(/\n/, " ");
|
||||
|
||||
let releaseDate = new Date();
|
||||
|
||||
let timeAgo = $(this).find(".chapter-time").text();
|
||||
|
||||
timeAgo = timeAgo.replace("one", 1);
|
||||
|
||||
timeAgo = timeAgo.match(/\d+/)[0];
|
||||
|
||||
if (timeAgo.includes("months ago") || timeAgo.includes("month ago")) {
|
||||
releaseDate.setMonth(releaseDate.getMonth() - timeAgo);
|
||||
}
|
||||
|
||||
if (timeAgo.includes("days ago")) {
|
||||
releaseDate.setDate(releaseDate.getDate() - timeAgo);
|
||||
}
|
||||
|
||||
releaseDate = moment(releaseDate).format("MM/DD/YY");
|
||||
|
||||
const chapterUrl = $(this).find("a").attr("href").split("/").pop();
|
||||
|
||||
const chapter = {
|
||||
chapterName,
|
||||
releaseDate,
|
||||
chapterUrl,
|
||||
};
|
||||
|
||||
novelChapters.push(chapter);
|
||||
});
|
||||
|
||||
novel.chapters = novelChapters.reverse();
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = `${baseUrl}book/${novelUrl}/${chapterUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let chapterName = $(".chapter-name").text();
|
||||
|
||||
let chapterText = $(".chapter-content").html();
|
||||
const chapter = {
|
||||
sourceId,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const url = `${baseUrl}search?q=${searchTerm}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".section-item").each(function () {
|
||||
const novelName = $(this).find(".book-name").text().trim();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
|
||||
let novelUrl = $(this).find("a").attr("href").replace("/book/", "");
|
||||
|
||||
const novel = {
|
||||
sourceId,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const WuxiaCityScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default WuxiaCityScraper;
|
||||
@@ -1,171 +0,0 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
|
||||
const baseUrl = "https://www.wuxiaworld.com/";
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
let totalPages = 1;
|
||||
const url = `${baseUrl}api/novels`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const data = await result.json();
|
||||
|
||||
let novels = [];
|
||||
|
||||
data.items.map((novel) => {
|
||||
let novelName = novel.name;
|
||||
let novelCover = novel.coverUrl;
|
||||
let novelUrl = novel.slug;
|
||||
|
||||
novels.push({
|
||||
sourceId: 7,
|
||||
novelUrl,
|
||||
novelName,
|
||||
novelCover,
|
||||
});
|
||||
});
|
||||
|
||||
return { totalPages, novels };
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = `${baseUrl}novel/${novelUrl}/`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.sourceId = 7;
|
||||
|
||||
novel.sourceName = "WuxiaWorld";
|
||||
|
||||
novel.url = url;
|
||||
|
||||
novel.novelUrl = novelUrl;
|
||||
|
||||
novel.novelName = $("h2").text();
|
||||
|
||||
novel.novelCover = $("img.img-thumbnail").attr("src");
|
||||
|
||||
novel.summary = $("h3")
|
||||
.filter(function () {
|
||||
return $(this).text().trim() === "Synopsis";
|
||||
})
|
||||
.next()
|
||||
.text()
|
||||
.trim();
|
||||
|
||||
novel.author = $("div > dt")
|
||||
.filter(function () {
|
||||
return $(this).text().trim() === "Author:";
|
||||
})
|
||||
.next()
|
||||
.text();
|
||||
|
||||
let genres = [];
|
||||
|
||||
$(".genres")
|
||||
.find("div")
|
||||
.each(function (res) {
|
||||
genres.push($(this).find("a").text());
|
||||
});
|
||||
|
||||
novel.genre = genres.join(",");
|
||||
|
||||
novel.artist = null;
|
||||
|
||||
novel.status = null;
|
||||
|
||||
novel.status = $("div.fr-view.pt-10").text().includes("Complete")
|
||||
? "Completed"
|
||||
: "Ongoing";
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
$(".chapter-item").each(function (result) {
|
||||
let chapterName = $(this).text();
|
||||
chapterName = chapterName.replace(/[\t\n]/g, "");
|
||||
|
||||
const releaseDate = null;
|
||||
|
||||
let chapterUrl = $(this).find("a").attr("href");
|
||||
chapterUrl = chapterUrl.replace(`/novel/${novelUrl}/`, "");
|
||||
|
||||
const chapter = {
|
||||
chapterName,
|
||||
releaseDate,
|
||||
chapterUrl,
|
||||
};
|
||||
|
||||
novelChapters.push(chapter);
|
||||
});
|
||||
|
||||
novel.chapters = novelChapters;
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
let url = `${baseUrl}novel/${novelUrl}/${chapterUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
$(".chapter-nav").remove();
|
||||
|
||||
let chapterName = $("#sidebar-toggler-container").next().text();
|
||||
chapterName = chapterName.replace(/[\t\n]/g, "");
|
||||
|
||||
$("#chapter-content > script").remove();
|
||||
|
||||
let chapterText = $("#chapter-content").html();
|
||||
|
||||
const chapter = {
|
||||
sourceId: 7,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const searchUrl = `https://www.wuxiaworld.com/api/novels/search?query=`;
|
||||
|
||||
const url = `${searchUrl}${searchTerm}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const data = await result.json();
|
||||
|
||||
let novels = [];
|
||||
|
||||
data.items.map((novel) => {
|
||||
let novelName = novel.name;
|
||||
let novelCover = novel.coverUrl;
|
||||
let novelUrl = novel.slug;
|
||||
|
||||
novels.push({
|
||||
sourceId: 7,
|
||||
novelUrl,
|
||||
novelName,
|
||||
novelCover,
|
||||
});
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const WuxiaWorldScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default WuxiaWorldScraper;
|
||||
@@ -1,186 +0,0 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
|
||||
const baseUrl = "http://wuxiaworld.cloud/";
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
let totalPages = 41;
|
||||
const url = `${baseUrl}/popular-novel?page=${page}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$("div.col-novel-main > div.list-novel > .row").each(function (result) {
|
||||
let novelUrl = $(this)
|
||||
.find("h3.novel-title > a")
|
||||
.attr("href")
|
||||
.replace(baseUrl + "novel/", "");
|
||||
novelUrl = `${novelUrl}/`;
|
||||
|
||||
const novelName = $(this).find("h3.novel-title > a").text();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
|
||||
const novel = {
|
||||
sourceId: 20,
|
||||
novelUrl,
|
||||
novelName,
|
||||
novelCover,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return { totalPages, novels };
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = `${baseUrl}novel/${novelUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.sourceId = 20;
|
||||
|
||||
novel.sourceName = "WuxiaWorld.cloud";
|
||||
|
||||
novel.url = url;
|
||||
|
||||
novel.novelUrl = novelUrl;
|
||||
|
||||
novel.novelName = $("div.book > img").attr("alt");
|
||||
|
||||
novel.novelCover = $("div.book > img").attr("src");
|
||||
|
||||
novel.summary = $("div.desc-text").text().trim();
|
||||
|
||||
novel.author = $("div.info > div > h3")
|
||||
.filter(function () {
|
||||
return $(this).text().trim() === "Author:";
|
||||
})
|
||||
.siblings()
|
||||
.text();
|
||||
|
||||
novel.genre = $("div.info > div")
|
||||
.filter(function () {
|
||||
return $(this).find("h3").text().trim() === "Genre:";
|
||||
})
|
||||
.text()
|
||||
.replace("Genre:", "")
|
||||
.replace(/\s/g, "");
|
||||
|
||||
novel.artist = null;
|
||||
|
||||
novel.status = $("div.info > div")
|
||||
.filter(function () {
|
||||
return $(this).find("h3").text().trim() === "Status:";
|
||||
})
|
||||
.text()
|
||||
.replace("Status:", "")
|
||||
.replace(/\s/g, "");
|
||||
|
||||
const novelId = $("#rating").attr("data-novel-id");
|
||||
|
||||
const getChapters = async (novelId) => {
|
||||
const chapterListUrl =
|
||||
baseUrl + "ajax/chapter-option?novelId=" + novelId;
|
||||
|
||||
const data = await fetch(chapterListUrl);
|
||||
const chapters = await data.text();
|
||||
|
||||
$ = cheerio.load(chapters);
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
$("select > option").each(function (result) {
|
||||
let chapterName = $(this).text();
|
||||
let releaseDate = null;
|
||||
let chapterUrl = $(this).attr("value");
|
||||
chapterUrl = chapterUrl.replace(baseUrl, "");
|
||||
|
||||
novelChapters.push({
|
||||
chapterName,
|
||||
releaseDate,
|
||||
chapterUrl,
|
||||
});
|
||||
});
|
||||
return novelChapters;
|
||||
};
|
||||
|
||||
novel.chapters = await getChapters(novelId);
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
chapterUrl = encodeURI(chapterUrl);
|
||||
|
||||
const url = `${baseUrl}${chapterUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
const chapterName = $(".chr-title").attr("title");
|
||||
let chapterText = $("#chr-content").html();
|
||||
const chapter = {
|
||||
sourceId: 20,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const searchUrl = `http://wuxiaworld.cloud/search?keyword=`;
|
||||
|
||||
const url = `${searchUrl}${searchTerm}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$("div.col-novel-main > div.list-novel > .row").each(function (result) {
|
||||
let novelUrl = $(this)
|
||||
.find("h3.novel-title > a")
|
||||
.attr("href")
|
||||
.replace(baseUrl + "novel/", "");
|
||||
novelUrl = `${novelUrl}/`;
|
||||
|
||||
const novelName = $(this).find("h3.novel-title > a").text();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
|
||||
const novel = {
|
||||
sourceId: 20,
|
||||
novelUrl,
|
||||
novelName,
|
||||
novelCover,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const WuxiaWorldCloudScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default WuxiaWorldCloudScraper;
|
||||
@@ -1,142 +0,0 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
|
||||
const baseUrl = "https://www.wuxiaworld.co/";
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
let totalPages = 1;
|
||||
const result = await fetch(baseUrl);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".section-item").each(function (result) {
|
||||
const novelName = $(this).find(".book-name").text();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
|
||||
let novelUrl = $(this).find("a").attr("href").slice(1);
|
||||
|
||||
const novel = {
|
||||
sourceId: 16,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return { totalPages, novels };
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = `${baseUrl}${novelUrl}/`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.sourceId = 16;
|
||||
|
||||
novel.sourceName = "WuxiaWorldCo";
|
||||
|
||||
novel.url = url;
|
||||
|
||||
novel.novelUrl = novelUrl;
|
||||
|
||||
novel.novelName = $(".book-name").text();
|
||||
|
||||
novel.novelCover = $("div.book-img > img").attr("src");
|
||||
|
||||
novel.genre = $("div.book-catalog > span.txt").text();
|
||||
|
||||
novel.status = $("div.book-state > span.txt").text();
|
||||
|
||||
novel.author = $("div.author > span.name").text();
|
||||
|
||||
novelSummary = $("div.content > p.desc").html();
|
||||
novel.summary = novelSummary.trim();
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
$(".chapter-item").each(function (result) {
|
||||
const chapterName = $(this).find(".chapter-name").text();
|
||||
const releaseDate = null;
|
||||
const chapterUrl = $(this).attr("href").split("/")[2];
|
||||
|
||||
novelChapters.push({
|
||||
chapterName,
|
||||
releaseDate,
|
||||
chapterUrl,
|
||||
});
|
||||
});
|
||||
|
||||
novel.chapters = novelChapters;
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = `${baseUrl}${novelUrl}/${chapterUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
const chapterName = $("h1.chapter-title").text();
|
||||
let chapterText = $("div.chapter-entity").html();
|
||||
const chapter = {
|
||||
sourceId: 16,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const url = `${baseUrl}search/${searchTerm}/1`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".list-item").each(function (result) {
|
||||
$(this).find("font").remove();
|
||||
|
||||
const novelName = $(this).find(".book-name").text();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
|
||||
let novelUrl = $(this).find("a").attr("href").slice(1);
|
||||
|
||||
const novel = {
|
||||
sourceId: 16,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const WuxiaWorldCoScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default WuxiaWorldCoScraper;
|
||||
@@ -1,186 +0,0 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
|
||||
const baseUrl = "https://wuxiaworldsite.co/";
|
||||
const searchUrl = "https://wuxiaworldsite.co/search/";
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
let totalPages = 222;
|
||||
const url = `https://wuxiaworldsite.co/ajax-story-power.ajax`;
|
||||
|
||||
const formData = new FormData();
|
||||
formData.append("page", page);
|
||||
formData.append("keyword", "");
|
||||
formData.append("count", 18);
|
||||
formData.append("genres_include", "");
|
||||
formData.append("limit", 18);
|
||||
formData.append("order_type", "DESC");
|
||||
formData.append("order_by", "views");
|
||||
|
||||
const result = await fetch(url, { method: "POST", body: formData });
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".a_item").each((i, el) => {
|
||||
const novelName = $(el).find("a").attr("title");
|
||||
const novelCover = baseUrl + $(el).find("a > img").attr("src");
|
||||
|
||||
let novelUrl = $(el).find(".name_views > a").attr("href");
|
||||
novelId = novelUrl.split("/");
|
||||
novelUrl = novelId[2] + "/";
|
||||
|
||||
const novel = {
|
||||
sourceId: 12,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return { totalPages, novels };
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = `${baseUrl}${novelUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
$(".category_list").remove();
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.sourceId = 12;
|
||||
|
||||
novel.sourceName = "WuxiaWorldSite";
|
||||
|
||||
novel.url = url;
|
||||
|
||||
novel.novelUrl = novelUrl;
|
||||
|
||||
novel.novelName = $(".content-reading > h1").text().trim();
|
||||
|
||||
novel.novelCover = baseUrl + $(".img-read> img").attr("src");
|
||||
|
||||
novel.summary = $(".story-introduction-content").text();
|
||||
|
||||
novel.author = $(".content-reading > p").text().trim();
|
||||
|
||||
novel.artist = null;
|
||||
|
||||
novel.genre = "";
|
||||
|
||||
$(".a_tag_item").each((i, el) => {
|
||||
novel.genre += $(el).text() + ",";
|
||||
});
|
||||
|
||||
novel.genre = novel.genre.split(",");
|
||||
novel.genre.pop();
|
||||
|
||||
novel.status = novel.genre.pop();
|
||||
|
||||
novel.genre = novel.genre.join(",");
|
||||
|
||||
const novelID = $(".show-more-list").attr("data-id");
|
||||
|
||||
const getChapters = async (novelID) => {
|
||||
const chapterListUrl = baseUrl + "/get-full-list.ajax?id=" + novelID;
|
||||
|
||||
const data = await fetch(chapterListUrl);
|
||||
const chapters = await data.text();
|
||||
|
||||
$ = cheerio.load(chapters);
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
$(".new-update-content").each((i, el) => {
|
||||
let chapterName = $(el).text().split(/\t+/);
|
||||
const releaseDate = chapterName.pop();
|
||||
chapterName = chapterName[0];
|
||||
let chapterUrl = $(el).attr("href");
|
||||
chapterUrl = chapterUrl.split("/").pop();
|
||||
|
||||
const novel = {
|
||||
chapterName,
|
||||
releaseDate,
|
||||
chapterUrl,
|
||||
};
|
||||
novelChapters.push(novel);
|
||||
});
|
||||
return novelChapters;
|
||||
};
|
||||
|
||||
novel.chapters = await getChapters(novelID);
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = `${baseUrl}/${novelUrl}/${chapterUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let chapterName = $("h1.text-center.heading_read").text();
|
||||
let chapterText = $(".content-story").html();
|
||||
|
||||
novelUrl += "/";
|
||||
chapterUrl += "/";
|
||||
|
||||
const chapter = {
|
||||
sourceId: 12,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const url = `${searchUrl}${searchTerm}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".a_item").each((i, el) => {
|
||||
const novelName = $(el).find(".name_views > a").attr("title");
|
||||
const novelCover = baseUrl + $(el).find("a > img").attr("src");
|
||||
let novelUrl = $(el).find(".name_views > a").attr("href");
|
||||
novelId = novelUrl.split("/");
|
||||
novelUrl = novelId[2] + "/";
|
||||
|
||||
const novel = {
|
||||
sourceId: 12,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const WuxiaWorldSiteScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default WuxiaWorldSiteScraper;
|
||||
@@ -1,194 +0,0 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
|
||||
const baseUrl = "https://einherjarproject.net/";
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
let totalPages = 1;
|
||||
let url = baseUrl + "proyectos-activos/";
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".wp-block-media-text").each(function (result) {
|
||||
const novelName = $(this).find("a").text();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
|
||||
let novelUrl = $(this)
|
||||
.find(".wp-block-media-text__content")
|
||||
.find("a")
|
||||
.attr("href");
|
||||
novelUrl = novelUrl.replace(baseUrl, "");
|
||||
|
||||
const novel = {
|
||||
sourceId: 25,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return { totalPages, novels };
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = baseUrl + novelUrl;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.sourceId = 25;
|
||||
|
||||
novel.sourceName = "Einherjar Project";
|
||||
|
||||
novel.url = url;
|
||||
|
||||
novel.novelUrl = novelUrl;
|
||||
|
||||
novel.novelName = $("h1").text();
|
||||
|
||||
novel.novelCover = $("img").attr("src");
|
||||
|
||||
$(".wp-block-columns")
|
||||
.find("li")
|
||||
.each(function (result) {
|
||||
if ($(this).text().includes("Autor:")) {
|
||||
novel.author = $(this)
|
||||
.text()
|
||||
.replace("Autor:", "")
|
||||
.slice(0, -1);
|
||||
}
|
||||
if ($(this).text().includes("Ilustrador: ")) {
|
||||
novel.artist = $(this)
|
||||
.text()
|
||||
.replace("Ilustrador: ", "")
|
||||
.slice(0, -1);
|
||||
}
|
||||
if ($(this).text().includes("Estado: ")) {
|
||||
novel.status = $(this)
|
||||
.text()
|
||||
.replace("Estado: ", "")
|
||||
.slice(0, -1);
|
||||
}
|
||||
});
|
||||
|
||||
novel.genre = $(".post-content > h6")
|
||||
.text()
|
||||
.replace(/GÉNEROS: /, "")
|
||||
.replace(/,\s/g, ",");
|
||||
|
||||
let novelSummary = $(".post-content > .has-text-align-center").html();
|
||||
|
||||
novel.summary = novelSummary.trim();
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
$(".wp-block-media-text")
|
||||
.find("p")
|
||||
.each(function (result) {
|
||||
if ($(this).find("a").text()) {
|
||||
const chapterName = $(this).text();
|
||||
const releaseDate = null;
|
||||
let chapterUrl = $(this)
|
||||
.find("a")
|
||||
.attr("href")
|
||||
.replace(baseUrl, "");
|
||||
if (chapterUrl.includes(novelUrl + "/"))
|
||||
chapterUrl = chapterUrl.replace(novelUrl + "/", "");
|
||||
|
||||
const chapter = { chapterName, releaseDate, chapterUrl };
|
||||
|
||||
novelChapters.push(chapter);
|
||||
}
|
||||
});
|
||||
|
||||
novel.chapters = novelChapters;
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = baseUrl + chapterUrl;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let chapterName = $("h1.post-title").text();
|
||||
|
||||
let chapterText = $(".post-content").html();
|
||||
novelUrl = novelUrl + "/";
|
||||
chapterUrl = chapterUrl + "/";
|
||||
|
||||
const chapter = {
|
||||
sourceId: 25,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const url = baseUrl + "?s=" + searchTerm;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".post-container")
|
||||
.find("div.type-page")
|
||||
.each(function (result) {
|
||||
const novelName = $(this)
|
||||
.find(".post-header")
|
||||
.text()
|
||||
.replace(/\n/g, "");
|
||||
if (
|
||||
!novelName.includes("EPUBs") &&
|
||||
!novelName.includes("Proyectos Activos") &&
|
||||
!novelName.includes("Chapter")
|
||||
) {
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
|
||||
let novelUrl = $(this)
|
||||
.find("a")
|
||||
.attr("href")
|
||||
.replace(baseUrl, "");
|
||||
|
||||
const novel = {
|
||||
sourceId: 25,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
}
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const EinharjarProjectScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default EinharjarProjectScraper;
|
||||
@@ -1,170 +0,0 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
|
||||
const baseUrl = "https://hasutl.wordpress.com/";
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
let totalPages = 1;
|
||||
let url = baseUrl + "light-novels-activas/";
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$("div.wp-block-columns").each(function (result) {
|
||||
const novelName = $(this).find(".wp-block-button").text();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
|
||||
let novelUrl = $(this).find(".wp-block-button > a").attr("href");
|
||||
novelUrl = novelUrl.replace(baseUrl, "");
|
||||
|
||||
const novel = {
|
||||
sourceId: 29,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return { totalPages, novels };
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = `${baseUrl}/${novelUrl}/`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.sourceId = 29;
|
||||
|
||||
novel.sourceName = "Hasu Translations";
|
||||
|
||||
novel.url = url;
|
||||
|
||||
novel.novelUrl = novelUrl;
|
||||
|
||||
novel.novelName = $(".post-header").text();
|
||||
|
||||
novel.novelCover = $(".featured-media > img").attr("src");
|
||||
|
||||
let details = $(".post-content").find("p").html();
|
||||
|
||||
detailName = details.match(/<strong>(.|\n)*?<\/strong>/g);
|
||||
details = details.match(/<\/strong>(.|\n)*?<br>/g);
|
||||
details = details.map((detail) => detail.replace(/<\/strong>|<br>/g, ""));
|
||||
|
||||
novel.genre = "";
|
||||
|
||||
detailName.map((detail, index) => {
|
||||
if (detail.includes("Autor")) {
|
||||
novel.author = details[index];
|
||||
}
|
||||
if (detail.includes("Géneros")) {
|
||||
novel.genre = details[index].replace(/\s/g, "");
|
||||
}
|
||||
if (detail.includes("Artista")) {
|
||||
novel.artist = details[index];
|
||||
}
|
||||
});
|
||||
|
||||
let novelSummary = $(".post-content").find("p").html();
|
||||
novel.summary = novelSummary;
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
$(".wp-block-media-text__content")
|
||||
.find("a")
|
||||
.each(function (result) {
|
||||
const chapterName = $(this).text().trim();
|
||||
|
||||
const releaseDate = null;
|
||||
|
||||
let chapterUrl = $(this).attr("href").split("/");
|
||||
chapterUrl = chapterUrl[chapterUrl.length - 2];
|
||||
|
||||
const chapter = { chapterName, releaseDate, chapterUrl };
|
||||
|
||||
novelChapters.push(chapter);
|
||||
});
|
||||
|
||||
novel.chapters = novelChapters;
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = `${baseUrl}/${novelUrl}/${chapterUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let chapterName = $(".post-title").text();
|
||||
|
||||
let chapterText = $(".post-content").html();
|
||||
novelUrl = novelUrl + "/";
|
||||
chapterUrl = chapterUrl;
|
||||
|
||||
const chapter = {
|
||||
sourceId: 29,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const url = `${baseUrl}?s=${searchTerm}&post_type=wp-manga`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".post-container ").each(function (result) {
|
||||
const novelName = $(this).find(".post-header").text();
|
||||
if (
|
||||
!novelName.includes("Cap") &&
|
||||
!novelName.includes("Vol") &&
|
||||
!novelName.includes("Light Novels")
|
||||
) {
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
|
||||
let novelUrl = $(this).find("a").attr("href");
|
||||
novelUrl = novelUrl.replace(baseUrl, "");
|
||||
|
||||
const novel = {
|
||||
sourceId: 29,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
}
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
const HasuTlScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default HasuTlScraper;
|
||||
@@ -1,188 +0,0 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
import { htmlToText } from "../helpers/htmlToText";
|
||||
|
||||
const baseUrl = "https://novelasligera.com/";
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
let totalPages = 1;
|
||||
let url = baseUrl;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".elementor-column").each(function (result) {
|
||||
const novelName = $(this)
|
||||
.find(".widget-image-caption.wp-caption-text")
|
||||
.text();
|
||||
if (novelName) {
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
|
||||
let novelUrl = $(this).find("a").attr("href");
|
||||
novelUrl = novelUrl.replace(baseUrl, "");
|
||||
novelUrl = novelUrl.replace("novela/", "");
|
||||
|
||||
const novel = {
|
||||
sourceId: 26,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
}
|
||||
});
|
||||
|
||||
return { totalPages, novels };
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = baseUrl + "novela/" + novelUrl;
|
||||
|
||||
console.log(url);
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.sourceId = 26;
|
||||
|
||||
novel.sourceName = "Novelas Ligera";
|
||||
|
||||
novel.url = url;
|
||||
|
||||
novel.novelUrl = novelUrl;
|
||||
|
||||
novel.novelName = $("h1").text();
|
||||
|
||||
novel.novelCover = $(".elementor-widget-container").find("img").attr("src");
|
||||
|
||||
$(".elementor-row")
|
||||
.find("p")
|
||||
.each(function (result) {
|
||||
if ($(this).text().includes("Autor:")) {
|
||||
novel.author = $(this).text().replace("Autor:", "").trim();
|
||||
}
|
||||
if ($(this).text().includes("Estado:")) {
|
||||
novel.status = $(this).text().replace("Estado: ", "").trim();
|
||||
}
|
||||
|
||||
if ($(this).text().includes("Género:")) {
|
||||
$(this).find("span").remove();
|
||||
novel.genre = $(this).text().replace(/,\s/g, ",");
|
||||
}
|
||||
});
|
||||
|
||||
novel.artist = null;
|
||||
|
||||
novel.novelSummary = $(".elementor-text-editor.elementor-clearfix").text();
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
$(".elementor-accordion-item").remove();
|
||||
|
||||
$(".elementor-tab-content")
|
||||
.find("li")
|
||||
.each(function (result) {
|
||||
const chapterName = $(this).text();
|
||||
const releaseDate = null;
|
||||
const chapterUrl = $(this)
|
||||
.find("a")
|
||||
.attr("href")
|
||||
.replace(baseUrl + "novela/", "");
|
||||
|
||||
const chapter = { chapterName, releaseDate, chapterUrl };
|
||||
|
||||
novelChapters.push(chapter);
|
||||
});
|
||||
|
||||
novel.chapters = novelChapters;
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = baseUrl + "novela/" + chapterUrl;
|
||||
console.log(url);
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let chapterName = $("h1.entry-title").text();
|
||||
|
||||
$(".osny-nightmode.osny-nightmode--left").remove();
|
||||
$(".code-block.code-block-1").remove();
|
||||
$(".adsb30").remove();
|
||||
$(".saboxplugin-wrap").remove();
|
||||
$(".wp-post-navigation").remove();
|
||||
|
||||
let chapterText = $(".entry-content").html();
|
||||
chapterText = htmlToText(chapterText, { preserveNewlines: true });
|
||||
|
||||
const chapter = {
|
||||
sourceId: 26,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const url = baseUrl + "?s=" + searchTerm + "&post_type=wp-manga";
|
||||
console.log(url);
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".inside-article").each(function (result) {
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
let novelUrl = $(this).find("a").attr("href").split("/")[4];
|
||||
|
||||
let novelName;
|
||||
|
||||
if (novelUrl) {
|
||||
novelName = novelUrl.replace(/-/g, " ").replace(/^./, function (x) {
|
||||
return x.toUpperCase();
|
||||
});
|
||||
}
|
||||
|
||||
novelUrl += "/";
|
||||
|
||||
const novel = {
|
||||
sourceId: 26,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
novels = [{ ...novels[1] }];
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const NovelasLigeraScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default NovelasLigeraScraper;
|
||||
@@ -1,225 +0,0 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
|
||||
const baseUrl = "http://www.novelawuxia.com/";
|
||||
|
||||
function getNovelName(y) {
|
||||
return y.replace(/-/g, " ").replace(/(?:^|\s)\S/g, (a) => a.toUpperCase());
|
||||
}
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
let totalPages = 1;
|
||||
let url = baseUrl + "p/todas-las-novelas.html";
|
||||
|
||||
let headers = new Headers({
|
||||
"User-Agent":
|
||||
"'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36",
|
||||
});
|
||||
|
||||
const result = await fetch(url, {
|
||||
method: "GET",
|
||||
headers: headers,
|
||||
});
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".post-body.entry-content")
|
||||
.find("a")
|
||||
.each(function (result) {
|
||||
let novelName = $(this)
|
||||
.attr("href")
|
||||
.split("/")
|
||||
.pop()
|
||||
.replace(".html", "");
|
||||
novelName = getNovelName(novelName);
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
|
||||
let novelUrl = $(this).attr("href");
|
||||
novelUrl = novelUrl.replace(`${baseUrl}p/`, "") + "/";
|
||||
|
||||
const novel = {
|
||||
sourceId: 31,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return { totalPages, novels };
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = `${baseUrl}p/${novelUrl.replace("/", "")}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.sourceId = 31;
|
||||
|
||||
novel.sourceName = "Novela Wuxia";
|
||||
|
||||
novel.url = url;
|
||||
|
||||
novel.novelUrl = novelUrl;
|
||||
|
||||
novel.novelName = $("h1.post-title").text().trim();
|
||||
|
||||
novel.novelCover = $("div.separator").find("a").attr("href");
|
||||
|
||||
novel.artist = "";
|
||||
novel.status = "";
|
||||
|
||||
$("div > b").each(function (result) {
|
||||
const detailName = $(this).text();
|
||||
let detail = $(this)[0].nextSibling;
|
||||
|
||||
if (detailName && detail) {
|
||||
detail = detail.nodeValue;
|
||||
|
||||
if (detailName.includes("Autor")) {
|
||||
novel.author = detail.replace("Autor:", "");
|
||||
}
|
||||
|
||||
if (detailName.includes("Estatus")) {
|
||||
novel.status = detail.replace("Estatus: ", "");
|
||||
}
|
||||
if (detailName.includes("Géneros:")) {
|
||||
novel.genre = detail
|
||||
.replace("Géneros: ", "")
|
||||
.replace(/,\s/g, ",");
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
$("div").each(function (result) {
|
||||
const detailName = $(this).text();
|
||||
if (detailName.includes("Sinopsis")) {
|
||||
novel.summary =
|
||||
$(this).next().text() !== ""
|
||||
? $(this).next().text().replace("Sinopsis", "").trim()
|
||||
: $(this)
|
||||
.next()
|
||||
.next()
|
||||
.text()
|
||||
.replace("Sinopsis", "")
|
||||
.trim();
|
||||
}
|
||||
|
||||
if (detailName.includes("Lista de Capítulos")) {
|
||||
$(this)
|
||||
.find("a")
|
||||
.each(function (res) {
|
||||
const chapterName = $(this).text();
|
||||
let chapterUrl = $(this).attr("href");
|
||||
const releaseDate = null;
|
||||
|
||||
if (
|
||||
chapterName &&
|
||||
chapterUrl &&
|
||||
chapterUrl.includes(novelUrl.replace(".html", "")) &&
|
||||
!novelChapters.some(
|
||||
(chap) => chap.chapterName === chapterName
|
||||
)
|
||||
) {
|
||||
chapterUrl = chapterUrl.replace(baseUrl, "");
|
||||
|
||||
const chapter = {
|
||||
chapterName,
|
||||
releaseDate,
|
||||
chapterUrl,
|
||||
};
|
||||
|
||||
novelChapters.push(chapter);
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
novel.chapters = novelChapters;
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = `${baseUrl}${year}/${month}/${chapterUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let chapterName = $("h1.post-title").text().trim();
|
||||
|
||||
let chapterText = $(".post-body.entry-content").html();
|
||||
novelUrl = novelUrl + "/";
|
||||
chapterUrl = chapterUrl + "/";
|
||||
|
||||
const chapter = {
|
||||
sourceId: 31,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const url = `${baseUrl}search?q=${searchTerm}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".date-outer").each(function (result) {
|
||||
let novelName = $(this)
|
||||
.find("a")
|
||||
.attr("href")
|
||||
.split("/")
|
||||
.pop()
|
||||
.replace(/-capitulo(.*?).html/, "");
|
||||
|
||||
const novelUrl = novelName + ".html/";
|
||||
|
||||
novelName = getNovelName(novelName);
|
||||
|
||||
const exists = novels.some((novel) => novel.novelName === novelName);
|
||||
|
||||
if (!exists) {
|
||||
const novelCover = null;
|
||||
const novel = {
|
||||
sourceId: 31,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
}
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const NovelaWuxiaScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default NovelaWuxiaScraper;
|
||||
@@ -1,205 +0,0 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
|
||||
const baseUrl = "https://oasistranslations.wordpress.com/";
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
let url = baseUrl;
|
||||
|
||||
let totalPages = 1;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".menu-item-1819")
|
||||
.find(".sub-menu > li")
|
||||
.each(function (result) {
|
||||
const novelName = $(this).text();
|
||||
if (!novelName.match(/Activas|Finalizadas|Dropeadas/)) {
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
|
||||
let novelUrl = $(this).find("a").attr("href");
|
||||
novelUrl = novelUrl.split("/");
|
||||
novelUrl = novelUrl[novelUrl.length - 2] + "/";
|
||||
|
||||
const novel = {
|
||||
sourceId: 30,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
}
|
||||
});
|
||||
|
||||
return { totalPages, novels };
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = baseUrl + novelUrl + "/";
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.sourceId = 30;
|
||||
|
||||
novel.sourceName = "Oasis Translations";
|
||||
|
||||
novel.url = url;
|
||||
|
||||
novel.novelUrl = novelUrl;
|
||||
|
||||
novel.novelName = $("h1.entry-title")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
novel.novelCover = $('img[loading="lazy"]').attr("src");
|
||||
|
||||
$(".entry-content > p").each(function (res) {
|
||||
if ($(this).text().includes("Autor")) {
|
||||
let details = $(this).html();
|
||||
details = details.match(/<\/strong>(.|\n)*?<br>/g);
|
||||
details = details.map((detail) =>
|
||||
detail.replace(/<strong>|<\/strong>|<br>|:\s/g, "")
|
||||
);
|
||||
|
||||
novel.genre = "";
|
||||
|
||||
novel.author = details[2];
|
||||
novel.genre = details[4].replace(/\s| /g, "");
|
||||
novel.artist = details[3];
|
||||
}
|
||||
});
|
||||
|
||||
// let novelSummary = $(this).next().html();
|
||||
novel.summary = "";
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
// if ($(".entry-content").find("li").length) {
|
||||
$(".entry-content")
|
||||
.find("a")
|
||||
.each(function (result) {
|
||||
let chapterUrl = $(this).attr("href");
|
||||
|
||||
if (chapterUrl && chapterUrl.includes(baseUrl)) {
|
||||
const chapterName = $(this).text();
|
||||
const releaseDate = null;
|
||||
|
||||
chapterUrl = chapterUrl.split("/");
|
||||
chapterUrl = chapterUrl[chapterUrl.length - 2] + "/";
|
||||
|
||||
const chapter = { chapterName, releaseDate, chapterUrl };
|
||||
|
||||
novelChapters.push(chapter);
|
||||
}
|
||||
});
|
||||
// } else {
|
||||
// $(".entry-content")
|
||||
// .find("p")
|
||||
// .each(function (result) {
|
||||
// let chapterUrl = $(this).find("a").attr("href");
|
||||
|
||||
// if (chapterUrl && chapterUrl.includes(baseUrl)) {
|
||||
// const chapterName = $(this).text();
|
||||
// const releaseDate = null;
|
||||
|
||||
// chapterUrl = chapterUrl.split("/");
|
||||
// chapterUrl = chapterUrl[chapterUrl.length - 2] + "/";
|
||||
|
||||
// const chapter = { chapterName, releaseDate, chapterUrl };
|
||||
|
||||
// novelChapters.push(chapter);
|
||||
// }
|
||||
// });
|
||||
// }
|
||||
|
||||
novel.chapters = novelChapters;
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = `${baseUrl}/${novelUrl}/${chapterUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let chapterName = $("h1.entry-title").text();
|
||||
|
||||
$("div#jp-post-flair").remove();
|
||||
|
||||
let chapterText = $(".entry-content").html();
|
||||
novelUrl = novelUrl + "/";
|
||||
chapterUrl = chapterUrl + "/";
|
||||
|
||||
const chapter = {
|
||||
sourceId: 30,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
searchTerm = searchTerm.toLowerCase();
|
||||
|
||||
let url = baseUrl;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
$(".menu-item-1819")
|
||||
.find(".sub-menu > li")
|
||||
.each(function (result) {
|
||||
const novelName = $(this).text();
|
||||
if (!novelName.match(/Activas|Finalizadas|Dropeadas/)) {
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
|
||||
let novelUrl = $(this).find("a").attr("href");
|
||||
novelUrl = novelUrl.split("/");
|
||||
novelUrl = novelUrl[novelUrl.length - 2] + "/";
|
||||
|
||||
const novel = {
|
||||
sourceId: 30,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
}
|
||||
});
|
||||
|
||||
novels = novels.filter((novel) =>
|
||||
novel.novelName.toLowerCase().includes(searchTerm)
|
||||
);
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const OasisTranslationsScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default OasisTranslationsScraper;
|
||||
@@ -1,148 +0,0 @@
|
||||
const baseUrl = "https://www.skynovels.net/";
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
let totalPages = 1;
|
||||
const url = "https://api.skynovels.net/api/novels?&q";
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.json();
|
||||
|
||||
const novels = [];
|
||||
|
||||
body.novels.map((res) => {
|
||||
const novelName = res.nvl_title;
|
||||
const novelCover =
|
||||
"https://api.skynovels.net/api/get-image/" +
|
||||
res.image +
|
||||
"/novels/false";
|
||||
const novelUrl = res.id + "/" + res.nvl_name + "/";
|
||||
|
||||
const novel = { sourceId: 24, novelName, novelUrl, novelCover };
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return { totalPages, novels };
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novUrl) => {
|
||||
let novelId = novUrl.split("/")[0];
|
||||
let novelUrl = novUrl[1];
|
||||
const url =
|
||||
"https://api.skynovels.net/api/novel/" + novelId + "/reading?&q";
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.json();
|
||||
|
||||
const item = body.novel[0];
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.sourceId = 24;
|
||||
|
||||
novel.sourceName = "SkyNovels";
|
||||
|
||||
novel.url = baseUrl + "novelas/" + novelId + "/" + novelUrl;
|
||||
|
||||
novel.novelUrl = novUrl;
|
||||
|
||||
novel.novelName = item.nvl_title;
|
||||
|
||||
novel.novelCover =
|
||||
"https://api.skynovels.net/api/get-image/" +
|
||||
item.image +
|
||||
"/novels/false";
|
||||
|
||||
let genres = [];
|
||||
item.genres.map((genre) => genres.push(genre.genre_name));
|
||||
novel.genre = genres.join(",");
|
||||
novel.author = item.nvl_writer;
|
||||
novel.summary = item.nvl_content;
|
||||
novel.status = item.nvl_status;
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
item.volumes.map((volume) => {
|
||||
volume.chapters.map((chapter) => {
|
||||
const chapterName = chapter.chp_index_title;
|
||||
const releaseDate = new Date(chapter.createdAt).toDateString();
|
||||
const chapterUrl = chapter.id + "/" + chapter.chp_name;
|
||||
|
||||
const chap = { chapterName, releaseDate, chapterUrl };
|
||||
|
||||
novelChapters.push(chap);
|
||||
});
|
||||
});
|
||||
|
||||
novel.chapters = novelChapters;
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
const parseChapter = async (novUrl, chapUrl) => {
|
||||
let novelId = novUrl.split("/")[0];
|
||||
let novelUrl = novUrl[1];
|
||||
let chapterId = chapUrl.split("/")[0];
|
||||
let chapterUrl = chapUrl[1];
|
||||
const url = `https://api.skynovels.net/api/novel-chapter/${chapterId}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.json();
|
||||
|
||||
const item = body.chapter[0];
|
||||
|
||||
let chapterName = item.chp_index_title;
|
||||
|
||||
let chapterText = item.chp_content;
|
||||
|
||||
novelUrl = novelId + "/" + novelUrl + "/";
|
||||
chapterUrl = item.id + "/" + item.chp_name;
|
||||
|
||||
const chapter = {
|
||||
sourceId: 24,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
searchTerm = searchTerm.toLowerCase();
|
||||
const url = "https://api.skynovels.net/api/novels?&q";
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.json();
|
||||
|
||||
let results = body.novels.filter((novel) =>
|
||||
novel.nvl_title.toLowerCase().includes(searchTerm)
|
||||
);
|
||||
|
||||
const novels = [];
|
||||
|
||||
results.map((res) => {
|
||||
const novelName = res.nvl_title;
|
||||
const novelCover =
|
||||
"https://api.skynovels.net/api/get-image/" +
|
||||
res.image +
|
||||
"/novels/false";
|
||||
const novelUrl = res.id + "/" + res.nvl_name + "/";
|
||||
|
||||
const novel = { sourceId: 24, novelName, novelUrl, novelCover };
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const SkyNovelsScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default SkyNovelsScraper;
|
||||
@@ -1,190 +0,0 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
import { defaultCoverUri, Status } from "../helpers/constants";
|
||||
|
||||
const sourceId = 23;
|
||||
const sourceName = "TuNovelaLigera";
|
||||
|
||||
const baseUrl = "https://tunovelaligera.com/";
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
let totalPages = 62;
|
||||
let url = baseUrl + "novelas/page/" + page + "/?m_orderby=rating";
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".page-item-detail").each(function () {
|
||||
const novelName = $(this).find(".h5 > a").text();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
|
||||
let novelUrl = $(this).find(".h5 > a").attr("href").split("/")[4];
|
||||
novelUrl += "/";
|
||||
|
||||
const novel = {
|
||||
sourceId,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return { totalPages, novels };
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = `${baseUrl}novelas/${novelUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = { sourceId, sourceName, url };
|
||||
|
||||
novel.novelUrl = novelUrl;
|
||||
|
||||
$(".manga-title-badges").remove();
|
||||
|
||||
novel.novelName = $(".post-title > h1").text().trim();
|
||||
|
||||
novelCover = $(".summary_image > a > img");
|
||||
|
||||
novel.novelCover =
|
||||
novelCover.attr("data-src") ||
|
||||
novelCover.attr("src") ||
|
||||
defaultCoverUri;
|
||||
|
||||
$(".post-content_item").each(function () {
|
||||
const detailName = $(this).find(".summary-heading > h5").text().trim();
|
||||
const detail = $(this).find(".summary-content").text().trim();
|
||||
|
||||
switch (detailName) {
|
||||
case "Generos":
|
||||
novel.genre = detail.replace(/, /g, ",");
|
||||
break;
|
||||
case "Autores":
|
||||
novel.author = detail;
|
||||
break;
|
||||
case "Estado":
|
||||
novel.status =
|
||||
detail.includes("OnGoing") || detail.includes("Updating")
|
||||
? Status.ONGOING
|
||||
: Status.COMPLETED;
|
||||
break;
|
||||
}
|
||||
});
|
||||
|
||||
novel.summary = $("div.summary__content > p").text().trim();
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
const novelId =
|
||||
$(".rating-post-id").attr("value") ||
|
||||
$("#manga-chapters-holder").attr("data-id");
|
||||
|
||||
let formData = new FormData();
|
||||
formData.append("action", "manga_get_chapters");
|
||||
formData.append("manga", novelId);
|
||||
|
||||
const data = await fetch(
|
||||
"https://tunovelaligera.com/wp-admin/admin-ajax.php",
|
||||
{
|
||||
method: "POST",
|
||||
body: formData,
|
||||
}
|
||||
);
|
||||
const text = await data.text();
|
||||
|
||||
$ = cheerio.load(text);
|
||||
|
||||
$(".wp-manga-chapter").each(function (result) {
|
||||
chapterName = $(this)
|
||||
.find("a")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
releaseDate = $(this).find("span").text().trim();
|
||||
|
||||
chapterUrl = $(this).find("a").attr("href").split("/");
|
||||
|
||||
chapterUrl[6]
|
||||
? (chapterUrl = chapterUrl[5] + "/" + chapterUrl[6])
|
||||
: (chapterUrl = chapterUrl[5]);
|
||||
|
||||
novelChapters.push({ chapterName, releaseDate, chapterUrl });
|
||||
});
|
||||
|
||||
novel.chapters = novelChapters.reverse();
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = `${baseUrl}novelas/${novelUrl}/${chapterUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let chapterName = $("h1#chapter-heading").text();
|
||||
|
||||
let chapterText = $(".text-left").html();
|
||||
novelUrl = novelUrl + "/";
|
||||
|
||||
const chapter = {
|
||||
sourceId,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const url = `${baseUrl}?s=${searchTerm}&post_type=wp-manga`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".c-tabs-item__content").each(function (result) {
|
||||
const novelName = $(this).find(".h4 > a").text();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
|
||||
let novelUrl = $(this).find(".h4 > a").attr("href");
|
||||
novelUrl = novelUrl.replace(`${baseUrl}novelas/`, "");
|
||||
|
||||
const novel = {
|
||||
sourceId,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const TuNovelaLigeraScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default TuNovelaLigeraScraper;
|
||||
@@ -1,204 +0,0 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
|
||||
const baseUrl = "https://yuukitls.com/";
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
let totalPages = 1;
|
||||
let url = baseUrl;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".menu-item-2869")
|
||||
.find(".menu-item.menu-item-type-post_type.menu-item-object-post")
|
||||
.each(function (result) {
|
||||
const novelName = $(this).text();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
|
||||
let novelUrl = $(this).find("a").attr("href");
|
||||
novelUrl = novelUrl.split("/");
|
||||
novelUrl = novelUrl[novelUrl.length - 2] + "/";
|
||||
|
||||
const novel = {
|
||||
sourceId: 28,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return { totalPages, novels };
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = `${baseUrl}${novelUrl}/`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.sourceId = 28;
|
||||
|
||||
novel.sourceName = "YuukiTls";
|
||||
|
||||
novel.url = url;
|
||||
|
||||
novel.novelUrl = novelUrl;
|
||||
|
||||
novel.novelName = $("h1.entry-title")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
novel.novelCover = $('img[loading="lazy"]').attr("src");
|
||||
|
||||
$(".entry-content")
|
||||
.find("div")
|
||||
.each(function (result) {
|
||||
if ($(this).text().includes("Escritor:")) {
|
||||
novel.author = $(this).text().replace("Escritor: ", "").trim();
|
||||
}
|
||||
if ($(this).text().includes("Ilustrador:")) {
|
||||
novel.artist = $(this)
|
||||
.text()
|
||||
.replace("Ilustrador: ", "")
|
||||
.trim();
|
||||
}
|
||||
|
||||
if ($(this).text().includes("Género:")) {
|
||||
novel.genre = $(this)
|
||||
.text()
|
||||
.replace(/Género: |\s/g, "");
|
||||
}
|
||||
|
||||
if ($(this).text().includes("Sinopsis:")) {
|
||||
novel.summary = $(this).next().text();
|
||||
}
|
||||
});
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
if ($(".entry-content").find("li").length) {
|
||||
$(".entry-content")
|
||||
.find("li")
|
||||
.each(function (result) {
|
||||
let chapterUrl = $(this).find("a").attr("href");
|
||||
|
||||
if (chapterUrl && chapterUrl.includes(baseUrl)) {
|
||||
const chapterName = $(this).text();
|
||||
const releaseDate = null;
|
||||
|
||||
chapterUrl = chapterUrl.split("/");
|
||||
chapterUrl = chapterUrl[chapterUrl.length - 2] + "/";
|
||||
|
||||
const chapter = { chapterName, releaseDate, chapterUrl };
|
||||
|
||||
novelChapters.push(chapter);
|
||||
}
|
||||
});
|
||||
} else {
|
||||
$(".entry-content")
|
||||
.find("p")
|
||||
.each(function (result) {
|
||||
let chapterUrl = $(this).find("a").attr("href");
|
||||
|
||||
if (chapterUrl && chapterUrl.includes(baseUrl)) {
|
||||
const chapterName = $(this).text();
|
||||
const releaseDate = null;
|
||||
|
||||
chapterUrl = chapterUrl.split("/");
|
||||
chapterUrl = chapterUrl[chapterUrl.length - 2] + "/";
|
||||
|
||||
const chapter = { chapterName, releaseDate, chapterUrl };
|
||||
|
||||
novelChapters.push(chapter);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
novel.chapters = novelChapters;
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = `${baseUrl}/${novelUrl}/${chapterUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let chapterName = $("h3").text();
|
||||
|
||||
let chapterText = $(".entry-content").html();
|
||||
novelUrl = novelUrl + "/";
|
||||
chapterUrl = chapterUrl + "/";
|
||||
|
||||
const chapter = {
|
||||
sourceId: 28,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
searchTerm = searchTerm.toLowerCase();
|
||||
|
||||
let url = baseUrl;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".menu-item-2869")
|
||||
.find(".menu-item.menu-item-type-post_type.menu-item-object-post")
|
||||
.each(function (result) {
|
||||
const novelName = $(this).text();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
|
||||
let novelUrl = $(this).find("a").attr("href");
|
||||
novelUrl = novelUrl.split("/");
|
||||
novelUrl = novelUrl[novelUrl.length - 2] + "/";
|
||||
|
||||
const novel = {
|
||||
sourceId: 28,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
novels = novels.filter((novel) =>
|
||||
novel.novelName.toLowerCase().includes(searchTerm)
|
||||
);
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const YuukiTlsScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default YuukiTlsScraper;
|
||||
@@ -1,151 +0,0 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
|
||||
const sourceId = 77;
|
||||
|
||||
const sourceName = "NovelDeGlace";
|
||||
|
||||
const baseUrl = "https://noveldeglace.com/";
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
const totalPages = 1;
|
||||
|
||||
let url = baseUrl + "roman";
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$("article").each(function () {
|
||||
const novelName = $(this).find("h2").text().trim();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
const novelUrl = $(this).find("h2 > a").attr("href").split("/")[4];
|
||||
|
||||
const novel = {
|
||||
sourceId,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return { novels, totalPages };
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = baseUrl + "roman/" + novelUrl;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = { sourceId, sourceName, url, novelUrl };
|
||||
|
||||
novel.novelName = $(
|
||||
"div.entry-content > div > strong"
|
||||
)[0].nextSibling.nodeValue.trim();
|
||||
|
||||
novel.novelCover = $(".su-row > div > div > img").attr("src");
|
||||
|
||||
novel.summary = $("div[data-title=Synopsis]").text();
|
||||
|
||||
author = $(
|
||||
"div.romans > div.project-large > div.su-row > div.su-column.su-column-size-3-4 > div > div:nth-child(3) > strong"
|
||||
)[0];
|
||||
|
||||
novel.author = author ? author.nextSibling.nodeValue.trim() : null;
|
||||
|
||||
novel.genre = $(".genre")
|
||||
.text()
|
||||
.replace("Genre : ", "")
|
||||
.replace(/, /g, ",");
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
$(".chpt").each(function () {
|
||||
const chapterName = $(this).find("a").text().trim();
|
||||
const releaseDate = null;
|
||||
const chapterUrl = $(this).find("a").attr("href");
|
||||
|
||||
const novel = {
|
||||
sourceId,
|
||||
chapterName,
|
||||
releaseDate,
|
||||
chapterUrl,
|
||||
};
|
||||
|
||||
novelChapters.push(novel);
|
||||
});
|
||||
|
||||
novel.chapters = novelChapters;
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = chapterUrl;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let chapterName = $("h2.western").text();
|
||||
let chapterText = $(".chapter-content").html();
|
||||
|
||||
const chapter = {
|
||||
sourceId,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
let url = baseUrl + "roman";
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$("article").each(function () {
|
||||
const novelName = $(this).find("h2").text().trim();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
const novelUrl = $(this).find("h2 > a").attr("href").split("/")[4];
|
||||
|
||||
const novel = {
|
||||
sourceId,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
novels = novels.filter((novel) =>
|
||||
novel.novelName.toLowerCase().includes(searchTerm.toLowerCase())
|
||||
);
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const NovelDeGlaceScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default NovelDeGlaceScraper;
|
||||
@@ -1,36 +0,0 @@
|
||||
export const cleanHtml = (html) => {
|
||||
let text = html;
|
||||
|
||||
if (!text) {
|
||||
return "Chapter is empty.\n\nReport if it's available in webview.";
|
||||
}
|
||||
|
||||
text = text
|
||||
/**
|
||||
* Remove inline styles
|
||||
*/
|
||||
.replace(
|
||||
/style="((?:font-family|line-height|font-size|text-align).*?)"/gi,
|
||||
""
|
||||
)
|
||||
.replace(/class=".*?"/gi, "")
|
||||
.replace(/id=".*?"/gi, "")
|
||||
|
||||
/**
|
||||
* Remove comments
|
||||
*/
|
||||
.replace(/<!--.*?-->/gim, "")
|
||||
.replace(/<\s*script[^>]*>[\s\S]*?<\/script>/gim, "")
|
||||
.replace(/<\s*noscript[^>]*>[\s\S]*?<\/noscript>/gim, "")
|
||||
.replace(/<\s*form[^>]*>[\s\S]*?<\/form>/gim, "")
|
||||
/**
|
||||
* Remove relative links
|
||||
*/
|
||||
.replace(
|
||||
/<\s*a[^>]*href=['"](\/.*?)['"][^>]*>([\s\S]*?)<\/\s*a\s*>/gi,
|
||||
"$2"
|
||||
)
|
||||
.trim();
|
||||
|
||||
return text;
|
||||
};
|
||||
@@ -1,8 +0,0 @@
|
||||
export const defaultCoverUri =
|
||||
"https://github.com/LNReader/lnreader-sources/blob/main/src/coverNotAvailable.jpg?raw=true";
|
||||
|
||||
export const Status = {
|
||||
ONGOING: "Ongoing",
|
||||
COMPLETED: "Completed",
|
||||
UNKNOWN: "Unknown",
|
||||
};
|
||||
@@ -1,263 +0,0 @@
|
||||
const htmlToText = (html, options = {}) => {
|
||||
const { removeLineBreaks = true } = options;
|
||||
|
||||
if (!html) {
|
||||
return "Chapter is empty.\n\nReport if it's available in webview.";
|
||||
}
|
||||
|
||||
html = html.trim();
|
||||
|
||||
text = removeLineBreaks && html.replace(/(?:\n|\r\n|\r)/gi, "");
|
||||
|
||||
text = html
|
||||
/**
|
||||
* <br> -> \n
|
||||
*/
|
||||
|
||||
.replace(/<\s*br[^>]*>/gi, "\n")
|
||||
|
||||
/**
|
||||
* <hr> -> [hr]()
|
||||
*/
|
||||
|
||||
.replace(/<\s*hr[^>]*>/gi, "")
|
||||
|
||||
/**
|
||||
* <li> -> \n
|
||||
*/
|
||||
.replace(/<\s*\/li[^>]*>/gi, "\n")
|
||||
|
||||
/**
|
||||
* <p> -> \n\n
|
||||
*/
|
||||
|
||||
.replace(/<\s*p[^>]*>/gi, "\n\n")
|
||||
|
||||
/**
|
||||
* Remove script and noscript tags
|
||||
*/
|
||||
.replace(/<\s*script[^>]*>[\s\S]*?<\/script>/gim, "")
|
||||
.replace(/<\s*noscript[^>]*>[\s\S]*?<\/noscript>/gim, "")
|
||||
|
||||
/**
|
||||
* Remove style tags
|
||||
*/
|
||||
|
||||
.replace(/<\s*style[^>]*>[\s\S]*?<\/style>/gim, "")
|
||||
|
||||
/**
|
||||
* Remove comments
|
||||
*/
|
||||
|
||||
.replace(/<!--.*?-->/gim, "")
|
||||
|
||||
/**
|
||||
* <a href='link'>Text</a> -> [Text](link)
|
||||
*/
|
||||
|
||||
.replace(
|
||||
/<\s*a[^>]*href=['"](http.*?)['"][^>]*>([\s\S]*?)<\/\s*a\s*>/gi,
|
||||
"$2"
|
||||
)
|
||||
// .replace(
|
||||
// /<\s*a[^>]*href=['"](http.*?)['"][^>]*>([\s\S]*?)<\/\s*a\s*>/gi,
|
||||
// "[$2]($1)"
|
||||
// )
|
||||
/**
|
||||
* <img> -> [img](src)
|
||||
*/
|
||||
|
||||
// .replace(/<\s*img[^>]*src=['"](.*?)['"][^>]*>/gi, "[img]($1)")
|
||||
|
||||
/**
|
||||
* Remove remaining tags
|
||||
*/
|
||||
|
||||
.replace(/(<([^>]+)>)/gi, "")
|
||||
|
||||
/**
|
||||
* Remove tabs.
|
||||
*/
|
||||
|
||||
.replace(/\t/g, "")
|
||||
|
||||
/**
|
||||
* Remove newlines at the beginning of the text.
|
||||
*/
|
||||
|
||||
.replace(/^\n+/m, "")
|
||||
|
||||
/**
|
||||
* Replace multiple spaces with a single space.
|
||||
*/
|
||||
|
||||
.replace(/ {2,}/g, " ")
|
||||
|
||||
/**
|
||||
* Make sure there are never more than two consecutive linebreaks.
|
||||
*/
|
||||
|
||||
.replace(/\n{2,}/g, "\n\n")
|
||||
|
||||
/**
|
||||
* Decode HTML entities.
|
||||
*/
|
||||
|
||||
.replace(/&([^;]+);/g, decodeHtmlEntity)
|
||||
|
||||
.trim();
|
||||
|
||||
return text;
|
||||
};
|
||||
|
||||
const decodeHtmlEntity = (m, n) => {
|
||||
let code;
|
||||
|
||||
if (n.substr(0, 1) == "#") {
|
||||
if (n.substr(1, 1) == "x") {
|
||||
code = parseInt(n.substr(2), 16);
|
||||
} else {
|
||||
code = parseInt(n.substr(1), 10);
|
||||
}
|
||||
} else {
|
||||
code = ENTITIES_MAP[n];
|
||||
}
|
||||
|
||||
return code === undefined || code === NaN
|
||||
? "&" + n + ";"
|
||||
: String.fromCharCode(code);
|
||||
};
|
||||
|
||||
let ENTITIES_MAP = {
|
||||
nbsp: 160,
|
||||
iexcl: 161,
|
||||
cent: 162,
|
||||
pound: 163,
|
||||
curren: 164,
|
||||
yen: 165,
|
||||
brvbar: 166,
|
||||
sect: 167,
|
||||
uml: 168,
|
||||
copy: 169,
|
||||
ordf: 170,
|
||||
laquo: 171,
|
||||
not: 172,
|
||||
shy: 173,
|
||||
reg: 174,
|
||||
macr: 175,
|
||||
deg: 176,
|
||||
plusmn: 177,
|
||||
sup2: 178,
|
||||
sup3: 179,
|
||||
acute: 180,
|
||||
micro: 181,
|
||||
para: 182,
|
||||
middot: 183,
|
||||
cedil: 184,
|
||||
sup1: 185,
|
||||
ordm: 186,
|
||||
raquo: 187,
|
||||
frac14: 188,
|
||||
frac12: 189,
|
||||
frac34: 190,
|
||||
iquest: 191,
|
||||
Agrave: 192,
|
||||
Aacute: 193,
|
||||
Acirc: 194,
|
||||
Atilde: 195,
|
||||
Auml: 196,
|
||||
Aring: 197,
|
||||
AElig: 198,
|
||||
Ccedil: 199,
|
||||
Egrave: 200,
|
||||
Eacute: 201,
|
||||
Ecirc: 202,
|
||||
Euml: 203,
|
||||
Igrave: 204,
|
||||
Iacute: 205,
|
||||
Icirc: 206,
|
||||
Iuml: 207,
|
||||
ETH: 208,
|
||||
Ntilde: 209,
|
||||
Ograve: 210,
|
||||
Oacute: 211,
|
||||
Ocirc: 212,
|
||||
Otilde: 213,
|
||||
Ouml: 214,
|
||||
times: 215,
|
||||
Oslash: 216,
|
||||
Ugrave: 217,
|
||||
Uacute: 218,
|
||||
Ucirc: 219,
|
||||
Uuml: 220,
|
||||
Yacute: 221,
|
||||
THORN: 222,
|
||||
szlig: 223,
|
||||
agrave: 224,
|
||||
aacute: 225,
|
||||
acirc: 226,
|
||||
atilde: 227,
|
||||
auml: 228,
|
||||
aring: 229,
|
||||
aelig: 230,
|
||||
ccedil: 231,
|
||||
egrave: 232,
|
||||
eacute: 233,
|
||||
ecirc: 234,
|
||||
euml: 235,
|
||||
igrave: 236,
|
||||
iacute: 237,
|
||||
icirc: 238,
|
||||
iuml: 239,
|
||||
eth: 240,
|
||||
ntilde: 241,
|
||||
ograve: 242,
|
||||
oacute: 243,
|
||||
ocirc: 244,
|
||||
otilde: 245,
|
||||
ouml: 246,
|
||||
divide: 247,
|
||||
oslash: 248,
|
||||
ugrave: 249,
|
||||
uacute: 250,
|
||||
ucirc: 251,
|
||||
uuml: 252,
|
||||
yacute: 253,
|
||||
thorn: 254,
|
||||
yuml: 255,
|
||||
quot: 34,
|
||||
amp: 38,
|
||||
lt: 60,
|
||||
gt: 62,
|
||||
OElig: 338,
|
||||
oelig: 339,
|
||||
Scaron: 352,
|
||||
scaron: 353,
|
||||
Yuml: 376,
|
||||
circ: 710,
|
||||
tilde: 732,
|
||||
ensp: 8194,
|
||||
emsp: 8195,
|
||||
thinsp: 8201,
|
||||
zwnj: 8204,
|
||||
zwj: 8205,
|
||||
lrm: 8206,
|
||||
rlm: 8207,
|
||||
ndash: 8211,
|
||||
mdash: 8212,
|
||||
lsquo: 8216,
|
||||
rsquo: 8217,
|
||||
sbquo: 8218,
|
||||
ldquo: 8220,
|
||||
rdquo: 8221,
|
||||
bdquo: 8222,
|
||||
dagger: 8224,
|
||||
Dagger: 8225,
|
||||
permil: 8240,
|
||||
lsaquo: 8249,
|
||||
rsaquo: 8250,
|
||||
euro: 8364,
|
||||
apos: 39,
|
||||
};
|
||||
|
||||
export { htmlToText };
|
||||
@@ -1,28 +0,0 @@
|
||||
import moment from "moment";
|
||||
|
||||
export const parseMadaraDate = (date) => {
|
||||
let releaseDate = date;
|
||||
|
||||
if (date.includes("ago")) {
|
||||
let timeAgo = date;
|
||||
releaseDate = new Date();
|
||||
|
||||
timeAgo = timeAgo.match(/\d+/)[0];
|
||||
|
||||
if (timeAgo.includes("hours ago") || timeAgo.includes("hour ago")) {
|
||||
releaseDate.setHours(releaseDate.getHours() - timeAgo);
|
||||
}
|
||||
|
||||
if (timeAgo.includes("days ago") || timeAgo.includes("day ago")) {
|
||||
releaseDate.setDate(releaseDate.getDate() - timeAgo);
|
||||
}
|
||||
|
||||
if (timeAgo.includes("months ago") || timeAgo.includes("month ago")) {
|
||||
releaseDate.setMonth(releaseDate.getMonth() - timeAgo);
|
||||
}
|
||||
|
||||
releaseDate = moment(releaseDate).format("MMMM DD, YYYY");
|
||||
}
|
||||
|
||||
return releaseDate;
|
||||
};
|
||||
@@ -1,261 +0,0 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
|
||||
import moment from "moment";
|
||||
|
||||
const baseUrl = "https://syosetu.com"; // base url for syosetu.com
|
||||
|
||||
// get given page of search (if pagenum is 0 or >100 (max possible on site) see first page)
|
||||
const searchUrl = (pagenum, order) => {
|
||||
return `https://yomou.syosetu.com/search.php?order=${order || "hyoka"}${
|
||||
!isNaN((pagenum = parseInt(pagenum))) // check if pagenum is a number
|
||||
? `&p=${pagenum <= 1 || pagenum > 100 ? "1" : pagenum}` // check if pagenum is between 1 and 100
|
||||
: "" // if isn't don't set ?p
|
||||
}`;
|
||||
};
|
||||
|
||||
// get the Syosetu ID of the chapter
|
||||
const getLastPartOfUrl = (url) =>
|
||||
(ex = /.*(?=\/)(.{2,})$/.exec(url)) ? ex[1].replace(/\//g, "") : null;
|
||||
|
||||
// get novelUrl from Syosetu ID
|
||||
const getNovelUrl = (id) => `https://ncode.syosetu.com/${id}`;
|
||||
|
||||
// get chapterUrl from Syosetu ID and chapter ID
|
||||
const getChapterUrl = (id, chn) =>
|
||||
`https://ncode.syosetu.com/${id}${chn === "oneshot" ? "" : `/${chn}`}`;
|
||||
|
||||
// Because syosetu does not have any covers, I did a replacement for them
|
||||
const novelCover =
|
||||
"https://raw.githubusercontent.com/skillgg/lnreader-sources/main/src/jp/syosetu/not_found.png";
|
||||
|
||||
// ID of this extension
|
||||
const sourceId = 36;
|
||||
|
||||
// Name of this extension
|
||||
const sourceName = "Syosetu";
|
||||
|
||||
// there are 20 mangas per page
|
||||
// this is number of pages loaded in the first "batch" of loads
|
||||
// TOOD: in-app dynamic loading on scroll
|
||||
const maxPageLoad = 3;
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
const totalPages = 100;
|
||||
// array of all the novels
|
||||
let novels = [];
|
||||
// returns list of novels from given page
|
||||
let getNovelsFromPage = async (pagenumber) => {
|
||||
// load page
|
||||
const result = await fetch(searchUrl(pagenumber || null));
|
||||
const body = await result.text();
|
||||
// Cheerio it!
|
||||
const cheerioQuery = cheerio.load(body, { decodeEntities: false });
|
||||
|
||||
let pageNovels = [];
|
||||
// find class=searchkekka_box
|
||||
cheerioQuery(".searchkekka_box").each(function (i, e) {
|
||||
// get div with link and name
|
||||
const novelDIV = cheerioQuery(this).find(".novel_h");
|
||||
// get link element
|
||||
const novelA = novelDIV.children()[0];
|
||||
// add new novel to array
|
||||
pageNovels.push({
|
||||
novelName: novelDIV.text(), // get the name
|
||||
novelUrl: getLastPartOfUrl(novelA.attribs.href), // get last part of the link
|
||||
sourceId,
|
||||
novelCover, // TODO: IDK what to do about covers... On Syo they don't have them
|
||||
});
|
||||
});
|
||||
// return all novels from this page
|
||||
return pageNovels;
|
||||
};
|
||||
|
||||
novels = await getNovelsFromPage(page);
|
||||
/** Use
|
||||
* novels.push(...(await getNovelsFromPage(pageNumber)))
|
||||
* if you want to load more
|
||||
*/
|
||||
|
||||
// respond with novels!
|
||||
return { totalPages, novels };
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = getNovelUrl(novelUrl);
|
||||
|
||||
let chapters = [];
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
const cheerioQuery = cheerio.load(body, { decodeEntities: false });
|
||||
|
||||
// create novel object
|
||||
let novel = {
|
||||
sourceId,
|
||||
sourceName,
|
||||
url: url,
|
||||
novelUrl,
|
||||
novelName: cheerioQuery(".novel_title").text(),
|
||||
author: cheerioQuery(".novel_writername").text().replace("作者：", ""),
|
||||
novelCover,
|
||||
};
|
||||
|
||||
// Get all the chapters
|
||||
const cqGetChapters = cheerioQuery(".novel_sublist2");
|
||||
if (cqGetChapters.length !== 0) {
|
||||
// has more than 1 chapter
|
||||
novel.summary = cheerioQuery("#novel_ex")
|
||||
.text()
|
||||
.replace(/<\s*br.*?>/g, "\n");
|
||||
cqGetChapters.each(function (i, e) {
|
||||
const chapterA = cheerioQuery(this).find("a");
|
||||
const [chapterName, releaseDate, chapterUrl] = [
|
||||
// set the variables
|
||||
chapterA.text(),
|
||||
cheerioQuery(this)
|
||||
.find("dt") // get title
|
||||
.text() // get text
|
||||
.replace(/（.）/g, "") // remove "(edited)" mark
|
||||
.trim(), // trim spaces
|
||||
getLastPartOfUrl(chapterA.attr("href")),
|
||||
];
|
||||
chapters.push({ chapterName, releaseDate, chapterUrl });
|
||||
});
|
||||
} else {
|
||||
/**
|
||||
* Because there are oneshots on the site, they have to be treated with special care
|
||||
* that's what pisses me off in Shosetsu app. They have this extension,
|
||||
* but every oneshot is set as "there are no chapters" and all contents are thrown into the description!!
|
||||
*/
|
||||
// get summary for oneshot chapter
|
||||
|
||||
const result = await fetch(searchUrl() + `&word=${novel.novelName}`);
|
||||
const body = await result.text();
|
||||
const summaryQuery = cheerio.load(body, { decodeEntities: false });
|
||||
const foundText = summaryQuery(".searchkekka_box")
|
||||
.first()
|
||||
.find(".ex")
|
||||
.text()
|
||||
.replace(/\s{2,}/g, "\n");
|
||||
novel.summary = foundText;
|
||||
|
||||
// add single chapter
|
||||
chapters.push({
|
||||
chapterName: "Oneshot",
|
||||
releaseDate: cheerioQuery("head")
|
||||
.find("meta[name='WWWC']")
|
||||
.attr("content"), // get date from metadata
|
||||
chapterUrl: "oneshot", // set chapterUrl to oneshot so that chapterScraper knows it's a one-shot
|
||||
});
|
||||
}
|
||||
novel.chapters = chapters;
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
let parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = getChapterUrl(novelUrl, chapterUrl); // get Url
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
// create cheerioQuery
|
||||
const cheerioQuery = cheerio.load(body, {
|
||||
decodeEntities: false,
|
||||
});
|
||||
|
||||
let chapterText = cheerioQuery("#novel_honbun") // get chapter text
|
||||
.html();
|
||||
// create chapter data structure
|
||||
let chapter = {
|
||||
sourceId: 36,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName: "",
|
||||
chapterText,
|
||||
};
|
||||
|
||||
if (chapterUrl === "oneshot")
|
||||
// oneshot get name
|
||||
chapter.chapterName = cheerioQuery("#novel_title").text();
|
||||
else {
|
||||
// single chapter
|
||||
|
||||
// get name
|
||||
chapter.chapterName = cheerioQuery(".novel_subtitle").first().text();
|
||||
|
||||
// get next/prev buttons
|
||||
const chapterButtons = cheerioQuery(
|
||||
"#novel_contents .novel_bn"
|
||||
).first();
|
||||
}
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
let searchNovels = async (searchTerm) => {
|
||||
// const orderBy = req.query.o;
|
||||
|
||||
// array of all the novels
|
||||
let novels = [];
|
||||
|
||||
let isNext = true;
|
||||
|
||||
// returns list of novels from given page
|
||||
let getNovelsFromPage = async (pagenumber) => {
|
||||
// load page
|
||||
const result = await fetch(
|
||||
searchUrl(pagenumber || null, null) + `&word=${searchTerm}`
|
||||
);
|
||||
const body = await result.text();
|
||||
// Cheerio it!
|
||||
const cheerioQuery = cheerio.load(body, { decodeEntities: false });
|
||||
|
||||
if (cheerioQuery(".nextlink").length === 0) isNext = false;
|
||||
|
||||
let pageNovels = [];
|
||||
// find class=searchkekka_box
|
||||
cheerioQuery(".searchkekka_box").each(function (i, e) {
|
||||
// get div with link and name
|
||||
const novelDIV = cheerioQuery(this).find(".novel_h");
|
||||
// get link element
|
||||
const novelA = novelDIV.children()[0];
|
||||
// add new novel to array
|
||||
pageNovels.push({
|
||||
novelName: novelDIV.text(), // get the name
|
||||
novelUrl: getLastPartOfUrl(novelA.attribs.href), // get last part of the link
|
||||
sourceId,
|
||||
novelCover, // TODO: IDK what to do about covers... On Syo they don't have them
|
||||
});
|
||||
});
|
||||
// return all novels from this page
|
||||
return pageNovels;
|
||||
};
|
||||
|
||||
// counter of loaded pages
|
||||
// let pagesLoaded = 0;
|
||||
// do {
|
||||
// // always load first one
|
||||
// novels.push(...(await getNovelsFromPage(pagesLoaded + 1)));
|
||||
// pagesLoaded++;
|
||||
// } while (pagesLoaded < maxPageLoad && isNext); // check if we should load more
|
||||
|
||||
novels = await getNovelsFromPage(1);
|
||||
|
||||
/** Use
|
||||
* novels.push(...(await getNovelsFromPage(pageNumber)))
|
||||
* if you want to load more
|
||||
*/
|
||||
|
||||
// respond with novels!
|
||||
return novels;
|
||||
};
|
||||
|
||||
const SyosetuScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default SyosetuScraper;
|
||||
@@ -1,188 +0,0 @@
|
||||
import MadaraScraper from "./MadaraScraper";
|
||||
|
||||
const getPath = (extensionId) => {
|
||||
const path = {
|
||||
38: { novels: "manga", novel: "manga", chapter: "manga" },
|
||||
39: { novels: "series", novel: "series", chapter: "series" },
|
||||
40: { novels: "novels", novel: "novel", chapter: "novel" },
|
||||
41: { novels: "manga", novel: "manga", chapter: "manga" },
|
||||
42: { novels: "all-novels", novel: "manga", chapter: "manga" },
|
||||
43: { novels: "series", novel: "series", chapter: "series" },
|
||||
44: { novels: "series", novel: "series", chapter: "series" },
|
||||
45: { novels: "novel", novel: "novel", chapter: "novel" },
|
||||
46: { novels: "novel", novel: "novel", chapter: "novel" },
|
||||
47: { novels: "novel-list", novel: "novel", chapter: "novel" },
|
||||
55: { novels: "manga", novel: "manga", chapter: "manga" },
|
||||
56: { novels: "novel", novel: "novel", chapter: "novel" },
|
||||
57: { novels: "series", novel: "series", chapter: "series" },
|
||||
58: { novels: "novel", novel: "novel", chapter: "novel" },
|
||||
59: { novels: "novel", novel: "novels", chapter: "novel" },
|
||||
60: { novels: "novel", novel: "novel", chapter: "novel" },
|
||||
61: { novels: "novel", novel: "novel", chapter: "novel" },
|
||||
62: { novels: "novel", novel: "novel", chapter: "novel" },
|
||||
63: { novels: "novel", novel: "novel", chapter: "novel" },
|
||||
75: { novels: "series", novel: "series", chapter: "series" },
|
||||
};
|
||||
|
||||
return (
|
||||
path[extensionId] ?? {
|
||||
novels: "novel",
|
||||
novel: "novel",
|
||||
chapter: "novel",
|
||||
}
|
||||
);
|
||||
};
|
||||
|
||||
export const SkyNovelScraper = new MadaraScraper(
|
||||
38,
|
||||
"https://skynovel.org/",
|
||||
"SkyNovel",
|
||||
getPath(38)
|
||||
);
|
||||
|
||||
export const NovelCakeScraper = new MadaraScraper(
|
||||
39,
|
||||
"https://novelcake.com/",
|
||||
"NovelCake",
|
||||
getPath(39)
|
||||
);
|
||||
|
||||
export const NovelsRockScraper = new MadaraScraper(
|
||||
40,
|
||||
"https://novelsrock.com/",
|
||||
"NovelsRock",
|
||||
getPath(40)
|
||||
);
|
||||
|
||||
export const ZinnNovelScraper = new MadaraScraper(
|
||||
41,
|
||||
"https://zinnovel.com/",
|
||||
"ZinnNovel",
|
||||
getPath(41)
|
||||
);
|
||||
|
||||
export const NovelTranslateScraper = new MadaraScraper(
|
||||
42,
|
||||
"https://noveltranslate.com/",
|
||||
"NovelTranslate",
|
||||
getPath(42)
|
||||
);
|
||||
|
||||
export const LunarLettersScraper = new MadaraScraper(
|
||||
43,
|
||||
"https://www.lunarletters.com/",
|
||||
"LunarLetters",
|
||||
getPath(43)
|
||||
);
|
||||
|
||||
export const SleepyTranslationsScraper = new MadaraScraper(
|
||||
44,
|
||||
"https://sleepytranslations.com/",
|
||||
"SleepyTranslations",
|
||||
getPath(44)
|
||||
);
|
||||
|
||||
export const FreeNovelScraper = new MadaraScraper(
|
||||
45,
|
||||
"https://freenovel.me/",
|
||||
"FreeNovelMe",
|
||||
getPath(45)
|
||||
);
|
||||
|
||||
export const FirstKissNovelScraper = new MadaraScraper(
|
||||
46,
|
||||
"https://1stkissnovel.love/",
|
||||
"1stKissNovel",
|
||||
getPath(46)
|
||||
);
|
||||
|
||||
export const DaoNovelScraper = new MadaraScraper(
|
||||
47,
|
||||
"https://daonovel.com/",
|
||||
"DaoNovel",
|
||||
getPath(47)
|
||||
);
|
||||
|
||||
export const MostNovelScraper = new MadaraScraper(
|
||||
55,
|
||||
"https://mostnovel.com/",
|
||||
"MostNovel",
|
||||
getPath(55)
|
||||
);
|
||||
|
||||
export const NovelMultiverseScraper = new MadaraScraper(
|
||||
56,
|
||||
"https://www.novelmultiverse.com/",
|
||||
"NovelMultiverse",
|
||||
getPath(56)
|
||||
);
|
||||
|
||||
export const LightNovelHeavenScraper = new MadaraScraper(
|
||||
57,
|
||||
"https://lightnovelheaven.com/",
|
||||
"LightNovelHeaven",
|
||||
getPath(57)
|
||||
);
|
||||
|
||||
export const LightNovelsHubScraper = new MadaraScraper(
|
||||
58,
|
||||
"https://lightnovelshub.com/",
|
||||
"LightNovelsHub",
|
||||
getPath(58)
|
||||
);
|
||||
|
||||
export const MeionNovelScraper = new MadaraScraper(
|
||||
60,
|
||||
"https://meionovel.id/",
|
||||
"MeionNovel",
|
||||
getPath(60)
|
||||
);
|
||||
|
||||
export const WebNovelLoverScraper = new MadaraScraper(
|
||||
61,
|
||||
"https://www.webnovelover.com/",
|
||||
"WebNovelLover",
|
||||
getPath(61)
|
||||
);
|
||||
|
||||
export const BoxNovelOnlineScraper = new MadaraScraper(
|
||||
63,
|
||||
"https://boxnovel.online/",
|
||||
"BoxNovel.online",
|
||||
getPath(63)
|
||||
);
|
||||
|
||||
export const ClickNovelScraper = new MadaraScraper(
|
||||
64,
|
||||
"https://clicknovel.net/",
|
||||
"ClickNovel",
|
||||
getPath(64)
|
||||
);
|
||||
|
||||
export const ReadWebNovelsScraper = new MadaraScraper(
|
||||
65,
|
||||
"https://readwebnovels.net/",
|
||||
"ReadWebNovels",
|
||||
getPath(65)
|
||||
);
|
||||
|
||||
export const WBNovelScraper = new MadaraScraper(
|
||||
66,
|
||||
"https://wbnovel.com/",
|
||||
"WBNovel",
|
||||
getPath(66)
|
||||
);
|
||||
|
||||
export const WuxiaWorldDotSiteScraper = new MadaraScraper(
|
||||
74,
|
||||
"https://wuxiaworld.site/",
|
||||
"WuxiaWorld.Site",
|
||||
getPath(74)
|
||||
);
|
||||
|
||||
export const MysticalSeriesScraper = new MadaraScraper(
|
||||
75,
|
||||
"https://mysticalmerries.com/",
|
||||
"MysticalSeries",
|
||||
getPath(75)
|
||||
);
|
||||
@@ -1,224 +0,0 @@
|
||||
import moment from "moment";
|
||||
import cheerio from "react-native-cheerio";
|
||||
import { Status } from "../../helpers/constants";
|
||||
import { parseMadaraDate } from "../../helpers/parseDate";
|
||||
|
||||
class MadaraScraper {
|
||||
constructor(sourceId, baseUrl, sourceName, path) {
|
||||
this.sourceId = sourceId;
|
||||
this.baseUrl = baseUrl;
|
||||
this.sourceName = sourceName;
|
||||
this.path = path;
|
||||
}
|
||||
|
||||
async popularNovels(page) {
|
||||
const totalPages = 100;
|
||||
let url =
|
||||
this.baseUrl +
|
||||
this.path.novels +
|
||||
"/page/" +
|
||||
page +
|
||||
"/?m_orderby=rating";
|
||||
let sourceId = this.sourceId;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
const $ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".manga-title-badges").remove();
|
||||
|
||||
$(".page-item-detail").each(function () {
|
||||
const novelName = $(this).find(".post-title").text().trim();
|
||||
let image = $(this).find("img");
|
||||
const novelCover = image.attr("data-src") || image.attr("src");
|
||||
|
||||
let novelUrl = $(this)
|
||||
.find(".post-title")
|
||||
.find("a")
|
||||
.attr("href")
|
||||
.split("/")[4];
|
||||
|
||||
const novel = {
|
||||
sourceId,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return { totalPages, novels };
|
||||
}
|
||||
|
||||
async parseNovelAndChapters(novelUrl) {
|
||||
const url = `${this.baseUrl}${this.path.novel}/${novelUrl}/`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
let $ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.sourceId = this.sourceId;
|
||||
|
||||
novel.sourceName = this.sourceName;
|
||||
|
||||
novel.url = url;
|
||||
|
||||
novel.novelUrl = novelUrl;
|
||||
|
||||
$(".manga-title-badges").remove();
|
||||
|
||||
novel.novelName = $(".post-title > h1").text().trim();
|
||||
|
||||
novel.novelCover =
|
||||
$(".summary_image > a > img").attr("data-src") ||
|
||||
$(".summary_image > a > img").attr("src") ||
|
||||
"https://github.com/LNReader/lnreader-sources/blob/main/src/coverNotAvailable.jpg?raw=true";
|
||||
|
||||
$(".post-content_item").each(function () {
|
||||
const detailName = $(this)
|
||||
.find(".summary-heading > h5")
|
||||
.text()
|
||||
.trim();
|
||||
const detail = $(this).find(".summary-content").text().trim();
|
||||
|
||||
switch (detailName) {
|
||||
case "Genre(s)":
|
||||
novel.genre = detail.replace(/[\t\n]/g, ",");
|
||||
break;
|
||||
case "Author(s)":
|
||||
novel.author = detail;
|
||||
break;
|
||||
case "Status":
|
||||
novel.status = detail.includes("OnGoing")
|
||||
? Status.ONGOING
|
||||
: Status.COMPLETED;
|
||||
break;
|
||||
}
|
||||
});
|
||||
|
||||
novel.summary = $("div.summary__content").text().trim();
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
const novelId =
|
||||
$(".rating-post-id").attr("value") ||
|
||||
$("#manga-chapters-holder").attr("data-id");
|
||||
|
||||
let formData = new FormData();
|
||||
formData.append("action", "manga_get_chapters");
|
||||
formData.append("manga", novelId);
|
||||
|
||||
const data = await fetch(this.baseUrl + "wp-admin/admin-ajax.php", {
|
||||
method: "POST",
|
||||
body: formData,
|
||||
});
|
||||
const text = await data.text();
|
||||
|
||||
if (text !== "0") {
|
||||
$ = cheerio.load(text);
|
||||
}
|
||||
|
||||
$(".wp-manga-chapter").each(function () {
|
||||
const chapterName = $(this)
|
||||
.find("a")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
let releaseDate = null;
|
||||
releaseDate = $(this)
|
||||
.find("span.chapter-release-date")
|
||||
.text()
|
||||
.trim();
|
||||
|
||||
if (releaseDate) {
|
||||
releaseDate = parseMadaraDate(releaseDate);
|
||||
} else {
|
||||
/**
|
||||
* Insert current date
|
||||
*/
|
||||
|
||||
releaseDate = moment().format("MMMM DD, YYYY");
|
||||
}
|
||||
|
||||
const chapterUrl = $(this).find("a").attr("href").split("/")[5];
|
||||
|
||||
novelChapters.push({ chapterName, releaseDate, chapterUrl });
|
||||
});
|
||||
|
||||
novel.chapters = novelChapters.reverse();
|
||||
|
||||
return novel;
|
||||
}
|
||||
|
||||
async parseChapter(novelUrl, chapterUrl) {
|
||||
let sourceId = this.sourceId;
|
||||
|
||||
const url = `${this.baseUrl}${this.path.chapter}/${novelUrl}/${chapterUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
const $ = cheerio.load(body);
|
||||
|
||||
let chapterName =
|
||||
$(".text-center").text() || $("#chapter-heading").text();
|
||||
|
||||
let chapterText = $(".text-left").html();
|
||||
|
||||
const chapter = {
|
||||
sourceId,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
}
|
||||
|
||||
async searchNovels(searchTerm) {
|
||||
const url = `${this.baseUrl}?s=${searchTerm}&post_type=wp-manga`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
const $ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
let sourceId = this.sourceId;
|
||||
|
||||
$(".c-tabs-item__content").each(function () {
|
||||
const novelName = $(this).find(".post-title").text().trim();
|
||||
|
||||
let image = $(this).find("img");
|
||||
const novelCover = image.attr("data-src") || image.attr("src");
|
||||
|
||||
let novelUrl = $(this)
|
||||
.find(".post-title")
|
||||
.find("a")
|
||||
.attr("href")
|
||||
.split("/")[4];
|
||||
|
||||
const novel = {
|
||||
sourceId,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
}
|
||||
}
|
||||
|
||||
export default MadaraScraper;
|
||||
@@ -1,13 +0,0 @@
|
||||
import WPMangaStreamScraper from "./WPMangaStreamScraper";
|
||||
|
||||
export const KolNovelScraper = new WPMangaStreamScraper(
|
||||
53,
|
||||
"https://kolnovel.com/",
|
||||
"KolNovel"
|
||||
);
|
||||
|
||||
export const RewayatArScraper = new WPMangaStreamScraper(
|
||||
54,
|
||||
"https://rewayat-ar.site/",
|
||||
"RewayatAr"
|
||||
);
|
||||
@@ -1,162 +0,0 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
|
||||
class WPMangaStreamScraper {
|
||||
constructor(sourceId, baseUrl, sourceName) {
|
||||
this.sourceId = sourceId;
|
||||
this.baseUrl = baseUrl;
|
||||
this.sourceName = sourceName;
|
||||
}
|
||||
|
||||
async popularNovels(page) {
|
||||
let totalPages = 100;
|
||||
let url =
|
||||
this.baseUrl + "series/?page=" + page + "&status=&order=popular";
|
||||
let sourceId = this.sourceId;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
const $ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$("article.bs").each(function () {
|
||||
const novelName = $(this).find(".ntitle").text().trim();
|
||||
let image = $(this).find("img");
|
||||
const novelCover = image.attr("src");
|
||||
|
||||
let novelUrl = $(this).find("a").attr("href").split("/")[4];
|
||||
|
||||
const novel = {
|
||||
sourceId,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return { totalPages, novels };
|
||||
}
|
||||
|
||||
async parseNovelAndChapters(novelUrl) {
|
||||
const url = `${this.baseUrl}series/${novelUrl}/`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
let $ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.sourceId = this.sourceId;
|
||||
|
||||
novel.sourceName = this.sourceName;
|
||||
|
||||
novel.url = url;
|
||||
|
||||
novel.novelUrl = novelUrl;
|
||||
|
||||
novel.novelName = $(".entry-title").text();
|
||||
|
||||
novel.novelCover = $("img.wp-post-image").attr("src");
|
||||
|
||||
$("div.spe > span").each(function () {
|
||||
const detailName = $(this).find("b").text().trim();
|
||||
const detail = $(this).find("b").next().text().trim();
|
||||
|
||||
if ($(this).text().includes("الحالة:")) {
|
||||
novel.status = $(this).text().replace("الحالة: ", "");
|
||||
}
|
||||
|
||||
switch (detailName) {
|
||||
case "المؤلف:":
|
||||
novel.auhtor = detail;
|
||||
break;
|
||||
}
|
||||
});
|
||||
|
||||
novel.genre = $(".genxed").text().replace(/\s/g, ",");
|
||||
|
||||
novel.summary = $('div[itemprop="description"]').text().trim();
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
$(".eplister")
|
||||
.find("li")
|
||||
.each(function () {
|
||||
const chapterName =
|
||||
$(this).find(".epl-num").text() +
|
||||
" - " +
|
||||
$(this).find(".epl-title").text();
|
||||
|
||||
const releaseDate = $(this).find(".epl-date").text().trim();
|
||||
|
||||
const chapterUrl = $(this).find("a").attr("href").split("/")[3];
|
||||
|
||||
novelChapters.push({ chapterName, releaseDate, chapterUrl });
|
||||
});
|
||||
|
||||
novel.chapters = novelChapters.reverse();
|
||||
|
||||
return novel;
|
||||
}
|
||||
|
||||
async parseChapter(novelUrl, chapterUrl) {
|
||||
let sourceId = this.sourceId;
|
||||
|
||||
const url = `${this.baseUrl}${chapterUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
const $ = cheerio.load(body);
|
||||
|
||||
let chapterName = $(".entry-title").text();
|
||||
let chapterText = $("div.epcontent").html();
|
||||
|
||||
const chapter = {
|
||||
sourceId,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
}
|
||||
|
||||
async searchNovels(searchTerm) {
|
||||
const url = `${this.baseUrl}?s=${searchTerm}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
const $ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
let sourceId = this.sourceId;
|
||||
|
||||
$("article.bs").each(function () {
|
||||
const novelName = $(this).find(".ntitle").text().trim();
|
||||
let image = $(this).find("img");
|
||||
const novelCover = image.attr("src");
|
||||
|
||||
let novelUrl = $(this).find("a").attr("href").split("/")[4];
|
||||
|
||||
const novel = {
|
||||
sourceId,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = WPMangaStreamScraper;
|
||||
@@ -1,152 +0,0 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
import { Status } from "../helpers/constants";
|
||||
|
||||
const sourceId = 69;
|
||||
|
||||
const sourceName = "RanobeHub";
|
||||
|
||||
const baseUrl = "https://ranobehub.org/";
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
let url =
|
||||
baseUrl + "api/search?page=" + page + "&sort=computed_rating&take=40";
|
||||
const totalPages = 69;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.json();
|
||||
|
||||
let novels = [];
|
||||
|
||||
body.resource.forEach((novel) => {
|
||||
const novelName = novel.names.rus;
|
||||
const novelCover = novel.poster.medium;
|
||||
const novelUrl = novel.url.split("/").pop();
|
||||
|
||||
novels.push({
|
||||
sourceId,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
});
|
||||
});
|
||||
|
||||
return { novels, totalPages };
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = baseUrl + "ranobe/" + novelUrl;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = { sourceId, sourceName, url, novelUrl };
|
||||
|
||||
novel.novelName = $("h1").text();
|
||||
|
||||
novel.novelCover = $("img.__posterbox ").attr("data-src");
|
||||
|
||||
novel.summary = $(".book-description__text").text().trim();
|
||||
|
||||
novel.genre = "";
|
||||
|
||||
novel.status = Status.UNKNOWN;
|
||||
|
||||
novel.author = $(".book-author").text().trim();
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
const novelId = novelUrl.split("-")[0];
|
||||
const fetchChaptersUrl = `https://ranobehub.org/api/ranobe/${novelId}/contents`;
|
||||
|
||||
const chaptersRaw = await fetch(fetchChaptersUrl);
|
||||
const chaptersJSON = await chaptersRaw.json();
|
||||
|
||||
chaptersJSON.volumes.map((volume) =>
|
||||
volume.chapters.map((chapter) =>
|
||||
novelChapters.push({
|
||||
chapterName: chapter.name,
|
||||
chapterUrl: chapter.url,
|
||||
releaseDate: null,
|
||||
})
|
||||
)
|
||||
);
|
||||
|
||||
novel.chapters = novelChapters;
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = chapterUrl;
|
||||
const chapterId = chapterUrl.split("/")[4];
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let chapterName = $(
|
||||
"body > div.pusher.container_main > div:nth-child(5) > div:nth-child(1) > div.title-wrapper > h1"
|
||||
).text();
|
||||
let chapterText = $(
|
||||
"body > div.pusher.container_main > div:nth-child(5) > div:nth-child(1)"
|
||||
).html();
|
||||
|
||||
if (chapterText) {
|
||||
/**
|
||||
* Convert relative url to absolute
|
||||
*/
|
||||
|
||||
chapterText = chapterText
|
||||
.replace(/<\s*script[^>]*>[\s\S]*?<\/script>/gim, "")
|
||||
.replace(
|
||||
/data-src="(.*)"/g,
|
||||
`src="https://ranobehub.org/img/ranobe/content/${chapterId}/$1.jpg"`
|
||||
);
|
||||
}
|
||||
|
||||
console.log(chapterText);
|
||||
|
||||
const chapter = {
|
||||
sourceId,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const url = "https://ranobehub.org/api/fulltext/ranobe?query=" + searchTerm;
|
||||
|
||||
const result = await fetch(url);
|
||||
const data = await result.json();
|
||||
|
||||
let novels = [];
|
||||
|
||||
data.data.map((novel) =>
|
||||
novels.push({
|
||||
sourceId,
|
||||
novelName: novel.names.rus,
|
||||
novelUrl: novel.url.match(
|
||||
/https:\/\/ranobehub\.org\/ranobe\/(.*?)\?utm_source=search_name&utm_medium=search&utm_campaign=search_using/
|
||||
)[1],
|
||||
novelCover: novel.image,
|
||||
})
|
||||
);
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const RanobeHubScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default RanobeHubScraper;
|
||||
-165
@@ -1,165 +0,0 @@
|
||||
import ComradeMaoScraper from "./en/comrademao";
|
||||
import ReadLightNovelScraper from "./en/readlightnovel";
|
||||
import BoxNovelScraper from "./en/boxnovel";
|
||||
import fastNovelScraper from "./en/fastnovel";
|
||||
import readNovelFullScraper from "./en/readnovelfull";
|
||||
import mtlNovelScraper from "./en/mtlnovel";
|
||||
import novelhallScraper from "./en/novelhall";
|
||||
import WuxiaWorldScraper from "./en/wuxiaworld";
|
||||
import novelFullScraper from "./en/novelfull";
|
||||
import novelTrenchScraper from "./en/noveltrench";
|
||||
import vipNovelScraper from "./en/vipnovel";
|
||||
import kissLightNovelScraper from "./en/kisslightnovels";
|
||||
import WuxiaWorldSiteScraper from "./en/wuxiaworldsite";
|
||||
import FreeWebNovelScraper from "./en/freewebnovel";
|
||||
import JPMTLScraper from "./en/jpmtl";
|
||||
import lightNovelPubScraper from "./en/lightnovelpub";
|
||||
import WuxiaWorldCoScraper from "./en/wuxiaworldco";
|
||||
import novelUpdatesCcScraper from "./en/novelupdatescc";
|
||||
import readLightNovelCcScraper from "./en/readlightnovelcc";
|
||||
import tapReadScraper from "./en/tapread";
|
||||
import WuxiaWorldCloudScraper from "./en/wuxiaworldcloud";
|
||||
import WoopReadScraper from "./en/woopread";
|
||||
import FoxaHolicScraper from "./en/foxaholic";
|
||||
import EinharjarProjectScraper from "./es/einherjarproject";
|
||||
import TuNovelaLigeraScraper from "./es/tunovelaligera";
|
||||
import SkyNovelsScraper from "./es/skynovels";
|
||||
import NovelasLigeraScraper from "./es/novelasligera";
|
||||
import YuukiTlsScraper from "./es/yuukitls";
|
||||
import NovelaWuxiaScraper from "./es/novelawuxia";
|
||||
import OasisTranslationsScraper from "./es/oasistranslations";
|
||||
import HasuTlScraper from "./es/hasutl";
|
||||
import NovelPassionScraper from "./en/novelpassion";
|
||||
import RoyalRoadScraper from "./en/royalroad";
|
||||
import ScribbleHubScraper from "./en/scribblehub";
|
||||
import SyosetuScraper from "./jp/syosetu";
|
||||
import LNMTLScraper from "./en/lnmtl";
|
||||
import {
|
||||
BoxNovelOnlineScraper,
|
||||
ClickNovelScraper,
|
||||
DaoNovelScraper,
|
||||
FirstKissNovelScraper,
|
||||
FreeNovelScraper,
|
||||
LightNovelHeavenScraper,
|
||||
LightNovelsHubScraper,
|
||||
LunarLettersScraper,
|
||||
MeionNovelScraper,
|
||||
MostNovelScraper,
|
||||
MysticalSeriesScraper,
|
||||
NovelCakeScraper,
|
||||
NovelMultiverseScraper,
|
||||
NovelsRockScraper,
|
||||
NovelTranslateScraper,
|
||||
ReadWebNovelsScraper,
|
||||
SkyNovelScraper,
|
||||
SleepyTranslationsScraper,
|
||||
WBNovelScraper,
|
||||
WebNovelLoverScraper,
|
||||
WuxiaWorldDotSiteScraper,
|
||||
ZinnNovelScraper,
|
||||
} from "./multisrc/madara/MadaraGenerator";
|
||||
import WuxiaBlogScraper from "./en/wuxiablog";
|
||||
import WuxiaCityScraper from "./en/wuxiacity";
|
||||
import NovelUpdatesScraper from "./en/novelupdates";
|
||||
import RanobesScraper from "./en/ranobes";
|
||||
import YushuboScraper from "./ch/yushubo";
|
||||
import {
|
||||
KolNovelScraper,
|
||||
RewayatArScraper,
|
||||
} from "./multisrc/wpmangastream/WPMangaStreamGenerator";
|
||||
import WLNUpdatesScraper from "./en/wlnupdates";
|
||||
import ReaperScansScraper from "./en/reaperscans";
|
||||
import ReadwnScraper from "./en/readwn";
|
||||
import RanobeHubScraper from "./ru/ranobehub";
|
||||
import ArNovelScraper from "./ar/arnovel";
|
||||
import DivineDaoLibraryScraper from "./en/divinedaolibrary";
|
||||
import NovelOnlineFullScraper from "./en/novelonlinefull";
|
||||
import LightNovelUpdatesScraper from "./en/lightnovelupdates";
|
||||
import EpikNovelScraper from "./tr/epiknovel";
|
||||
import TravisTranslationsScraper from "./en/travistranslations";
|
||||
import NovelDeGlaceScraper from "./fr/noveldeglace";
|
||||
import RainOfSnowScraper from "./en/rainofsnow";
|
||||
|
||||
export const getSource = (sourceId) => {
|
||||
const scrapers = {
|
||||
1: BoxNovelScraper,
|
||||
2: ReadLightNovelScraper,
|
||||
3: fastNovelScraper,
|
||||
4: readNovelFullScraper,
|
||||
5: mtlNovelScraper,
|
||||
6: novelhallScraper,
|
||||
7: WuxiaWorldScraper,
|
||||
8: novelFullScraper,
|
||||
9: novelTrenchScraper,
|
||||
10: vipNovelScraper,
|
||||
11: kissLightNovelScraper,
|
||||
12: WuxiaWorldSiteScraper,
|
||||
13: FreeWebNovelScraper,
|
||||
14: JPMTLScraper,
|
||||
15: lightNovelPubScraper,
|
||||
16: WuxiaWorldCoScraper,
|
||||
17: tapReadScraper,
|
||||
18: novelUpdatesCcScraper,
|
||||
19: readLightNovelCcScraper,
|
||||
20: WuxiaWorldCloudScraper,
|
||||
21: WoopReadScraper,
|
||||
22: FoxaHolicScraper,
|
||||
23: TuNovelaLigeraScraper,
|
||||
24: SkyNovelsScraper,
|
||||
25: EinharjarProjectScraper,
|
||||
26: NovelasLigeraScraper,
|
||||
27: ComradeMaoScraper,
|
||||
28: YuukiTlsScraper,
|
||||
29: HasuTlScraper,
|
||||
30: OasisTranslationsScraper,
|
||||
31: NovelaWuxiaScraper,
|
||||
33: NovelPassionScraper,
|
||||
34: RoyalRoadScraper,
|
||||
35: ScribbleHubScraper,
|
||||
36: SyosetuScraper,
|
||||
37: LNMTLScraper,
|
||||
38: SkyNovelScraper,
|
||||
39: NovelCakeScraper,
|
||||
40: NovelsRockScraper,
|
||||
41: ZinnNovelScraper,
|
||||
42: NovelTranslateScraper,
|
||||
43: LunarLettersScraper,
|
||||
44: SleepyTranslationsScraper,
|
||||
45: FreeNovelScraper,
|
||||
46: FirstKissNovelScraper,
|
||||
47: DaoNovelScraper,
|
||||
48: WuxiaBlogScraper,
|
||||
49: WuxiaCityScraper,
|
||||
50: NovelUpdatesScraper,
|
||||
51: RanobesScraper,
|
||||
52: YushuboScraper,
|
||||
53: KolNovelScraper,
|
||||
54: RewayatArScraper,
|
||||
55: MostNovelScraper,
|
||||
56: NovelMultiverseScraper,
|
||||
57: LightNovelHeavenScraper,
|
||||
58: LightNovelsHubScraper,
|
||||
59: ArNovelScraper,
|
||||
60: MeionNovelScraper,
|
||||
61: WebNovelLoverScraper,
|
||||
62: WLNUpdatesScraper,
|
||||
63: BoxNovelOnlineScraper,
|
||||
64: ClickNovelScraper,
|
||||
65: ReadWebNovelsScraper,
|
||||
66: WBNovelScraper,
|
||||
67: ReaperScansScraper,
|
||||
68: ReadwnScraper,
|
||||
69: RanobeHubScraper,
|
||||
70: DivineDaoLibraryScraper,
|
||||
71: NovelOnlineFullScraper,
|
||||
72: LightNovelUpdatesScraper,
|
||||
73: EpikNovelScraper,
|
||||
74: WuxiaWorldDotSiteScraper,
|
||||
75: MysticalSeriesScraper,
|
||||
76: TravisTranslationsScraper,
|
||||
77: NovelDeGlaceScraper,
|
||||
78: RainOfSnowScraper,
|
||||
};
|
||||
|
||||
return scrapers[sourceId];
|
||||
};
|
||||
@@ -1,541 +0,0 @@
|
||||
[
|
||||
{
|
||||
"sourceId": 1,
|
||||
"url": "https://boxnovel.com/",
|
||||
"sourceName": "BoxNovel",
|
||||
"icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/en/boxnovel/icon.png?raw=true",
|
||||
"lang": "English"
|
||||
},
|
||||
{
|
||||
"sourceId": 2,
|
||||
"sourceName": "ReadLightNovel",
|
||||
"url": "https://www.readlightnovel.org",
|
||||
"lang": "English",
|
||||
"icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/en/readlightnovel/icon.png?raw=true"
|
||||
},
|
||||
{
|
||||
"sourceId": 3,
|
||||
"url": "https://fastnovel.net/",
|
||||
"sourceName": "FastNovel",
|
||||
"icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/en/fastnovel/icon.png?raw=true",
|
||||
"lang": "English"
|
||||
},
|
||||
{
|
||||
"sourceId": 4,
|
||||
"url": "https://readnovelfull.com/",
|
||||
"sourceName": "ReadNovelFull",
|
||||
"icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/en/readnovelfull/icon.png?raw=true",
|
||||
"lang": "English"
|
||||
},
|
||||
{
|
||||
"sourceId": 5,
|
||||
"url": "https://www.mtlnovel.com/",
|
||||
"sourceName": "MTLNovel",
|
||||
"icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/en/mtlnovel/icon.png?raw=true",
|
||||
"lang": "English"
|
||||
},
|
||||
{
|
||||
"sourceId": 6,
|
||||
"url": "https://www.novelhall.com/",
|
||||
"sourceName": "NovelHall",
|
||||
"icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/en/novelhall/icon.png?raw=true",
|
||||
"lang": "English"
|
||||
},
|
||||
{
|
||||
"sourceId": 7,
|
||||
"url": "https://www.wuxiaworld.com/",
|
||||
"sourceName": "WuxiaWorld",
|
||||
"icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/en/wuxiaworld/icon.png?raw=true",
|
||||
"lang": "English"
|
||||
},
|
||||
{
|
||||
"sourceId": 8,
|
||||
"url": "https://novelfull.com/",
|
||||
"sourceName": "NovelFull",
|
||||
"icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/en/novelfull/icon.png?raw=true",
|
||||
"lang": "English"
|
||||
},
|
||||
{
|
||||
"sourceId": 9,
|
||||
"url": "https://noveltrench.com/",
|
||||
"sourceName": "NovelTrench",
|
||||
"icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/en/noveltrench/icon.png?raw=true",
|
||||
"lang": "English"
|
||||
},
|
||||
{
|
||||
"sourceId": 10,
|
||||
"url": "https://vipnovel.com/",
|
||||
"sourceName": "VipNovel",
|
||||
"icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/en/vipnovel/icon.png?raw=true",
|
||||
"lang": "English"
|
||||
},
|
||||
{
|
||||
"sourceId": 11,
|
||||
"url": "https://kisslightnovels.info/",
|
||||
"sourceName": "KissLightNovels",
|
||||
"icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/en/kisslightnovels/icon.png?raw=true",
|
||||
"lang": "English"
|
||||
},
|
||||
{
|
||||
"sourceId": 12,
|
||||
"url": "https://wuxiaworldsite.co/",
|
||||
"sourceName": "WuxiaWorldSite",
|
||||
"icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/en/wuxiaworldsite/icon.png?raw=true",
|
||||
"lang": "English"
|
||||
},
|
||||
{
|
||||
"sourceId": 13,
|
||||
"url": "https://freewebnovel.com/",
|
||||
"sourceName": "FreeWebNovel",
|
||||
"icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/en/freewebnovel/icon.png?raw=true",
|
||||
"lang": "English"
|
||||
},
|
||||
{
|
||||
"sourceId": 14,
|
||||
"url": "https://jpmtl.com/",
|
||||
"sourceName": "JPMTL",
|
||||
"icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/en/jpmtl/icon.png?raw=true",
|
||||
"lang": "English"
|
||||
},
|
||||
{
|
||||
"sourceId": 15,
|
||||
"url": "https://www.lightnovelpub.com/",
|
||||
"sourceName": "LightNovelPub",
|
||||
"icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/en/lightnovelpub/icon.png?raw=true",
|
||||
"lang": "English"
|
||||
},
|
||||
{
|
||||
"sourceId": 16,
|
||||
"url": "https://www.wuxiaworld.co/",
|
||||
"sourceName": "WuxiaWorld.co",
|
||||
"icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/en/wuxiaworldco/icon.png?raw=true",
|
||||
"lang": "English"
|
||||
},
|
||||
{
|
||||
"sourceId": 17,
|
||||
"url": "http://www.tapread.com/",
|
||||
"sourceName": "TapRead",
|
||||
"icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/en/tapread/icon.png?raw=true",
|
||||
"lang": "English"
|
||||
},
|
||||
{
|
||||
"sourceId": 18,
|
||||
"url": "https://www.novelupdates.cc/",
|
||||
"sourceName": "NovelUpdates.cc",
|
||||
"icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/en/novelupdatescc/icon.png?raw=true",
|
||||
"lang": "English"
|
||||
},
|
||||
{
|
||||
"sourceId": 19,
|
||||
"url": "https://m.readlightnovel.cc/",
|
||||
"sourceName": "ReadLightNovel.cc",
|
||||
"icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/en/readlightnovelcc/icon.png?raw=true",
|
||||
"lang": "English"
|
||||
},
|
||||
{
|
||||
"sourceId": 20,
|
||||
"url": "http://wuxiaworld.cloud/",
|
||||
"sourceName": "WuxiaWorld.cloud",
|
||||
"icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/en/wuxiaworldcloud/icon.png?raw=true",
|
||||
"lang": "English"
|
||||
},
|
||||
{
|
||||
"sourceId": 21,
|
||||
"url": "https://woopread.com/",
|
||||
"sourceName": "WoopRead",
|
||||
"icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/en/woopread/icon.png?raw=true",
|
||||
"lang": "English"
|
||||
},
|
||||
{
|
||||
"sourceId": 22,
|
||||
"url": "https://foxaholic.com/",
|
||||
"sourceName": "Foxaholic",
|
||||
"icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/en/foxaholic/icon.png?raw=true",
|
||||
"lang": "English"
|
||||
},
|
||||
{
|
||||
"sourceId": 23,
|
||||
"url": "https://tunovelaligera.com/",
|
||||
"sourceName": "TuNovelaLigera",
|
||||
"icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/es/tunovelaligera/icon.png?raw=true",
|
||||
"lang": "Spanish"
|
||||
},
|
||||
{
|
||||
"sourceId": 24,
|
||||
"url": "https://www.skynovels.net/",
|
||||
"sourceName": "SkyNovels",
|
||||
"icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/es/skynovels/icon.png?raw=true",
|
||||
"lang": "Spanish"
|
||||
},
|
||||
{
|
||||
"sourceId": 25,
|
||||
"url": "https://einherjarproject.net/",
|
||||
"sourceName": "EinherjarProject",
|
||||
"icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/es/einherjarproject/icon.png?raw=true",
|
||||
"lang": "Spanish"
|
||||
},
|
||||
{
|
||||
"sourceId": 26,
|
||||
"url": "https://novelasligera.com/",
|
||||
"sourceName": "NovelasLigera",
|
||||
"icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/es/novelasligera/icon.png?raw=true",
|
||||
"lang": "Spanish"
|
||||
},
|
||||
{
|
||||
"sourceId": 27,
|
||||
"url": "https://comrademao.com/",
|
||||
"sourceName": "ComradeMao",
|
||||
"icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/en/comrademao/icon.png?raw=true",
|
||||
"lang": "English"
|
||||
},
|
||||
{
|
||||
"sourceId": 28,
|
||||
"url": "https://yuukitls.com/",
|
||||
"sourceName": "YuukiTls",
|
||||
"icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/es/yuukitls/icon.png?raw=true",
|
||||
"lang": "Spanish"
|
||||
},
|
||||
{
|
||||
"sourceId": 29,
|
||||
"url": "https://hasutl.wordpress.com/",
|
||||
"sourceName": "Hasu Translations",
|
||||
"icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/es/hasutl/icon.png?raw=true",
|
||||
"lang": "Spanish"
|
||||
},
|
||||
{
|
||||
"sourceId": 30,
|
||||
"url": "https://oasistranslations.wordpress.com/",
|
||||
"sourceName": "Oasis Translations",
|
||||
"icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/es/oasistranslations/icon.png?raw=true",
|
||||
"lang": "Spanish"
|
||||
},
|
||||
{
|
||||
"sourceId": 31,
|
||||
"url": "http://www.novelawuxia.com/",
|
||||
"sourceName": "Novela Wuxia",
|
||||
"icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/es/novelawuxia/icon.png?raw=true",
|
||||
"lang": "Spanish"
|
||||
},
|
||||
{
|
||||
"sourceId": 33,
|
||||
"url": "https://www.novelpassion.com/",
|
||||
"sourceName": "NovelPassion",
|
||||
"icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/en/novelpassion/icon.png?raw=true",
|
||||
"lang": "English"
|
||||
},
|
||||
{
|
||||
"sourceId": 34,
|
||||
"url": "https://www.royalroad.com/",
|
||||
"sourceName": "Royal Road",
|
||||
"icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/en/royalroad/icon.png?raw=true",
|
||||
"lang": "English"
|
||||
},
|
||||
{
|
||||
"sourceId": 35,
|
||||
"url": "https://www.scribblehub.com/",
|
||||
"sourceName": "Scribble Hub",
|
||||
"icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/en/scribblehub/icon.png?raw=true",
|
||||
"lang": "English"
|
||||
},
|
||||
{
|
||||
"sourceId": 36,
|
||||
"url": "https://syosetu.com",
|
||||
"sourceName": "Syosetu",
|
||||
"icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/jp/syosetu/icon.png?raw=true",
|
||||
"lang": "Japanese"
|
||||
},
|
||||
{
|
||||
"sourceId": 37,
|
||||
"url": "https://lnmtl.com/",
|
||||
"sourceName": "LNMTL",
|
||||
"icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/en/lnmtl/icon.png?raw=true",
|
||||
"lang": "English"
|
||||
},
|
||||
{
|
||||
"sourceId": 38,
|
||||
"url": "https://skynovel.org/",
|
||||
"sourceName": "SkyNovel",
|
||||
"icon": "https://github.com/LNReader/lnreader-sources/blob/main/multisrc/madara/icons/skynovel.png?raw=true",
|
||||
"lang": "English"
|
||||
},
|
||||
{
|
||||
"sourceId": 39,
|
||||
"url": "https://novelcake.com/",
|
||||
"sourceName": "NovelCake",
|
||||
"icon": "https://github.com/LNReader/lnreader-sources/blob/main/multisrc/madara/icons/novelcake.png?raw=true",
|
||||
"lang": "English"
|
||||
},
|
||||
{
|
||||
"sourceId": 40,
|
||||
"url": "https://novelsrock.com/",
|
||||
"sourceName": "NovelsRock",
|
||||
"icon": "https://github.com/LNReader/lnreader-sources/blob/main/multisrc/madara/icons/novelsrock.png?raw=true",
|
||||
"lang": "English"
|
||||
},
|
||||
{
|
||||
"sourceId": 41,
|
||||
"url": "https://zinnovel.com/",
|
||||
"sourceName": "ZinnNovel",
|
||||
"icon": "https://github.com/LNReader/lnreader-sources/blob/main/multisrc/madara/icons/zinnovel.png?raw=true",
|
||||
"lang": "English"
|
||||
},
|
||||
{
|
||||
"sourceId": 42,
|
||||
"url": "https://noveltranslate.com/",
|
||||
"sourceName": "NovelTranslate",
|
||||
"icon": "https://github.com/LNReader/lnreader-sources/blob/main/multisrc/madara/icons/noveltranslate.png?raw=true",
|
||||
"lang": "English"
|
||||
},
|
||||
{
|
||||
"sourceId": 43,
|
||||
"url": "https://www.lunarletters.com/",
|
||||
"sourceName": "LunarLetters",
|
||||
"icon": "https://github.com/LNReader/lnreader-sources/blob/main/multisrc/madara/icons/lunarletters.png?raw=true",
|
||||
"lang": "English"
|
||||
},
|
||||
{
|
||||
"sourceId": 44,
|
||||
"url": "https://sleepytranslations.com/",
|
||||
"sourceName": "SleepyTranslations",
|
||||
"icon": "https://github.com/LNReader/lnreader-sources/blob/main/multisrc/madara/icons/sleepytranslations.png?raw=true",
|
||||
"lang": "English"
|
||||
},
|
||||
{
|
||||
"sourceId": 45,
|
||||
"url": "https://freenovel.me/",
|
||||
"sourceName": "FreeNovelMe",
|
||||
"icon": "https://github.com/LNReader/lnreader-sources/blob/main/multisrc/madara/icons/freenovel.png?raw=true",
|
||||
"lang": "English"
|
||||
},
|
||||
{
|
||||
"sourceId": 46,
|
||||
"url": "https://1stkissnovel.love/",
|
||||
"sourceName": "1stKissNovel",
|
||||
"icon": "https://github.com/LNReader/lnreader-sources/blob/main/multisrc/madara/icons/1stkissnovel.png?raw=true",
|
||||
"lang": "English"
|
||||
},
|
||||
{
|
||||
"sourceId": 47,
|
||||
"url": "https://daonovel.com/",
|
||||
"sourceName": "DaoNovel",
|
||||
"icon": "https://github.com/LNReader/lnreader-sources/blob/main/multisrc/madara/icons/daonovel.png?raw=true",
|
||||
"lang": "English"
|
||||
},
|
||||
{
|
||||
"sourceId": 48,
|
||||
"url": "https://www.wuxia.blog/",
|
||||
"sourceName": "Wuxia.Blog",
|
||||
"icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/en/wuxiablog/icon.png?raw=true",
|
||||
"lang": "English"
|
||||
},
|
||||
{
|
||||
"sourceId": 49,
|
||||
"url": "https://wuxia.city/",
|
||||
"sourceName": "Wuxia.City",
|
||||
"icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/en/wuxiacity/icon.png?raw=true",
|
||||
"lang": "English"
|
||||
},
|
||||
{
|
||||
"sourceId": 50,
|
||||
"url": "https://www.novelupdates.com/",
|
||||
"sourceName": "Novel Updates",
|
||||
"icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/en/novelupdates/icon.png?raw=true",
|
||||
"lang": "English"
|
||||
},
|
||||
{
|
||||
"sourceId": 51,
|
||||
"url": "https://ranobes.net/",
|
||||
"sourceName": "Ranobes",
|
||||
"icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/en/ranobes/icon.png?raw=true",
|
||||
"lang": "English"
|
||||
},
|
||||
{
|
||||
"sourceId": 52,
|
||||
"url": "https://m.yushubo.com",
|
||||
"sourceName": "Yushubo",
|
||||
"icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/ch/yushubo/icon.png?raw=true",
|
||||
"lang": "Chinese"
|
||||
},
|
||||
{
|
||||
"sourceId": 53,
|
||||
"url": "https://kolnovel.com/",
|
||||
"sourceName": "KolNovel",
|
||||
"icon": "https://github.com/LNReader/lnreader-sources/blob/main/multisrc/wpmangastream/icons/kolnovel.png?raw=true",
|
||||
"lang": "Arabic"
|
||||
},
|
||||
{
|
||||
"sourceId": 54,
|
||||
"url": "https://rewayat-ar.site/",
|
||||
"sourceName": "RewayatAr",
|
||||
"icon": "https://github.com/LNReader/lnreader-sources/blob/main/multisrc/wpmangastream/icons/rewayatar.png?raw=true",
|
||||
"lang": "Arabic"
|
||||
},
|
||||
{
|
||||
"sourceId": 55,
|
||||
"url": "https://mostnovel.com/",
|
||||
"sourceName": "MostNovel",
|
||||
"icon": "https://github.com/LNReader/lnreader-sources/blob/main/multisrc/madara/icons/mostnovel.png?raw=true",
|
||||
"lang": "English"
|
||||
},
|
||||
{
|
||||
"sourceId": 56,
|
||||
"url": "https://www.novelmultiverse.com/",
|
||||
"sourceName": "NovelMultiverse",
|
||||
"icon": "https://github.com/LNReader/lnreader-sources/blob/main/multisrc/madara/icons/novelmultiverse.png?raw=true",
|
||||
"lang": "English"
|
||||
},
|
||||
{
|
||||
"sourceId": 57,
|
||||
"url": "https://lightnovelheaven.com/",
|
||||
"sourceName": "LightNovelHeaven",
|
||||
"icon": "https://github.com/LNReader/lnreader-sources/blob/main/multisrc/madara/icons/lightnovelheaven.png?raw=true",
|
||||
"lang": "English"
|
||||
},
|
||||
{
|
||||
"sourceId": 58,
|
||||
"url": "https://lightnovelshub.com/",
|
||||
"sourceName": "LightNovelsHub",
|
||||
"icon": "https://github.com/LNReader/lnreader-sources/blob/main/multisrc/madara/icons/lightnovelshub.png?raw=true",
|
||||
"lang": "English"
|
||||
},
|
||||
{
|
||||
"sourceId": 59,
|
||||
"url": "https://arnovel.me/",
|
||||
"sourceName": "ArNovel",
|
||||
"icon": "https://github.com/LNReader/lnreader-sources/blob/main/multisrc/madara/icons/arnovel.png?raw=true",
|
||||
"lang": "Arabic"
|
||||
},
|
||||
{
|
||||
"sourceId": 60,
|
||||
"url": "https://meionovel.id/",
|
||||
"sourceName": "MeioNovel",
|
||||
"icon": "https://github.com/LNReader/lnreader-sources/blob/main/multisrc/madara/icons/meionovel.png?raw=true",
|
||||
"lang": "Indonesian"
|
||||
},
|
||||
{
|
||||
"sourceId": 61,
|
||||
"url": "https://www.webnovelover.com/",
|
||||
"sourceName": "WebNovelLover",
|
||||
"icon": "https://github.com/LNReader/lnreader-sources/blob/main/multisrc/madara/icons/webnovellover.png?raw=true",
|
||||
"lang": "English"
|
||||
},
|
||||
{
|
||||
"sourceId": 62,
|
||||
"url": "https://www.wlnupdates.com/",
|
||||
"sourceName": "WLNUpdates",
|
||||
"icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/en/wlnupdates/icon.png?raw=true",
|
||||
"lang": "English"
|
||||
},
|
||||
{
|
||||
"sourceId": 63,
|
||||
"url": "https://boxnovel.online/",
|
||||
"sourceName": "BoxNovel.online",
|
||||
"icon": "https://github.com/LNReader/lnreader-sources/blob/main/multisrc/madara/icons/boxnovel.online.png?raw=true",
|
||||
"lang": "English"
|
||||
},
|
||||
{
|
||||
"sourceId": 64,
|
||||
"url": "https://clicknovel.net/",
|
||||
"sourceName": "ClickNovel",
|
||||
"icon": "https://github.com/LNReader/lnreader-sources/blob/main/multisrc/madara/icons/clicknovel.png?raw=true",
|
||||
"lang": "English"
|
||||
},
|
||||
{
|
||||
"sourceId": 65,
|
||||
"url": "https://readwebnovels.net/",
|
||||
"sourceName": "ReadWebNovels",
|
||||
"icon": "https://github.com/LNReader/lnreader-sources/blob/main/multisrc/madara/icons/readwebnovels.png?raw=true",
|
||||
"lang": "English"
|
||||
},
|
||||
{
|
||||
"sourceId": 66,
|
||||
"url": "https://wbnovel.com/",
|
||||
"sourceName": "WBNovel",
|
||||
"icon": "https://github.com/LNReader/lnreader-sources/blob/main/multisrc/madara/icons/wbnovel.png?raw=true",
|
||||
"lang": "English"
|
||||
},
|
||||
{
|
||||
"sourceId": 67,
|
||||
"url": "https://reaperscans.com/all-series/novels/",
|
||||
"sourceName": "ReaperScans",
|
||||
"icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/en/reaperscans/icon.png?raw=true",
|
||||
"lang": "English"
|
||||
},
|
||||
{
|
||||
"sourceId": 68,
|
||||
"url": "https://www.readwn.com/",
|
||||
"sourceName": "Readwn.com",
|
||||
"icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/en/readwn/icon.png?raw=true",
|
||||
"lang": "English"
|
||||
},
|
||||
{
|
||||
"sourceId": 69,
|
||||
"url": "https://ranobehub.org/",
|
||||
"sourceName": "Ranobehub",
|
||||
"icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/ru/ranobehub/icon.png?raw=true",
|
||||
"lang": "Russian"
|
||||
},
|
||||
{
|
||||
"sourceId": 70,
|
||||
"url": "https://divinedaolibrary.com/",
|
||||
"sourceName": "Divine Dao Library",
|
||||
"icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/en/divinedaolibrary/icon.png?raw=true",
|
||||
"lang": "English"
|
||||
},
|
||||
{
|
||||
"sourceId": 71,
|
||||
"url": "https://novelonlinefull.com/",
|
||||
"sourceName": "NovelOnlineFull.com",
|
||||
"icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/en/novelonlinefull/icon.png?raw=true",
|
||||
"lang": "English"
|
||||
},
|
||||
{
|
||||
"sourceId": 72,
|
||||
"url": "https://www.lightnovelupdates.com/",
|
||||
"sourceName": "LightNovelUpdates",
|
||||
"icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/en/lightnovelupdates/icon.png?raw=true",
|
||||
"lang": "English"
|
||||
},
|
||||
{
|
||||
"sourceId": 73,
|
||||
"url": "https://www.epiknovel.com/",
|
||||
"sourceName": "EpikNovel",
|
||||
"icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/tr/epiknovel/icon.png?raw=true",
|
||||
"lang": "Turkish"
|
||||
},
|
||||
{
|
||||
"sourceId": 74,
|
||||
"url": "https://wuxiaworld.site/",
|
||||
"sourceName": "WuxiaWorld.Site",
|
||||
"icon": "https://github.com/LNReader/lnreader-sources/blob/main/multisrc/madara/icons/wuxiaworldsite.png?raw=true",
|
||||
"lang": "English"
|
||||
},
|
||||
{
|
||||
"sourceId": 75,
|
||||
"url": "https://mysticalmerries.com/",
|
||||
"sourceName": "MysticalSeries",
|
||||
"icon": "https://github.com/LNReader/lnreader-sources/blob/main/multisrc/madara/icons/mysticalseries.png?raw=true",
|
||||
"lang": "English"
|
||||
},
|
||||
{
|
||||
"sourceId": 76,
|
||||
"url": "https://travistranslations.com/",
|
||||
"sourceName": "TravisTranslations",
|
||||
"icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/en/travistranslations/icon.png?raw=true",
|
||||
"lang": "English"
|
||||
},
|
||||
{
|
||||
"sourceId": 77,
|
||||
"url": "https://noveldeglace.com/",
|
||||
"sourceName": "NovelDeGlace",
|
||||
"icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/fr/noveldeglace/icon.png?raw=true",
|
||||
"lang": "French"
|
||||
},
|
||||
{
|
||||
"sourceId": 78,
|
||||
"url": "https://rainofsnow.com/",
|
||||
"sourceName": "RainOfSnow",
|
||||
"icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/en/rainofsnow/icon.png?raw=true",
|
||||
"lang": "French"
|
||||
}
|
||||
]
|
||||
@@ -1,170 +0,0 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
|
||||
const sourceId = 73;
|
||||
|
||||
const sourceName = "EpikNovel";
|
||||
|
||||
const baseUrl = "https://www.epiknovel.com/";
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
let url = baseUrl + "seri-listesi?Sayfa=" + page;
|
||||
const totalPages = 5;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$("div.col-lg-12.col-md-12").each(function () {
|
||||
const novelName = $(this).find("h3").text();
|
||||
const novelCover = $(this).find("img").attr("data-src");
|
||||
const novelUrl = $(this).find("h3 > a").attr("href");
|
||||
|
||||
const novel = {
|
||||
sourceId,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
console.log(novels);
|
||||
|
||||
return { novels, totalPages };
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = novelUrl;
|
||||
console.log(url);
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = { sourceId, sourceName, url, novelUrl };
|
||||
|
||||
novel.novelName = $("h1#tables").text().trim();
|
||||
|
||||
novel.novelCover = $("img.manga-cover").attr("src");
|
||||
|
||||
novel.summary = $(
|
||||
"#wrapper > div.row > div.col-md-9 > div:nth-child(6) > p:nth-child(3)"
|
||||
)
|
||||
.text()
|
||||
.trim();
|
||||
|
||||
novel.status = $(
|
||||
"#wrapper > div.row > div.col-md-9 > div.row > div.col-md-9 > h4:nth-child(3) > a"
|
||||
)
|
||||
.text()
|
||||
.trim();
|
||||
|
||||
novel.author = $("#NovelInfo > p:nth-child(4)")
|
||||
.text()
|
||||
.replace(/Publisher:|\s/g, "")
|
||||
.trim();
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
$("table").find("tr").first().remove();
|
||||
|
||||
$("table")
|
||||
.find("tr")
|
||||
.each(function () {
|
||||
const releaseDate = $(this).find("td:nth-child(3)").text();
|
||||
|
||||
let chapterName = $(this).find("td:nth-child(1) > a").text();
|
||||
|
||||
if ($(this).find("td:nth-child(1) > span").length > 0) {
|
||||
chapterName = "🔒 " + chapterName;
|
||||
}
|
||||
|
||||
const chapterUrl = $(this)
|
||||
.find(" td:nth-child(1) > a")
|
||||
.attr("href");
|
||||
|
||||
novelChapters.push({
|
||||
chapterName,
|
||||
chapterUrl,
|
||||
releaseDate,
|
||||
});
|
||||
});
|
||||
|
||||
novel.chapters = novelChapters;
|
||||
console.log(novel);
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = chapterUrl;
|
||||
|
||||
console.log(url);
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let chapterName, chapterText;
|
||||
|
||||
if (result.url == "https://www.epiknovel.com/login") {
|
||||
chapterName = "";
|
||||
chapterText = "Premium Chapter";
|
||||
} else {
|
||||
chapterName = $("#icerik > center > h4 > b").text();
|
||||
chapterText = $("div#icerik").html();
|
||||
}
|
||||
|
||||
const chapter = {
|
||||
sourceId,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const url = baseUrl + "seri-listesi?q=" + searchTerm;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$("div.col-lg-12.col-md-12").each(function () {
|
||||
const novelName = $(this).find("h3").text();
|
||||
const novelCover = $(this).find("img").attr("data-src");
|
||||
const novelUrl = $(this).find("h3 > a").attr("href");
|
||||
|
||||
const novel = {
|
||||
sourceId,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const EpikNovelScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default EpikNovelScraper;
|
||||
Reference in New Issue
Block a user