Add more sources and fix sources 1-9

This commit is contained in:
Rajarshee Chatterjee
2021-06-10 18:29:13 +05:30
parent d4140fcc52
commit e87ecb48b4
22 changed files with 4030 additions and 1 deletions
+2 -1
View File
@@ -66,7 +66,8 @@ const Extension = ({ navigation, route }) => {
};
const clearSearchbar = () => {
getNovels();
setNovels([]);
getNovels(1);
setLoading(true);
setSearchText("");
};
+177
View File
@@ -0,0 +1,177 @@
import cheerio from "react-native-cheerio";
import { htmlToText } from "../helpers/htmlToText";
const baseUrl = "https://fastnovel.net";
const searchUrl = `https://fastnovel.net/search/`;
const popularNovels = async (page) => {
const url = `${baseUrl}/list/most-popular.html?page=${page}`;
const result = await fetch(url);
const body = await result.text();
$ = cheerio.load(body);
let novels = [];
$(".film-item").each(function (result) {
const novelName = $(this).find(".name").text();
const novelCover = $(this).find(".img").attr("data-original");
const novelUrl = $(this).find("a").attr("href").substring(1);
const novel = {
sourceId: 3,
novelName,
novelCover,
novelUrl,
};
novels.push(novel);
});
return novels;
};
const parseNovelAndChapters = async (novelUrl) => {
const url = `${baseUrl}/${novelUrl}/`;
const result = await fetch(url);
const body = await result.text();
$ = cheerio.load(body);
let novel = {};
novel.sourceId = 3;
novel.sourceName = "FastNovel";
novel.url = url;
novel.novelUrl = novelUrl;
novel.novelName = $("h1").text();
novel.novelCover = $(".book-cover").attr("data-original");
novel.summary = $("div.film-content > p").text();
novel.author = $("li > label")
.filter(function () {
return $(this).text().trim() === "Author:";
})
.next()
.text();
novel.genre = $("li")
.filter(function () {
return $(this).find("label").text().trim() === "Genre:";
})
.text()
.replace("Genre:", "");
novel.artist = null;
novel.Status = null;
let novelChapters = [];
$(".chapter").each(function (result) {
const chapterName = $(this).text();
const releaseDate = null;
let chapterUrl = $(this).attr("href");
chapterUrl = chapterUrl.replace(`/${novelUrl}/`, "");
const novel = {
chapterName,
releaseDate,
chapterUrl,
};
novelChapters.push(novel);
});
novel.chapters = novelChapters;
return novel;
};
const parseChapter = async (novelUrl, chapterUrl) => {
const url = `${baseUrl}/${novelUrl}/${chapterUrl}`;
const result = await fetch(url);
const body = await result.text();
$ = cheerio.load(body);
const chapterName = $(".episode-name").text();
let chapterText = $("#chapter-body").html();
chapterText = htmlToText(chapterText);
let nextChapter = null;
nextChapter = $("div.btn-group > a")
.filter(function () {
return $(this).text().trim() === "Next Chapter";
})
.attr("href")
.replace("/", "");
let prevChapter = null;
prevChapter = $("div.btn-group > a")
.filter(function () {
return $(this).text().trim() === "Prev Chapter";
})
.attr("href")
.replace("/", "");
const chapter = {
sourceId: 3,
novelUrl,
chapterUrl,
chapterName,
chapterText,
nextChapter,
prevChapter,
};
return chapter;
};
const searchNovels = async (searchTerm) => {
const url = `${searchUrl}${searchTerm}`;
const result = await fetch(url);
const body = await result.text();
$ = cheerio.load(body);
let novels = [];
$(".film-item").each(function (result) {
const novelName = $(this).find("div.title > p.name").text();
const novelCover = $(this).find(".img").attr("data-original");
const novelUrl = $(this).find("a").attr("href").substring(1);
const novel = {
sourceId: 3,
novelName,
novelCover,
novelUrl,
};
novels.push(novel);
});
return novels;
};
const fastNovelScraper = {
popularNovels,
parseNovelAndChapters,
parseChapter,
searchNovels,
};
export default fastNovelScraper;
+212
View File
@@ -0,0 +1,212 @@
import cheerio from "react-native-cheerio";
import { htmlToText } from "../helpers/htmlToText";
const baseUrl = "https://foxaholic.com/";
const popularNovels = async (page) => {
let url = baseUrl + "novel/?m_orderby=rating";
const result = await fetch(url);
const body = await result.text();
$ = cheerio.load(body);
let novels = [];
$(".page-item-detail").each(function (result) {
const novelName = $(this).find(".h5 > a").text();
const novelCover = $(this).find("img").attr("data-src");
let novelUrl = $(this).find(".h5 > a").attr("href").split("/")[4];
novelUrl += "/";
const novel = {
sourceId: 22,
novelName,
novelCover,
novelUrl,
};
novels.push(novel);
});
return novels;
};
const parseNovelAndChapters = async (novelUrl) => {
const url = `${baseUrl}novel/${novelUrl}/`;
const result = await fetch(url);
const body = await result.text();
$ = cheerio.load(body);
let novel = {};
novel.sourceId = 22;
novel.sourceName = "Foxaholic";
novel.url = url;
novel.novelUrl = novelUrl;
novel.novelName = $(".post-title > h1")
.text()
.replace(/[\t\n]/g, "")
.trim();
novel.novelCover = $(".summary_image > a > img").attr("data-src");
$(".post-content_item").each(function (result) {
detailName = $(this)
.find(".summary-heading > h5")
.text()
.replace(/[\t\n]/g, "")
.trim();
detail = $(this)
.find(".summary-content")
.text()
.replace(/[\t\n]/g, "")
.trim();
novel[detailName] = detail;
});
novel.genre = novel.Genre.replace(/, /g, ",");
novel.author = novel.Author;
novel.Status = null;
novel.artist = null;
delete novel.Genre;
delete novel.Author;
$(".description-summary > div.summary__content").find("em").remove();
novel.summary = $(".description-summary > div.summary__content")
.text()
.replace(/[\t\n]/g, "");
let novelChapters = [];
const novelId = $("#manga-chapters-holder").attr("data-id");
let formData = new FormData();
formData.append("action", "manga_get_chapters");
formData.append("manga", novelId);
const data = await fetch("https://foxaholic.com/wp-admin/admin-ajax.php", {
method: "POST",
body: formData,
});
const text = await data.text();
$ = cheerio.load(text);
$(".wp-manga-chapter").each(function (result) {
chapterName = $(this)
.find("a")
.text()
.replace(/[\t\n]/g, "")
.trim();
releaseDate = $(this).find("span").text().trim();
chapterUrl = $(this).find("a").attr("href").split("/");
chapterUrl[6]
? (chapterUrl = chapterUrl[5] + "/" + chapterUrl[6])
: (chapterUrl = chapterUrl[5]);
novelChapters.push({ chapterName, releaseDate, chapterUrl });
});
novel.chapters = novelChapters.reverse();
return novel;
};
const parseChapter = async (novelUrl, chapterUrl) => {
const url = `${baseUrl}novel/${novelUrl}/${chapterUrl}/`;
const result = await fetch(url);
const body = await result.text();
$ = cheerio.load(body);
const chapterName = $("h1#chapter-heading").text();
let chapterText = $(".reading-content").html();
chapterText = htmlToText(chapterText);
let nextChapter = null;
if ($(".nav-next").length) {
nextChapter = $(".nav-next").find("a").attr("href").split("/");
nextChapter[6]
? (nextChapter = nextChapter[5] + "/" + nextChapter[6])
: (nextChapter = nextChapter[5]);
}
let prevChapter = null;
if ($(".nav-previous").length) {
prevChapter = $(".nav-previous").find("a").attr("href").split("/");
prevChapter[6]
? (prevChapter = prevChapter[5] + "/" + prevChapter[6])
: (prevChapter = prevChapter[5]);
}
novelUrl += "/";
chapterUrl = optionalUrl + chapterUrl;
const chapter = {
sourceId: 22,
novelUrl,
chapterUrl,
chapterName,
chapterText,
nextChapter,
prevChapter,
};
return chapter;
};
const searchNovels = async (searchTerm) => {
const url = `${baseUrl}?s=${searchTerm}&post_type=wp-manga`;
const result = await fetch(url);
const body = await result.text();
$ = cheerio.load(body);
let novels = [];
$(".c-tabs-item__content").each(function (result) {
const novelName = $(this).find(".h4 > a").text();
const novelCover = $(this).find("img").attr("data-src");
let novelUrl = $(this).find(".h4 > a").attr("href").split("/")[4];
novelUrl += "/";
const novel = {
sourceId: 22,
novelName,
novelCover,
novelUrl,
};
novels.push(novel);
});
return novels;
};
const FoxaHolicScraper = {
popularNovels,
parseNovelAndChapters,
parseChapter,
searchNovels,
};
export default FoxaHolicScraper;
+213
View File
@@ -0,0 +1,213 @@
import cheerio from "react-native-cheerio";
import { htmlToText } from "../helpers/htmlToText";
const baseUrl = "https://freewebnovel.com/";
const popularNovels = async (page) => {
let url = baseUrl + "most-popular-novel/";
const result = await fetch(url);
const body = await result.text();
$ = cheerio.load(body);
let novels = [];
$(".li-row").each(function (result) {
const novelName = $(this).find(".tit").text();
const novelCover = $(this).find("img").attr("src");
let novelUrl = $(this)
.find("h3 > a")
.attr("href")
.replace(".html", "")
.slice(1);
novelUrl += "/";
const novel = {
sourceId: 13,
novelName,
novelCover,
novelUrl,
};
novels.push(novel);
});
return novels;
};
const parseNovelAndChapters = async (novelUrl) => {
const url = `${baseUrl}${novelUrl}.html`;
const result = await fetch(url);
const body = await result.text();
$ = cheerio.load(body);
let novel = {};
novel.sourceId = 13;
novel.sourceName = "FreeWebNovel";
novel.url = url;
novel.novelUrl = `${novelUrl.replace(".html", "")}/`;
novel.novelName = $("h1.tit").text();
novel.novelCover = $(".pic > img").attr("src");
novel.genre = $("[title=Genre]")
.next()
.text()
.replace(/[\t\n]/g, "");
novel.author = $("[title=Author]")
.next()
.text()
.replace(/[\t\n]/g, "");
novel.artist = null;
novel.Status = $("[title=Status]")
.next()
.text()
.replace(/[\t\n]/g, "");
novel.Alternative = $("[title='Alternative names']")
.next()
.text()
.replace(/[\t\n]/g, "");
let novelSummary = $(".inner").html();
novel.summary = htmlToText(novelSummary);
let novelChapters = [];
let latestChapter;
$("h3.tit").each(function (res) {
if ($(this).find("a").text() === novel.novelName) {
// console.log($(this).find("a").text());
// console.log(novel.novelName);
latestChapter = $(this).next().find("span.s3").text().match(/\d+/);
}
});
latestChapter = latestChapter[0];
for (let i = 1; i <= parseInt(latestChapter); i++) {
const chapterName = "Chapter " + i;
const releaseDate = null;
const chapterUrl = "chapter-" + i;
const chapter = { chapterName, releaseDate, chapterUrl };
novelChapters.push(chapter);
}
novel.chapters = novelChapters;
return novel;
};
const parseChapter = async (novelUrl, chapterUrl) => {
const url = `${baseUrl}${novelUrl}/${chapterUrl}.html`;
const result = await fetch(url);
const body = await result.text();
$ = cheerio.load(body);
let chapterName = $("h1.tit").text();
let chapterText = $("div.txt").html();
chapterText = htmlToText(chapterText);
let nextChapter = null;
if ($("[title='Read Next chapter']").length) {
nextChapter = $("[title='Read Next chapter']")
.attr("href")
.replace("/" + novelUrl + "/", "")
.replace(".html", "");
}
let prevChapter = null;
if ($("[title='Read Privious Chapter']").length) {
prevChapter = $("[title='Read Privious Chapter']")
.attr("href")
.replace("/" + novelUrl + "/", "")
.replace(".html", "");
}
chapterUrl = chapterUrl;
const chapter = {
sourceId: 13,
novelUrl,
chapterUrl,
chapterName,
chapterText,
nextChapter,
prevChapter,
};
return chapter;
};
const searchNovels = async (searchTerm) => {
const url = baseUrl + "search/";
const formData = new FormData();
formData.append("searchkey", searchTerm);
const result = await fetch(url, {
method: "POST",
body: formData,
});
const body = await result.text();
$ = cheerio.load(body);
let novels = [];
$(".li-row > .li > .con").each(function (result) {
const novelName = $(this).find(".tit").text();
const novelCover = $(this).find(".pic > a > img").attr("data-cfsrc");
let novelUrl = $(this)
.find("h3 > a")
.attr("href")
.replace(".html", "")
.slice(1);
novelUrl += "/";
const novel = {
sourceId: 13,
novelName,
novelCover,
novelUrl,
};
novels.push(novel);
});
return novels;
};
const FreeWebNovelScraper = {
popularNovels,
parseNovelAndChapters,
parseChapter,
searchNovels,
};
export default FreeWebNovelScraper;
+179
View File
@@ -0,0 +1,179 @@
import cheerio from "react-native-cheerio";
import { htmlToText } from "../helpers/htmlToText";
const baseUrl = "https://jpmtl.com/";
const popularNovels = async (page) => {
let url =
"https://jpmtl.com/v2/book/show/browse?query=&categories=&content_type=0&direction=0&page=2&limit=20&type=5&status=all&language=3&exclude_categories=";
const result = await fetch(url);
const body = await result.json();
let novels = [];
body.map((item) => {
const novelName = item.title;
const novelCover = item.cover;
const novelUrl = item.id + "/";
const novel = { sourceId: 14, novelName, novelCover, novelUrl };
novels.push(novel);
});
return novels;
};
const parseNovelAndChapters = async (novelUrl) => {
const url = `${baseUrl}books/${novelUrl}`;
const result = await fetch(url);
const body = await result.text();
$ = cheerio.load(body);
let novel = {};
novel.sourceId = 14;
novel.sourceName = "JPMTL";
novel.url = url;
novel.novelUrl = novelUrl;
novel.novelName = $("h1.book-sidebar__title").text();
novel.novelCover = $(".book-sidebar").find("img").attr("src");
$(".post-content_item").each(function (result) {
detailName = $(this)
.find(".summary-heading > h5")
.text()
.replace(/[\t\n]/g, "")
.trim();
detail = $(this)
.find(".summary-content")
.text()
.replace(/[\t\n]/g, "")
.trim();
novel[detailName] = detail;
});
novel.summary = $(".main-book__synopsis").text();
novel.genre = "";
$("a.main-book__category").each(function (result) {
novel.genre += $(this).text();
});
novel.author = $(".book-sidebar__author > .book-sidebar__info")
.text()
.replace(/[\t\n]/g, "")
.trim();
let novelChapters = [];
const chapterListUrl = `https://jpmtl.com/v2/chapter/${novelUrl}/list?state=published&structured=true&d`;
const chapterResult = await fetch(chapterListUrl);
const volumes = await chapterResult.json();
volumes.map((volume) => {
volume.chapters.map((chapter) => {
const chapterName = chapter.title;
const releaseDate = chapter.created_at;
const chapterUrl = chapter.id;
const obj = {
sourceId: 14,
chapterName,
releaseDate,
chapterUrl,
};
novelChapters.push(obj);
});
});
novel.chapters = novelChapters;
return novel;
};
const parseChapter = async (novelUrl, chapterUrl) => {
const url = `${baseUrl}books/${novelUrl}/${chapterUrl}`;
const result = await fetch(url);
const body = await result.text();
console.log(url);
$ = cheerio.load(body);
const chapterName = $(".chapter-content__title").text();
let chapterText = $(".chapter-content__content").html();
chapterText = htmlToText(chapterText);
let nextChapter = null;
let prevChapter = null;
$(".chapter-wrapper__nav").each(function (result) {
const chId = $(this)
.attr("href")
.replace("/books/" + novelUrl + "/", "");
if (chId < chapterUrl) {
prevChapter = chId;
} else {
nextChapter = chId;
}
});
novelUrl += "/";
const chapter = {
sourceId: 14,
novelUrl,
chapterUrl,
chapterName,
chapterText,
nextChapter,
prevChapter,
};
return chapter;
};
const searchNovels = async (searchTerm) => {
const url = `https://jpmtl.com/v2/book/show/browse?query=${searchTerm}&categories=&content_type=2&direction=0&page=1&limit=20&type=5&status=all&language=3&exclude_categories=`;
const result = await fetch(url);
const body = await result.json();
const novels = [];
body.map((item) => {
const novelName = item.title;
const novelCover = item.cover;
const novelUrl = item.id + "/";
const novel = { sourceId: 14, novelName, novelCover, novelUrl };
novels.push(novel);
});
return novels;
};
const JPMTLScraper = {
popularNovels,
parseNovelAndChapters,
parseChapter,
searchNovels,
};
export default JPMTLScraper;
+188
View File
@@ -0,0 +1,188 @@
import cheerio from "react-native-cheerio";
import { htmlToText } from "../helpers/htmlToText";
const baseUrl = "https://kisslightnovels.info/";
const popularNovels = async (page) => {
const result = await fetch(baseUrl);
const body = await result.text();
$ = cheerio.load(body);
let novels = [];
$(".page-item-detail").each(function (result) {
const novelName = $(this).find("h5 > a").text();
const novelCover = $(this).find("img").attr("src");
let novelUrl = $(this).find("h5 > a").attr("href");
novelUrl = novelUrl.replace(`${baseUrl}novel/`, "");
const novel = {
sourceId: 11,
novelName,
novelCover,
novelUrl,
};
novels.push(novel);
});
return novels;
};
const parseNovelAndChapters = async (novelUrl) => {
const url = `${baseUrl}novel/${novelUrl}/`;
const result = await fetch(url);
const body = await result.text();
$ = cheerio.load(body);
let novel = {};
novel.sourceId = 11;
novel.sourceName = "KissLightNovels";
novel.url = url;
novel.novelUrl = novelUrl;
novel.novelName = $(".post-title > h1")
.text()
.replace(/[\t\n]/g, "")
.trim();
novel.novelCover = $(".summary_image > a > img").attr("src");
$(".post-content_item").each(function (result) {
detailName = $(this)
.find(".summary-heading > h5")
.text()
.replace(/[\t\n]/g, "")
.trim();
detail = $(this)
.find(".summary-content")
.text()
.replace(/[\t\n]/g, "")
.trim();
novel[detailName] = detail;
});
$(".description-summary > div.summary__content").find("em").remove();
novel.summary = $(".description-summary > div.summary__content")
.text()
.replace(/[\t\n]/g, "");
let novelChapters = [];
$(".wp-manga-chapter").each(function (result) {
chapterName = $(this)
.find("a")
.text()
.replace(/[\t\n]/g, "")
.trim();
releaseDate = $(this)
.find("span")
.text()
.replace(/[\t\n]/g, "")
.trim();
chapterUrl = $(this).find("a").attr("href").replace(url, "");
novelChapters.push({ chapterName, releaseDate, chapterUrl });
});
novel.chapters = novelChapters.reverse();
return novel;
};
const parseChapter = async (novelUrl, chapterUrl) => {
const url = `${baseUrl}novel/${novelUrl}/${chapterUrl}`;
const result = await fetch(url);
const body = await result.text();
$ = cheerio.load(body);
$("#snippet-box").remove();
const chapterName = $("div.text-left > h4").text();
let chapterText = $(".reading-content").html();
chapterText = htmlToText(chapterText);
let nextChapter = null;
if ($(".nav-next").length) {
nextChapter = $(".nav-next")
.find("a")
.attr("href")
.replace(baseUrl + "novel/" + novelUrl + "/", "");
}
let prevChapter = null;
if ($(".nav-previous").length) {
prevChapter = $(".nav-previous")
.find("a")
.attr("href")
.replace(baseUrl + "novel/" + novelUrl + "/", "");
}
const chapter = {
sourceId: 11,
novelUrl,
chapterUrl,
chapterName,
chapterText,
nextChapter,
prevChapter,
};
return chapter;
};
const searchNovels = async (searchTerm) => {
const url = `${baseUrl}?s=${searchTerm}&post_type=wp-manga&m_orderby=rating`;
const result = await fetch(url);
const body = await result.text();
$ = cheerio.load(body);
let novels = [];
$(".c-tabs-item__content").each(function (result) {
const novelName = $(this).find("h4 > a").text();
const novelCover = $(this).find("img").attr("src");
let novelUrl = $(this).find("h4 > a").attr("href");
novelUrl = novelUrl.replace(`${baseUrl}novel/`, "");
const novel = {
sourceId: 11,
novelName,
novelCover,
novelUrl,
};
novels.push(novel);
});
return novels;
};
const kissLightNovelScraper = {
popularNovels,
parseNovelAndChapters,
parseChapter,
searchNovels,
};
export default kissLightNovelScraper;
+198
View File
@@ -0,0 +1,198 @@
import cheerio from "react-native-cheerio";
import { htmlToText } from "../helpers/htmlToText";
const baseUrl = "https://www.lightnovelpub.com/";
const popularNovels = async (page) => {
let url = baseUrl + "browse/all/popular/all/1";
const result = await fetch(url);
const body = await result.text();
$ = cheerio.load(body);
let novels = [];
$(".novel-item.ads").remove();
$(".novel-item").each(function (result) {
const novelName = $(this)
.find(".novel-title")
.text()
.replace(/[\t\n]/g, "");
const novelCover = $(this).find("img").attr("data-src");
let novelUrl = $(this)
.find(".novel-title > a")
.attr("href")
.replace(`/novel/`, "");
novelUrl += "/";
const novel = {
sourceId: 15,
novelName,
novelCover,
novelUrl,
};
novels.push(novel);
});
return novels;
};
const parseNovelAndChapters = async (novelUrl) => {
const url = `${baseUrl}novel/${novelUrl}/`;
const result = await fetch(url);
const body = await result.text();
$ = cheerio.load(body);
let novel = {};
novel.sourceId = 15;
novel.sourceName = "LightNovelPub";
novel.url = url;
novel.novelUrl = novelUrl;
novel.novelName = $("h1.novel-title")
.text()
.replace(/[\t\n]/g, "")
.trim();
novel.novelCover = $("figure.cover > img").attr("src");
novel.genre = "";
$("div.categories > ul > li").each(function (result) {
novel.genre +=
$(this)
.text()
.replace(/[\t\n]/g, "") + ",";
});
$("div.header-stats > span").each(function (result) {
if ($(this).find("small").text() === "Status") {
novel.Status = $(this).find("strong").text();
}
});
novel.genre = novel.genre.slice(0, -1);
novel.author = $(".author > a > span").text();
novelSummary = $(".summary > .content").html();
novel.summary = htmlToText(novelSummary);
let novelChapters = [];
let firstChapterNo, totalChapters;
totalChapters = $(".header-stats > span").first().text().match(/\d+/)[0];
firstChapterNo = $(".chapter-no").first().text().match(/\d+/)[0];
for (let i = firstChapterNo; i <= totalChapters; i++) {
const chapterName = "Chapter " + i;
const releaseDate = null;
const chapterUrl = "chapter-" + i;
const chapter = { chapterName, releaseDate, chapterUrl };
novelChapters.push(chapter);
}
novel.chapters = novelChapters;
return novel;
};
const parseChapter = async (novelUrl, chapterUrl) => {
const url = `${baseUrl}novel/${novelUrl}/${chapterUrl}`;
const result = await fetch(url);
const body = await result.text();
$ = cheerio.load(body);
const chapterName = $("h2").text();
let chapterText = $("#chapter-container").html();
chapterText = htmlToText(chapterText);
let nextChapter = null;
if (!$("a.nextchap.isDisabled").length) {
nextChapter = $("a.nextchap")
.attr("href")
.replace("/novel/" + novelUrl + "/", "");
}
let prevChapter = null;
if (!$("a.prevchap.isDisabled").length) {
prevChapter = $("a.prevchap")
.attr("href")
.replace("/novel/" + novelUrl + "/", "");
}
const chapter = {
sourceId: 15,
novelUrl,
chapterUrl,
chapterName,
chapterText,
nextChapter,
prevChapter,
};
return chapter;
};
const searchNovels = async (searchTerm) => {
const url = `${baseUrl}lnwsearchlive?inputContent=${searchTerm}`;
const result = await scraper(url);
$ = cheerio.load(result);
let novels = [];
let results = JSON.parse($("body").text());
$ = cheerio.load(results.resultview);
$(".novel-item").each(function (result) {
const novelName = $(this).find("h4.novel-title").text();
const novelCover = $(this).find("img").attr("src");
let novelUrl = $(this).find("a").attr("href").replace(`/novel/`, "");
novelUrl += "/";
const novel = {
sourceId: 15,
novelName,
novelCover,
novelUrl,
};
novels.push(novel);
});
return novels;
};
const lightNovelPubScraper = {
popularNovels,
parseNovelAndChapters,
parseChapter,
searchNovels,
};
export default lightNovelPubScraper;
+196
View File
@@ -0,0 +1,196 @@
import cheerio from "react-native-cheerio";
import { htmlToText } from "../helpers/htmlToText";
const baseUrl = "https://www.mtlnovel.com";
const popularNovels = async (page) => {
const url = `${baseUrl}/alltime-rank/page/${page}`;
const result = await fetch(url);
const body = await result.text();
$ = cheerio.load(body);
let novels = [];
$("div.box.wide").each(function (result) {
const novelName = $(this).find("a.list-title").text().slice(4);
const novelCover = $(this).find("amp-img").attr("src");
let novelUrl = $(this).find("a.list-title").attr("href");
novelUrl = novelUrl.replace("https://www.mtlnovel.com/", "");
const novel = {
sourceId: 5,
novelUrl,
novelName,
novelCover,
};
novels.push(novel);
});
return novels;
};
const parseNovelAndChapters = async (novelUrl) => {
const url = `${baseUrl}/${novelUrl}`;
const result = await fetch(url);
const body = await result.text();
$ = cheerio.load(body);
let novel = {};
novel.sourceId = 5;
novel.sourceName = "MTLNovel";
novel.url = url;
novel.novelUrl = novelUrl;
novel.novelName = $("h1.entry-title").text();
novel.novelCover = $(".nov-head > amp-img").attr("src");
novel.summary = $("div.desc > h2").next().text();
novel.author = $("tr > td")
.filter(function () {
return $(this).prev().text().trim() === "Author";
})
.next()
.text()
.replace("Auhtor:", "");
novel.Status = $("tr > td")
.filter(function () {
return $(this).prev().text().trim() === "Status";
})
.next()
.text()
.replace("Status:", "");
novel.genre = $("td")
.filter(function () {
return $(this).prev().text().trim() === "Genre";
})
.next()
.text()
.replace("Genre:", "");
novel.artist = null;
const chapterListUrl = url + "/chapter-list/";
const getChapters = async () => {
const result = await fetch(chapterListUrl);
const body = await result.text();
$ = cheerio.load(body);
let novelChapters = [];
$("div.ch-list")
.find("a.ch-link")
.each(function (result) {
const chapterName = $(this).text().replace("~ ", "");
const releaseDate = null;
let chapterUrl = $(this).attr("href");
chapterUrl = chapterUrl.replace(
`https://www.mtlnovel.com/${novelUrl}/`,
""
);
novelChapters.push({
chapterUrl,
chapterName,
releaseDate,
});
});
return novelChapters.reverse();
};
novel.chapters = await getChapters();
return novel;
};
const parseChapter = async (novelUrl, chapterUrl) => {
const url = chapterUrl;
console.log(url);
const result = await fetch(url);
const body = await result.text();
$ = cheerio.load(body);
const chapterName = $("h1.main-title").text();
let chapterText = $("div.par").html();
chapterText = htmlToText(chapterText);
let nextChapter = null;
if ($("a.next").attr("href")) {
nextChapter = $("a.next").attr("href").replace(`/${novelUrl}/`, "");
}
let prevChapter = null;
if ($("a.prev").attr("href")) {
prevChapter = $("a.prev").attr("href").replace(`/${novelUrl}/`, "");
}
chapter = {
sourceId: 5,
novelUrl,
chapterUrl,
chapterName,
chapterText,
nextChapter,
prevChapter,
};
return chapter;
};
const searchNovels = async (searchTerm) => {
const searchUrl =
"https://www.mtlnovel.com/wp-admin/admin-ajax.php?action=autosuggest&q=" +
searchTerm +
"&__amp_source_origin=https%3A%2F%2Fwww.mtlnovel.com";
const res = await fetch(searchUrl);
const result = await res.text();
$ = cheerio.load(result);
let body = JSON.parse($("body").text());
let novels = [];
body.items[0].results.map((item) => {
const novelName = item.title.replace(/<\/?strong>/g, "");
const novelCover = item.thumbnail;
const novelUrl = item.permalink.replace(
"https://www.mtlnovel.com/",
""
);
const novel = { sourceId: 5, novelName, novelCover, novelUrl };
novels.push(novel);
});
return novels;
};
const mtlNovelScraper = {
popularNovels,
parseNovelAndChapters,
parseChapter,
searchNovels,
};
export default mtlNovelScraper;
+206
View File
@@ -0,0 +1,206 @@
import cheerio from "react-native-cheerio";
import { htmlToText } from "../helpers/htmlToText";
const baseUrl = "https://novelfull.com";
const popularNovels = async (page) => {
const url = `${baseUrl}/most-popular?page=${page}`;
const result = await fetch(url);
const body = await result.text();
$ = cheerio.load(body);
let novels = [];
$("div.col-truyen-main > div.list-truyen > .row").each(function (result) {
const novelName = $(this).find("h3.truyen-title > a").text();
let novelCover = $(this).find("img").attr("src");
novelCover = baseUrl + novelCover;
let novelUrl = $(this)
.find("h3.truyen-title > a")
.attr("href")
.replace(".html", "")
.substring(1);
novelUrl = `${novelUrl}/`;
const novel = {
sourceId: 8,
novelUrl,
novelName,
novelCover,
};
novels.push(novel);
});
return novels;
};
const parseNovelAndChapters = async (novelUrl) => {
const url = `${baseUrl}/${novelUrl.slice(0, -1)}.html`;
const result = await fetch(url);
const body = await result.text();
$ = cheerio.load(body);
let novel = {};
novel.sourceId = 8;
novel.sourceName = "NovelFull";
novel.url = url;
novel.novelUrl = novelUrl;
novel.novelName = $("div.book > img").attr("alt");
novel.novelCover = baseUrl + $("div.book > img").attr("src");
novel.summary = $("div.desc-text").text();
novel.author = $("div.info > div > h3")
.filter(function () {
return $(this).text().trim() === "Author:";
})
.siblings()
.text();
novel.genre = $("div.info > div")
.filter(function () {
return $(this).find("h3").text().trim() === "Genre:";
})
.text()
.replace("Genre:", "");
novel.artist = null;
novel.Status = $("div.info > div > h3")
.filter(function () {
return $(this).text().trim() === "Status:";
})
.next()
.text();
const novelId = $("#rating").attr("data-novel-id");
const getChapters = async (novelId) => {
const chapterListUrl =
baseUrl + "/ajax/chapter-option?novelId=" + novelId;
const data = await fetch(chapterListUrl);
const chapters = await data.text();
$ = cheerio.load(chapters);
let novelChapters = [];
$("select > option").each(function (result) {
let chapterName = $(this).text();
let releaseDate = null;
let chapterUrl = $(this).attr("value");
chapterUrl = chapterUrl.replace(`/${novelUrl}/`, "");
novelChapters.push({
chapterName,
releaseDate,
chapterUrl,
});
});
return novelChapters;
};
novel.chapters = await getChapters(novelId);
return novel;
};
const parseChapter = async (novelUrl, chapterUrl) => {
const url = `${baseUrl}${chapterUrl}`;
const result = await fetch(url);
const body = await result.text();
$ = cheerio.load(body);
const chapterName = $(".chapter-title").attr("title");
let chapterText = $("#chapter-content").html();
chapterText = htmlToText(chapterText);
let nextChapter = null;
if ($("a#next_chap").attr("href")) {
nextChapter = $("a#next_chap")
.attr("href")
.replace(`/${novelUrl}/`, "");
}
let prevChapter = null;
if ($("a#prev_chap").attr("href")) {
prevChapter = $("a#prev_chap")
.attr("href")
.replace(`/${novelUrl}/`, "");
}
const chapter = {
sourceId: 8,
novelUrl,
chapterUrl,
chapterName,
chapterText,
nextChapter,
prevChapter,
};
return chapter;
};
const searchNovels = async (searchTerm) => {
const searchUrl = `https://novelfull.com/search?keyword=`;
const url = `${searchUrl}${searchTerm}`;
const result = await fetch(url);
const body = await result.text();
$ = cheerio.load(body);
let novels = [];
$("div.col-truyen-main > div.list-truyen > .row").each(function (result) {
let novelUrl = $(this)
.find("h3.truyen-title > a")
.attr("href")
.replace(".html", "")
.substring(1);
novelUrl = `${novelUrl}/`;
const novelName = $(this).find("h3.truyen-title > a").text();
let novelCover = $(this).find("img").attr("src");
novelCover = baseUrl + novelCover;
const novel = {
sourceId: 8,
novelUrl,
novelName,
novelCover,
};
novels.push(novel);
});
return novels;
};
const novelFullScraper = {
popularNovels,
parseNovelAndChapters,
parseChapter,
searchNovels,
};
export default novelFullScraper;
+178
View File
@@ -0,0 +1,178 @@
import cheerio from "react-native-cheerio";
import { htmlToText } from "../helpers/htmlToText";
const baseUrl = "https://www.novelhall.com/";
const popularNovels = async (page) => {
const result = await fetch(baseUrl);
const body = await result.text();
$ = cheerio.load(body);
let novels = [];
$("div.section1")
.find("li")
.each(function (result) {
const novelName = $(this).find(".book-info > h2 > a").text();
const novelCover = $(this).find("img").attr("src");
const novelUrl = $(this).find("a").attr("href").substring(1);
const novel = {
sourceId: 6,
novelName,
novelCover,
novelUrl,
};
novels.push(novel);
});
return novels;
};
const parseNovelAndChapters = async (novelUrl) => {
const url = `${baseUrl}${novelUrl}/`;
const result = await fetch(url);
const body = await result.text();
$ = cheerio.load(body);
let novel = {};
novel.sourceId = 6;
novel.sourceName = "NovelHall";
novel.url = url;
novel.novelUrl = novelUrl;
novel.novelName = $("h1").text();
novel.novelCover = $("div.book-img > img").attr("src");
novel.summary = $("div.intro")
.text()
.replace(/[\t\n]/g, "");
novel.author = $("span.blue").first().text().replace("Author:", "");
novel.genre = $("a.red").text();
novel.artist = null;
novel.Status = $("span.blue").first().next().text().replace("Status:", "");
let novelChapters = [];
$("div.book-catalog.hidden-xs#morelist")
.find("li.post-11")
.each(function (result) {
let chapterName = $(this).find("a").text();
let releaseDate = null;
let chapterUrl = $(this).find("a").attr("href");
if (chapterUrl) {
chapterUrl = chapterUrl.replace(`/${novelUrl}/`, "");
}
const chapter = {
chapterName,
releaseDate,
chapterUrl,
};
novelChapters.push(chapter);
});
novel.chapters = novelChapters;
return novel;
};
const parseChapter = async (novelUrl, chapterUrl) => {
const url = `${baseUrl}${novelUrl}/${chapterUrl}`;
const result = await fetch(url);
const body = await result.text();
$ = cheerio.load(body);
const chapterName = $("h1").text();
let chapterText = $("div.entry-content").html();
chapterText = htmlToText(chapterText);
let nextChapter = null;
nextChapter = $('a[rel="next"]').attr("href");
if (nextChapter) {
nextChapter = nextChapter.replace(`/${novelUrl}/`, "");
}
let prevChapter = null;
prevChapter = $('a[rel="prev"]').attr("href");
if (prevChapter) {
prevChapter = prevChapter.replace(`/${novelUrl}/`, "");
}
chapter = {
sourceId: 6,
novelUrl,
chapterUrl,
chapterName,
chapterText,
nextChapter,
prevChapter,
};
return chapter;
};
const searchNovels = async (searchTerm) => {
const searchUrl = `${baseUrl}index.php?s=so&module=book&keyword=`;
const url = `${searchUrl}${searchTerm}`;
const result = await fetch(url);
const body = await result.text();
$ = cheerio.load(body);
let novels = [];
$("tr").each(function (result) {
let novelName = $(this).find("td:nth-child(2)").text();
novelName = novelName.replace(/[\t\n]/g, "");
const novelCover = "https://cdn.novelupdates.com/imgmid/noimagemid.jpg";
let novelUrl = $(this).find("td:nth-child(2) >").attr("href");
if (novelUrl) {
novelUrl = novelUrl.slice(1);
}
novel = {
sourceId: 6,
novelName,
novelCover,
novelUrl,
};
novels.push(novel);
});
return novels;
};
const novelhallScraper = {
popularNovels,
parseNovelAndChapters,
parseChapter,
searchNovels,
};
export default novelhallScraper;
+200
View File
@@ -0,0 +1,200 @@
import cheerio from "react-native-cheerio";
import { htmlToText } from "../helpers/htmlToText";
const baseUrl = "https://noveltrench.com/";
const popularNovels = async (page) => {
let url = baseUrl + "manga/page/" + page;
const result = await fetch(url);
const body = await result.text();
$ = cheerio.load(body);
let novels = [];
$(".page-item-detail").each(function (result) {
const novelName = $(this).find(".h5 > a").text();
const novelCover = $(this).find("img").attr("data-src");
let novelUrl = $(this).find(".h5 > a").attr("href");
novelUrl = novelUrl.replace(`${baseUrl}manga/`, "");
const novel = {
sourceId: 9,
novelName,
novelCover,
novelUrl,
};
novels.push(novel);
});
return novels;
};
const parseNovelAndChapters = async (novelUrl) => {
const url = `${baseUrl}manga/${novelUrl}/`;
const result = await fetch(url);
const body = await result.text();
$ = cheerio.load(body);
let novel = {};
novel.sourceId = 9;
novel.sourceName = "NovelTrench";
novel.url = url;
novel.novelUrl = novelUrl;
novel.novelName = $(".post-title > h1")
.text()
.replace(/[\t\n]/g, "")
.trim();
novel.novelCover = $(".summary_image > a > img").attr("data-src");
$(".post-content_item").each(function (result) {
detailName = $(this)
.find(".summary-heading > h5")
.text()
.replace(/[\t\n]/g, "")
.trim();
detail = $(this)
.find(".summary-content")
.text()
.replace(/[\t\n]/g, "")
.trim();
switch (detailName) {
case "Genre(s)":
novel.genre = detail.trim().replace(/[\t\n]/g, ",");
break;
case "Author(s)":
novel.author = detail.trim();
break;
case "Artist(s)":
novel.status = detail.trim();
break;
}
});
$(".description-summary > div.summary__content").find("em").remove();
novel.summary = $(".description-summary > div.summary__content")
.text()
.replace(/[\t\n]/g, "");
let novelChapters = [];
$(".wp-manga-chapter").each(function (result) {
chapterName = $(this)
.find("a")
.text()
.replace(/[\t\n]/g, "")
.trim();
releaseDate = $(this)
.find("span")
.text()
.replace(/[\t\n]/g, "")
.trim();
chapterUrl = $(this).find("a").attr("href").replace(url, "");
novelChapters.push({ chapterName, releaseDate, chapterUrl });
});
novel.chapters = novelChapters.reverse();
return novel;
};
const parseChapter = async (novelUrl, chapterUrl) => {
const url = chapterUrl;
console.log(url);
const result = await fetch(url);
const body = await result.text();
$ = cheerio.load(body);
const chapterName = $("h1#chapter-heading").text();
let chapterText = $(".reading-content").html();
chapterText = htmlToText(chapterText);
let nextChapter = null;
if ($(".nav-next").length) {
nextChapter = $(".nav-next")
.find("a")
.attr("href")
.replace(baseUrl + "manga/" + novelUrl + "/", "");
}
let prevChapter = null;
if ($(".nav-previous").length) {
prevChapter = $(".nav-previous")
.find("a")
.attr("href")
.replace(baseUrl + "manga/" + novelUrl + "/", "");
}
const chapter = {
sourceId: 9,
novelUrl,
chapterUrl,
chapterName,
chapterText,
nextChapter,
prevChapter,
};
return chapter;
};
const searchNovels = async (searchTerm) => {
const url = `${baseUrl}?s=${searchTerm}&post_type=wp-manga`;
const result = await fetch(url);
const body = await result.text();
$ = cheerio.load(body);
let novels = [];
$(".c-tabs-item__content").each(function (result) {
const novelName = $(this).find(".h4 > a").text();
const novelCover = $(this).find("img").attr("data-src");
let novelUrl = $(this).find(".h4 > a").attr("href");
novelUrl = novelUrl.replace(`${baseUrl}manga/`, "");
const novel = {
sourceId: 9,
novelName,
novelCover,
novelUrl,
};
novels.push(novel);
});
return novels;
};
const novelTrenchScraper = {
popularNovels,
parseNovelAndChapters,
parseChapter,
searchNovels,
};
export default novelTrenchScraper;
+159
View File
@@ -0,0 +1,159 @@
import cheerio from "react-native-cheerio";
import { htmlToText } from "../helpers/htmlToText";
const baseUrl = "https://www.novelupdates.cc/";
const popularNovels = async (page) => {
const result = await fetch(baseUrl);
const body = await result.text();
$ = cheerio.load(body);
let novels = [];
$(".section-item").each(function (result) {
const novelName = $(this).find(".book-name").text();
const novelCover = $(this).find("img").attr("src");
let novelUrl = $(this).find("a").attr("href").slice(1);
const novel = {
sourceId: 18,
novelName,
novelCover,
novelUrl,
};
novels.push(novel);
});
return novels;
};
const parseNovelAndChapters = async (novelUrl) => {
const url = `${baseUrl}${novelUrl}/`;
const result = await fetch(url);
const body = await result.text();
$ = cheerio.load(body);
let novel = {};
novel.sourceId = 18;
novel.sourceName = "NovelUpdates.cc";
novel.url = url;
novel.novelUrl = novelUrl;
novel.novelName = $(".book-name").text();
novel.novelCover = $("div.book-img > img").attr("src");
novel.genre = $("div.book-catalog > span.txt").text();
novel.Status = $("div.book-state > span.txt").text();
novel.author = $("div.author > span.name").text();
novelSummary = $("div.content > p.desc").html();
novel.summary = htmlToText(novelSummary);
let novelChapters = [];
$(".chapter-item").each(function (result) {
const chapterName = $(this).find(".chapter-name").text();
const releaseDate = null;
const chapterUrl = $(this).attr("href").slice(1);
novelChapters.push({
chapterName,
releaseDate,
chapterUrl,
});
});
novel.chapters = novelChapters;
return novel;
};
const parseChapter = async (novelUrl, chapterUrl) => {
const url = `${baseUrl}${chapterUrl}`;
const result = await fetch(url);
const body = await result.text();
$ = cheerio.load(body);
const chapterName = $("h1.chapter-title").text();
let chapterText = $("div.chapter-entity").html();
chapterText = htmlToText(chapterText);
let nextChapter = null;
if ($("a.next").length) {
nextChapter = $("a.next").attr("href").slice(1);
}
let prevChapter = null;
if ($("a.prev").length) {
prevChapter = $("a.prev").attr("href").slice(1);
}
const chapter = {
sourceId: 18,
novelUrl,
chapterUrl,
chapterName,
chapterText,
nextChapter,
prevChapter,
};
return chapter;
};
const searchNovels = async (searchTerm) => {
const url = `${baseUrl}search/${searchTerm}/1`;
const result = await fetch(url);
const body = await result.text();
$ = cheerio.load(body);
let novels = [];
$(".list-item").each(function (result) {
$(this).find("font").remove();
const novelName = $(this).find(".book-name").text();
const novelCover = $(this).find("img").attr("src");
let novelUrl = $(this).find("a").attr("href").slice(1);
const novel = {
sourceId: 18,
novelName,
novelCover,
novelUrl,
};
novels.push(novel);
});
return novels;
};
const novelUpdatesCcScraper = {
popularNovels,
parseNovelAndChapters,
parseChapter,
searchNovels,
};
export default novelUpdatesCcScraper;
+159
View File
@@ -0,0 +1,159 @@
import cheerio from "react-native-cheerio";
import { htmlToText } from "../helpers/htmlToText";
const baseUrl = "https://m.readlightnovel.cc/";
const popularNovels = async (page) => {
const result = await fetch(baseUrl);
const body = await result.text();
$ = cheerio.load(body);
let novels = [];
$(".section-item").each(function (result) {
const novelName = $(this).find(".book-name").text();
const novelCover = $(this).find("img").attr("src");
let novelUrl = $(this).find("a").attr("href").slice(1);
const novel = {
sourceId: 19,
novelName,
novelCover,
novelUrl,
};
novels.push(novel);
});
return novels;
};
const parseNovelAndChapters = async (novelUrl) => {
const url = `${baseUrl}${novelUrl}/`;
const result = await fetch(url);
const body = await result.text();
$ = cheerio.load(body);
let novel = {};
novel.sourceId = 19;
novel.sourceName = "ReadLightNovel.cc";
novel.url = url;
novel.novelUrl = novelUrl;
novel.novelName = $(".book-name").text();
novel.novelCover = $("div.book-img > img").attr("src");
novel.genre = $("div.book-catalog > span.txt").text();
novel.Status = $("div.book-state > span.txt").text();
novel.author = $("div.author > span.name").text();
novelSummary = $("div.content > p.desc").html();
novel.summary = htmlToText(novelSummary);
let novelChapters = [];
$(".chapter-item").each(function (result) {
const chapterName = $(this).find(".chapter-name").text();
const releaseDate = null;
const chapterUrl = $(this).attr("href").slice(1);
novelChapters.push({
chapterName,
releaseDate,
chapterUrl,
});
});
novel.chapters = novelChapters;
return novel;
};
const parseChapter = async (novelUrl, chapterUrl) => {
const url = `${baseUrl}${chapterUrl}`;
const result = await fetch(url);
const body = await result.text();
$ = cheerio.load(body);
const chapterName = $("h1.chapter-title").text();
let chapterText = $("div.chapter-entity").html();
chapterText = htmlToText(chapterText);
let nextChapter = null;
if ($("a.next").length) {
nextChapter = $("a.next").attr("href").slice(1);
}
let prevChapter = null;
if ($("a.prev").length) {
prevChapter = $("a.prev").attr("href").slice(1);
}
const chapter = {
sourceId: 19,
novelUrl,
chapterUrl,
chapterName,
chapterText,
nextChapter,
prevChapter,
};
return chapter;
};
const searchNovels = async (searchTerm) => {
const url = `${baseUrl}search/${searchTerm}/1`;
const result = await fetch(url);
const body = await result.text();
$ = cheerio.load(body);
let novels = [];
$(".list-item").each(function (result) {
$(this).find("font").remove();
const novelName = $(this).find(".book-name").text();
const novelCover = $(this).find("img").attr("src");
let novelUrl = $(this).find("a").attr("href").slice(1);
const novel = {
sourceId: 19,
novelName,
novelCover,
novelUrl,
};
novels.push(novel);
});
return novels;
};
const readLightNovelCcScraper = {
popularNovels,
parseNovelAndChapters,
parseChapter,
searchNovels,
};
export default readLightNovelCcScraper;
+205
View File
@@ -0,0 +1,205 @@
import cheerio from "react-native-cheerio";
import { htmlToText } from "../helpers/htmlToText";
const baseUrl = "https://readnovelfull.com";
const popularNovels = async (page) => {
const url = `${baseUrl}/most-popular-novel?page=${page}`;
const result = await fetch(url);
const body = await result.text();
$ = cheerio.load(body);
let novels = [];
$("div.col-novel-main > div.list-novel > .row").each(function (result) {
let novelUrl = $(this)
.find("h3.novel-title > a")
.attr("href")
.replace(".html", "")
.substring(1);
novelUrl = `${novelUrl}/`;
const novelName = $(this).find("h3.novel-title > a").text();
const novelCover = $(this).find("img").attr("src");
const novel = {
sourceId: 4,
novelUrl,
novelName,
novelCover,
};
novels.push(novel);
});
return novels;
};
const parseNovelAndChapters = async (novelUrl) => {
const url = `${baseUrl}/${novelUrl.slice(0, -1)}.html`;
const result = await fetch(url);
const body = await result.text();
$ = cheerio.load(body);
let novel = {};
novel.sourceId = 4;
novel.sourceName = "ReadNovelFull";
novel.url = url;
novel.novelUrl = novelUrl;
novel.novelName = $("div.book > img").attr("alt");
novel.novelCover = $("div.book > img").attr("src");
novel.summary = $("div.desc-text").text();
novel.author = $("li > h3")
.filter(function () {
return $(this).text().trim() === "Author:";
})
.siblings()
.text();
novel.genre = $("li")
.filter(function () {
return $(this).find("h3").text().trim() === "Genre:";
})
.text()
.replace("Genre:", "");
novel.artist = null;
novel.Status = $("li > h3")
.filter(function () {
return $(this).text().trim() === "Status:";
})
.siblings()
.text();
const novelId = $("#rating").attr("data-novel-id");
const getChapters = async (novelId) => {
const chapterListUrl =
"https://readnovelfull.com/ajax/chapter-archive?novelId=" + novelId;
const data = await fetch(chapterListUrl);
const chapters = await data.text();
$ = cheerio.load(chapters);
let novelChapters = [];
$(".panel-body")
.find("li")
.each(function (result) {
let chapterName = $(this).find("a").attr("title");
let releaseDate = null;
let chapterUrl = $(this).find("a").attr("href");
chapterUrl = chapterUrl.replace(`/${novelUrl}/`, "");
novelChapters.push({
chapterName,
releaseDate,
chapterUrl,
});
});
return novelChapters;
};
novel.chapters = await getChapters(novelId);
return novel;
};
const parseChapter = async (novelUrl, chapterUrl) => {
const url = `${baseUrl}${chapterUrl}`;
const result = await fetch(url);
const body = await result.text();
$ = cheerio.load(body);
const chapterName = $(".chr-title").attr("title");
let chapterText = $("#chr-content").html();
chapterText = htmlToText(chapterText);
let nextChapter = null;
if ($("a#next_chap").attr("href")) {
nextChapter = $("a#next_chap")
.attr("href")
.replace(`/${novelUrl}/`, "");
}
let prevChapter = null;
if ($("a#prev_chap").attr("href")) {
prevChapter = $("a#prev_chap")
.attr("href")
.replace(`/${novelUrl}/`, "");
}
const chapter = {
sourceId: 4,
novelUrl,
chapterUrl,
chapterName,
chapterText,
nextChapter,
prevChapter,
};
return chapter;
};
const searchNovels = async (searchTerm) => {
const searchUrl = `https://readnovelfull.com/search?keyword=`;
const url = `${searchUrl}${searchTerm}`;
const result = await fetch(url);
const body = await result.text();
$ = cheerio.load(body);
let novels = [];
$("div.col-novel-main > div.list-novel > .row").each(function (result) {
let novelUrl = $(this)
.find("h3.novel-title > a")
.attr("href")
.replace(".html", "")
.substring(1);
novelUrl = `${novelUrl}/`;
const novelName = $(this).find("h3.novel-title > a").text();
const novelCover = $(this).find("img").attr("src");
const novel = {
sourceId: 4,
novelUrl,
novelName,
novelCover,
};
novels.push(novel);
});
return novels;
};
const readNovelFullScraper = {
popularNovels,
parseNovelAndChapters,
parseChapter,
searchNovels,
};
export default readNovelFullScraper;
+175
View File
@@ -0,0 +1,175 @@
import cheerio from "react-native-cheerio";
import { htmlToText } from "../helpers/htmlToText";
const baseUrl = "http://www.tapread.com";
const popularNovels = async (page) => {
const result = await fetch(baseUrl);
const body = await result.text();
$ = cheerio.load(body);
let novels = [];
$('.section-item[data-classify="1"]').each(function (result) {
const novelName = $(this).find(".book-name").text();
const novelCover = "http:" + $(this).find("img").attr("src");
let novelUrl = $(this).attr("data-id");
novelUrl += "/";
const novel = {
sourceId: 17,
novelName,
novelCover,
novelUrl,
};
novels.push(novel);
});
return novels;
};
const parseNovelAndChapters = async (novelUrl) => {
const url = `${baseUrl}/book/detail/${novelUrl}`;
const result = await fetch(url);
const body = await result.text();
$ = cheerio.load(body);
let novel = {};
novel.sourceId = 17;
novel.sourceName = "TapRead";
novel.url = url;
novel.novelUrl = novelUrl;
novel.novelName = $(".book-name").text();
novel.novelCover = "http:" + $("div.book-img > img").attr("src");
novel.genre = $("div.book-catalog > span.txt").text();
novel.Status = $("div.book-state > span.txt").text();
novel.author = $("div.author > span.name").text();
novelSummary = $("div.content > p.desc").text();
const getChapters = async (novelId) => {
const chapterListUrl = "http://www.tapread.com/ajax/book/contents";
let formData = new FormData();
formData.append("bookId", novelId);
const data = await fetch(chapterListUrl, {
method: "POST",
body: formData,
});
const chapters = await data.json();
let novelChapters = [];
chapters.result.chapterList.map((chapter) => {
let chapterName = chapter.chapterName;
const releaseDate = chapter.pubTime;
const chapterUrl = chapter.chapterId.toString();
if (chapter.lock) {
chapterName = "🔒 " + chapterName;
}
novelChapters.push({
chapterUrl,
chapterName,
releaseDate,
});
});
return novelChapters;
};
novel.chapters = await getChapters(novelUrl);
return novel;
};
const parseChapter = async (novelUrl, chapterUrl) => {
let formData = new FormData();
formData.append("bookId", novelUrl);
formData.append("chapterId", chapterUrl);
const url = "http://www.tapread.com/ajax/book/chapter";
const result = await fetch(url, {
method: "POST",
body: formData,
});
const body = await result.json();
const chapterName = body.result.chapterName;
const chapterText = htmlToText(body.result.content);
const nextChapter =
body.result.nextChapterId === 0
? null
: body.result.nextChapterId.toString();
const prevChapter = body.result.preChapterId.toString();
novelUrl += "/";
const chapter = {
sourceId: 17,
novelUrl,
chapterUrl,
chapterName,
chapterText,
nextChapter,
prevChapter,
};
return chapter;
};
const searchNovels = async (searchTerm) => {
let formData = new FormData();
formData.append("storyType", 1);
formData.append("pageNo", 1);
formData.append("searchText", searchTerm);
const url = "http://www.tapread.com/ajax/search/story";
const data = await fetch(url, {
method: "POST",
body: formData,
});
const body = await data.json();
const novels = [];
body.result.storyList.map((novel) => {
const novelName = novel.storyName.replace(
'<font color="#FFCE2E">Demon</font>',
""
);
const novelUrl = novel.storyId + "/";
const novelCover = "http://static.tapread.com" + novel.coverUrl;
novels.push({ sourceId: 17, novelName, novelCover, novelUrl });
});
return novels;
};
const tapReadScraper = {
popularNovels,
parseNovelAndChapters,
parseChapter,
searchNovels,
};
export default tapReadScraper;
+204
View File
@@ -0,0 +1,204 @@
import cheerio from "react-native-cheerio";
import { htmlToText } from "../helpers/htmlToText";
const baseUrl = "https://vipnovel.com/";
const popularNovels = async (page) => {
let url = baseUrl;
const result = await fetch(url);
const body = await result.text();
$ = cheerio.load(body);
let novels = [];
$(".page-item-detail").each(function (result) {
const novelName = $(this).find("h5 > a").text();
const novelCover = $(this).find("img").attr("src");
let novelUrl = $(this).find("h5 > a").attr("href");
novelUrl = novelUrl.replace(`${baseUrl}vipnovel/`, "");
const novel = {
sourceId: 10,
novelName,
novelCover,
novelUrl,
};
novels.push(novel);
});
return novels;
};
const parseNovelAndChapters = async (novelUrl) => {
const url = `${baseUrl}vipnovel/${novelUrl}/`;
const result = await fetch(url);
const body = await result.text();
$ = cheerio.load(body);
let novel = {};
novel.sourceId = 10;
novel.sourceName = "VipNovel";
novel.url = url;
novel.novelUrl = novelUrl;
novel.novelName = $(".post-title > h3")
.text()
.replace(/[\t\n]/g, "")
.trim();
novel.novelCover = $(".summary_image > a > img").attr("src");
$(".post-content_item").each(function (result) {
detailName = $(this)
.find(".summary-heading > h5")
.text()
.replace(/[\t\n]/g, "")
.trim();
detail = $(this)
.find(".summary-content")
.text()
.replace(/[\t\n]/g, "")
.trim();
switch (detailName) {
case "Genre(s)":
novel.genre = detail.trim().replace(/[\t\n]/g, ",");
break;
case "Author(s)":
novel.author = detail.trim();
break;
case "Artist(s)":
novel.status = detail.trim();
break;
}
});
// $(".description-summary > div.summary__content").find("em").remove();
let novelSummary = $(".description-summary > div.summary__content").html();
novel.summary = htmlToText(novelSummary);
let novelChapters = [];
$(".wp-manga-chapter").each(function (result) {
const chapterName = $(this)
.find("a")
.text()
.replace(/[\t\n]/g, "")
.trim();
const releaseDate = $(this)
.find("span")
.text()
.replace(/[\t\n]/g, "")
.trim();
const chapterUrl = $(this).find("a").attr("href").replace(url, "");
const chapter = { chapterName, releaseDate, chapterUrl };
novelChapters.push(chapter);
});
novel.chapters = novelChapters.reverse();
return novel;
};
const parseChapter = async (novelUrl, chapterUrl) => {
const url = `${baseUrl}vipnovel/${novelUrl}/${chapterUrl}`;
const result = await fetch(url);
const body = await result.text();
$ = cheerio.load(body);
let chapterName = $(".text-center").text();
if (!chapterName) {
chapterName = $(".text-left > div> h3").text();
}
let chapterText = $(".text-left").html();
chapterText = htmlToText(chapterText);
let nextChapter = null;
if ($(".nav-next").length) {
nextChapter = $(".nav-next")
.find("a")
.attr("href")
.replace(baseUrl + "vipnovel/" + novelUrl + "/", "");
}
let prevChapter = null;
if ($(".nav-previous").length) {
prevChapter = $(".nav-previous")
.find("a")
.attr("href")
.replace(baseUrl + "vipnovel/" + novelUrl + "/", "");
}
const chapter = {
sourceId: 10,
novelUrl,
chapterUrl,
chapterName,
chapterText,
nextChapter,
prevChapter,
};
return chapter;
};
const searchNovels = async (searchTerm) => {
const url = `${baseUrl}?s=${searchTerm}&post_type=wp-manga`;
const result = await fetch(url);
const body = await result.text();
$ = cheerio.load(body);
let novels = [];
$(".c-tabs-item__content").each(function (result) {
const novelName = $(this).find("h4 > a").text();
const novelCover = $(this).find("img").attr("src");
let novelUrl = $(this).find("h4 > a").attr("href");
novelUrl = novelUrl.replace(`${baseUrl}vipnovel/`, "");
const novel = {
sourceId: 10,
novelName,
novelCover,
novelUrl,
};
novels.push(novel);
});
return novels;
};
const vipNovelScraper = {
popularNovels,
parseNovelAndChapters,
parseChapter,
searchNovels,
};
export default vipNovelScraper;
+202
View File
@@ -0,0 +1,202 @@
import cheerio from "react-native-cheerio";
import { htmlToText } from "../helpers/htmlToText";
const baseUrl = "https://woopread.com/";
const popularNovels = async (page) => {
let url = baseUrl + "novellist/?m_orderby=rating";
const result = await fetch(url);
const body = await result.text();
$ = cheerio.load(body);
let novels = [];
$(".page-item-detail").each(function (result) {
const novelName = $(this).find(".h5 > a").text();
const novelCover = $(this).find("img").attr("src");
let novelUrl = $(this).find(".h5 > a").attr("href");
novelUrl = novelUrl.replace(baseUrl + "series/", "");
const novel = {
sourceId: 21,
novelName,
novelCover,
novelUrl,
};
novels.push(novel);
});
return novels;
};
const parseNovelAndChapters = async (novelUrl) => {
const url = `${baseUrl}series/${novelUrl}/`;
const result = await fetch(url);
const body = await result.text();
$ = cheerio.load(body);
let novel = {};
novel.sourceId = 21;
novel.sourceName = "WoopRead";
novel.url = url;
novel.novelUrl = novelUrl;
novel.novelName = $(".post-title > h1")
.text()
.replace(/[\t\n]/g, "")
.trim();
novel.novelCover = $(".summary_image > a > img").attr("src");
$(".post-content_item").each(function (result) {
detailName = $(this)
.find(".summary-heading > h5")
.text()
.replace(/[\t\n]/g, "")
.trim();
detail = $(this)
.find(".summary-content")
.text()
.replace(/[\t\n]/g, "")
.trim();
novel[detailName] = detail;
});
$(".description-summary > div.summary__content").find("em").remove();
novel.summary = $(".description-summary > div.summary__content")
.text()
.replace(/[\t\n]/g, "");
let novelChapters = [];
const novelId = $(".rating-post-id").attr("value");
let formData = new FormData();
formData.append("action", "manga_get_chapters");
formData.append("manga", novelId);
const data = await fetch("https://woopread.com/wp-admin/admin-ajax.php", {
method: "POST",
body: formData,
});
const text = await data.text();
$ = cheerio.load(text);
$(".wp-manga-chapter.free-chap").each(function (result) {
chapterName = $(this)
.find("a")
.text()
.replace(/[\t\n]/g, "")
.trim();
releaseDate = $(this).find("span").text().replace("Free", "").trim();
chapterUrl = $(this).find("a").attr("href").replace(url, "");
novelChapters.push({ chapterName, releaseDate, chapterUrl });
});
novel.chapters = novelChapters.reverse();
return novel;
};
const parseChapter = async (novelUrl, chapterUrl) => {
const url = `${baseUrl}series/${novelUrl}/${chapterUrl}/`;
const result = await fetch(url);
const body = await result.text();
$ = cheerio.load(body);
const chapterName = $("h1#chapter-heading").text();
let chapterText = $(".reading-content").html();
chapterText = htmlToText(chapterText);
chapterText = chapterText.replace(/(?<=[[])[\n](?=[h])/g, "");
let nextChapter = null;
if ($(".nav-next").length) {
nextChapter = $(".nav-next")
.find("a")
.attr("href")
.replace(baseUrl + "series/" + novelUrl + "/", "");
}
let prevChapter = null;
if ($(".nav-previous").length) {
prevChapter = $(".nav-previous")
.find("a")
.attr("href")
.replace(baseUrl + "series/" + novelUrl + "/", "");
}
novelUrl += "/";
chapterUrl = optionalUrl + chapterUrl + "/";
const chapter = {
sourceId: 21,
novelUrl,
chapterUrl,
chapterName,
chapterText,
nextChapter,
prevChapter,
};
return chapter;
};
const searchNovels = async (searchTerm) => {
const url = `${baseUrl}?s=${searchTerm}&post_type=wp-manga&op=&author=&artist=&release=&adult=`;
const result = await fetch(url);
const body = await result.text();
$ = cheerio.load(body);
let novels = [];
$(".c-tabs-item__content").each(function (result) {
const novelName = $(this).find(".h4 > a").text();
const novelCover = $(this).find("img").attr("src");
let novelUrl = $(this).find(".h4 > a").attr("href");
novelUrl = novelUrl.replace(`${baseUrl}series/`, "");
const novel = {
sourceId: 21,
novelName,
novelCover,
novelUrl,
};
novels.push(novel);
});
return novels;
};
const WoopReadScraper = {
popularNovels,
parseNovelAndChapters,
parseChapter,
searchNovels,
};
export default WoopReadScraper;
+179
View File
@@ -0,0 +1,179 @@
import cheerio from "react-native-cheerio";
import { htmlToText } from "../helpers/htmlToText";
const baseUrl = "https://www.wuxiaworld.com/";
const popularNovels = async (page) => {
const url = `${baseUrl}api/novels`;
const result = await fetch(url);
const data = await result.json();
let novels = [];
data.items.map((novel) => {
let novelName = novel.name;
let novelCover = novel.coverUrl;
let novelUrl = novel.slug;
novels.push({
sourceId: 7,
novelUrl,
novelName,
novelCover,
});
});
return novels;
};
const parseNovelAndChapters = async (novelUrl) => {
const url = `${baseUrl}novel/${novelUrl}/`;
const result = await fetch(url);
const body = await result.text();
$ = cheerio.load(body);
let novel = {};
novel.sourceId = 7;
novel.sourceName = "WuxiaWorld";
novel.url = url;
novel.novelUrl = novelUrl;
novel.novelName = $("h2").text();
novel.novelCover = $("img.img-thumbnail").attr("src");
novel.summary = $(".fr-view > p").text();
novel.author = $("div > dt")
.filter(function () {
return $(this).text().trim() === "Author:";
})
.next()
.text();
let genres = [];
$(".genres")
.find("div")
.each(function (res) {
genres.push($(this).find("a").text());
});
novel.genre = genres.join(",");
novel.artist = null;
novel.Status = null;
let novelChapters = [];
$(".chapter-item").each(function (result) {
let chapterName = $(this).text();
chapterName = chapterName.replace(/[\t\n]/g, "");
const releaseDate = null;
let chapterUrl = $(this).find("a").attr("href");
chapterUrl = chapterUrl.replace(`/novel/${novelUrl}/`, "");
const chapter = {
chapterName,
releaseDate,
chapterUrl,
};
novelChapters.push(chapter);
});
novel.chapters = novelChapters;
return novel;
};
const parseChapter = async (novelUrl, chapterUrl) => {
let url = `${baseUrl}novel/${novelUrl}/${chapterUrl}`;
const result = await fetch(url);
const body = await result.text();
$ = cheerio.load(body);
let chapterName = $("#sidebar-toggler-container").next().text();
chapterName = chapterName.replace(/[\t\n]/g, "");
$("#chapter-content > script").remove();
let chapterText = $("#chapter-content").html();
chapterText = htmlToText(chapterText);
let nextChapter = null;
if ($("li.next > a").attr("href") !== "#") {
nextChapter = $("li.next > a")
.attr("href")
.replace(`/novel/${novelUrl}`, "");
}
let prevChapter = null;
if ($("li.prev > a").attr("href") !== "#") {
prevChapter = $("li.prev > a")
.attr("href")
.replace(`/novel/${novelUrl}`, "");
}
const chapter = {
sourceId: 7,
novelUrl,
chapterUrl,
chapterName,
chapterText,
nextChapter,
prevChapter,
};
return chapter;
};
const searchNovels = async (searchTerm) => {
const searchUrl = `https://www.wuxiaworld.com/api/novels/search?query=`;
const url = `${searchUrl}${searchTerm}`;
const result = await fetch(url);
const data = await result.json();
let novels = [];
data.items.map((novel) => {
let novelName = novel.name;
let novelCover = novel.coverUrl;
let novelUrl = novel.slug;
novels.push({
sourceId: 7,
novelUrl,
novelName,
novelCover,
});
});
return novels;
};
const WuxiaWorldScraper = {
popularNovels,
parseNovelAndChapters,
parseChapter,
searchNovels,
};
export default WuxiaWorldScraper;
+200
View File
@@ -0,0 +1,200 @@
import cheerio from "react-native-cheerio";
import { htmlToText } from "../helpers/htmlToText";
const baseUrl = "http://wuxiaworld.cloud/";
const popularNovels = async (page) => {
const url = `${baseUrl}/popular-novel`;
const result = await fetch(url);
const body = await result.text();
$ = cheerio.load(body);
let novels = [];
$("div.col-novel-main > div.list-novel > .row").each(function (result) {
let novelUrl = $(this)
.find("h3.novel-title > a")
.attr("href")
.replace(baseUrl + "novel/", "");
novelUrl = `${novelUrl}/`;
const novelName = $(this).find("h3.novel-title > a").text();
const novelCover = $(this).find("img").attr("src");
const novel = {
sourceId: 20,
novelUrl,
novelName,
novelCover,
};
novels.push(novel);
});
return novels;
};
const parseNovelAndChapters = async (novelUrl) => {
const url = `${baseUrl}novel/${novelUrl}`;
const result = await fetch(url);
const body = await result.text();
$ = cheerio.load(body);
let novel = {};
novel.sourceId = 20;
novel.sourceName = "WuxiaWorld.cloud";
novel.url = url;
novel.novelUrl = novelUrl;
novel.novelName = $("div.book > img").attr("alt");
novel.novelCover = $("div.book > img").attr("src");
novel.summary = $("div.desc-text").text();
novel.author = $("div.info > div > h3")
.filter(function () {
return $(this).text().trim() === "Author:";
})
.siblings()
.text();
novel.genre = $("div.info > div")
.filter(function () {
return $(this).find("h3").text().trim() === "Genre:";
})
.text()
.replace("Genre:", "")
.replace(/\s/g, "");
novel.artist = null;
novel.Status = $("li > h3")
.filter(function () {
return $(this).text().trim() === "Status:";
})
.siblings()
.text();
const novelId = $("#rating").attr("data-novel-id");
const getChapters = async (novelId) => {
const chapterListUrl =
baseUrl + "ajax/chapter-option?novelId=" + novelId;
const data = await fetch(chapterListUrl);
const chapters = await data.text();
$ = cheerio.load(chapters);
let novelChapters = [];
$("select > option").each(function (result) {
let chapterName = $(this).text();
let releaseDate = null;
let chapterUrl = $(this).attr("value");
chapterUrl = chapterUrl.replace(baseUrl, "");
novelChapters.push({
chapterName,
releaseDate,
chapterUrl,
});
});
return novelChapters;
};
novel.chapters = await getChapters(novelId);
return novel;
};
const parseChapter = async (novelUrl, chapterUrl) => {
chapterUrl = encodeURI(chapterUrl);
const url = `${baseUrl}${chapterUrl}`;
const result = await fetch(url);
const body = await result.text();
$ = cheerio.load(body);
const chapterName = $(".chr-title").attr("title");
let chapterText = $("#chr-content").html();
chapterText = htmlToText(chapterText);
let nextChapter = null;
if ($("a#next_chap").attr("href")) {
nextChapter = $("a#next_chap").attr("href").replace(baseUrl, "");
}
let prevChapter = null;
if ($("a#prev_chap").attr("href")) {
prevChapter = $("a#prev_chap").attr("href").replace(baseUrl, "");
}
const chapter = {
sourceId: 20,
novelUrl,
chapterUrl,
chapterName,
chapterText,
nextChapter,
prevChapter,
};
return chapter;
};
const searchNovels = async (searchTerm) => {
const searchUrl = `http://wuxiaworld.cloud/search?keyword=`;
const url = `${searchUrl}${searchTerm}`;
const result = await fetch(url);
const body = await result.text();
$ = cheerio.load(body);
let novels = [];
$("div.col-novel-main > div.list-novel > .row").each(function (result) {
let novelUrl = $(this)
.find("h3.novel-title > a")
.attr("href")
.replace(baseUrl + "novel/", "");
novelUrl = `${novelUrl}/`;
const novelName = $(this).find("h3.novel-title > a").text();
const novelCover = $(this).find("img").attr("src");
const novel = {
sourceId: 20,
novelUrl,
novelName,
novelCover,
};
novels.push(novel);
});
return novels;
};
const WuxiaWorldCloudScraper = {
popularNovels,
parseNovelAndChapters,
parseChapter,
searchNovels,
};
export default WuxiaWorldCloudScraper;
+163
View File
@@ -0,0 +1,163 @@
import cheerio from "react-native-cheerio";
import { htmlToText } from "../helpers/htmlToText";
const baseUrl = "https://www.wuxiaworld.co/";
const popularNovels = async (page) => {
const result = await fetch(baseUrl);
const body = await result.text();
$ = cheerio.load(body);
let novels = [];
$(".section-item").each(function (result) {
const novelName = $(this).find(".book-name").text();
const novelCover = $(this).find("img").attr("src");
let novelUrl = $(this).find("a").attr("href").slice(1);
const novel = {
sourceId: 16,
novelName,
novelCover,
novelUrl,
};
novels.push(novel);
});
return novels;
};
const parseNovelAndChapters = async (novelUrl) => {
const url = `${baseUrl}${novelUrl}/`;
const result = await fetch(url);
const body = await result.text();
$ = cheerio.load(body);
let novel = {};
novel.sourceId = 16;
novel.sourceName = "WuxiaWorldCo";
novel.url = url;
novel.novelUrl = novelUrl;
novel.novelName = $(".book-name").text();
novel.novelCover = $("div.book-img > img").attr("src");
novel.genre = $("div.book-catalog > span.txt").text();
novel.Status = $("div.book-state > span.txt").text();
novel.author = $("div.author > span.name").text();
novelSummary = $("div.content > p.desc").html();
novel.summary = htmlToText(novelSummary);
let novelChapters = [];
$(".chapter-item").each(function (result) {
const chapterName = $(this).find(".chapter-name").text();
const releaseDate = null;
const chapterUrl = $(this).attr("href").split("/")[2];
novelChapters.push({
chapterName,
releaseDate,
chapterUrl,
});
});
novel.chapters = novelChapters;
return novel;
};
const parseChapter = async (novelUrl, chapterUrl) => {
const url = `${baseUrl}${novelUrl}/${chapterUrl}`;
const result = await fetch(url);
const body = await result.text();
$ = cheerio.load(body);
const chapterName = $("h1.chapter-title").text();
let chapterText = $("div.chapter-entity").html();
chapterText = htmlToText(chapterText);
let nextChapter = null;
if ($("a.next").length) {
nextChapter = $("a.next")
.attr("href")
.replace("/" + novelUrl + "/", "");
}
let prevChapter = null;
if ($("a.prev").length) {
prevChapter = $("a.prev")
.attr("href")
.replace("/" + novelUrl + "/", "");
}
const chapter = {
sourceId: 16,
novelUrl,
chapterUrl,
chapterName,
chapterText,
nextChapter,
prevChapter,
};
return chapter;
};
const searchNovels = async (searchTerm) => {
const url = `${baseUrl}search/${searchTerm}/1`;
const result = await fetch(url);
const body = await result.text();
$ = cheerio.load(body);
let novels = [];
$(".list-item").each(function (result) {
$(this).find("font").remove();
const novelName = $(this).find(".book-name").text();
const novelCover = $(this).find("img").attr("src");
let novelUrl = $(this).find("a").attr("href").slice(1);
const novel = {
sourceId: 16,
novelName,
novelCover,
novelUrl,
};
novels.push(novel);
});
return novels;
};
const WuxiaWorldCoScraper = {
popularNovels,
parseNovelAndChapters,
parseChapter,
searchNovels,
};
export default WuxiaWorldCoScraper;
+200
View File
@@ -0,0 +1,200 @@
import cheerio from "react-native-cheerio";
const baseUrl = "https://wuxiaworldsite.co/";
const searchUrl = "https://wuxiaworldsite.co/search/";
const popularNovels = async (page) => {
const url = `${baseUrl}/power-ranking`;
const result = await fetch(url);
const body = await result.text();
$ = cheerio.load(body);
let novels = [];
$(".a_item").each((i, el) => {
const novelName = $(el).find("a").attr("title");
const novelCover = baseUrl + $(el).find("a > img").attr("src");
let novelUrl = $(el).find(".name_views > a").attr("href");
novelId = novelUrl.split("/");
novelUrl = novelId[2] + "/";
const novel = {
sourceId: 12,
novelName,
novelCover,
novelUrl,
};
novels.push(novel);
});
return novels;
};
const parseNovelAndChapters = async (novelUrl) => {
const url = `${baseUrl}${novelUrl}`;
const result = await fetch(url);
const body = await result.text();
$ = cheerio.load(body);
$(".category_list").remove();
let novel = {};
novel.sourceId = 12;
novel.sourceName = "WuxiaWorldSite";
novel.url = url;
novel.novelUrl = novelUrl;
novel.novelName = $(".content-reading > h1").text().trim();
novel.novelCover = baseUrl + $(".img-read> img").attr("src");
novel.summary = $(".story-introduction-content").text();
novel.author = $(".content-reading > p").text();
novel.artist = null;
novel.genre = "";
$(".a_tag_item").each((i, el) => {
novel.genre += $(el).text() + ",";
});
novel.genre = novel.genre.split(",");
novel.genre.pop();
novel.Status = novel.genre.pop();
novel.genre = novel.genre.join(",");
const novelID = $(".show-more-list").attr("data-id");
const getChapters = async (novelID) => {
const chapterListUrl = baseUrl + "/get-full-list.ajax?id=" + novelID;
const data = await fetch(chapterListUrl);
const chapters = await data.text();
$ = cheerio.load(chapters);
let novelChapters = [];
$(".new-update-content").each((i, el) => {
let chapterName = $(el).text().split(/\t+/);
const releaseDate = chapterName.pop();
chapterName = chapterName[0];
let chapterUrl = $(el).attr("href");
chapterUrl = chapterUrl.split("/").pop();
const novel = {
chapterName,
releaseDate,
chapterUrl,
};
novelChapters.push(novel);
});
return novelChapters;
};
novel.chapters = await getChapters(novelID);
return novel;
};
const parseChapter = async (novelUrl, chapterUrl) => {
const url = `${baseUrl}/${novelUrl}/${chapterUrl}`;
const result = await fetch(url);
const body = await result.text();
$ = cheerio.load(body);
let chapterText = $("#content").html() || "";
let chapterName = "";
if (chapterText) {
chapterText = chapterText.split("<br>");
chapterName = chapterText.shift();
} else {
$("p").each((i, el) => {
if ($(el).css("display") !== "none") {
chapterText += $(el).text() + "\n";
}
});
chapterText = chapterText.split("\n");
chapterName = chapterText.shift();
}
chapterText = chapterText.join("\n");
const prevChapter = $(".pre").attr("href")
? $(".pre").attr("href").split("/").pop()
: null;
const nextChapter = $(".next_sesction > a").attr("href")
? $(".next_sesction > a").attr("href").split("/").pop()
: null;
novelUrl += "/";
chapterUrl += "/";
const chapter = {
sourceId: 12,
novelUrl,
chapterUrl,
chapterName,
chapterText,
nextChapter,
prevChapter,
};
return chapter;
};
const searchNovels = async (searchTerm) => {
const url = `${searchUrl}${searchTerm}`;
const result = await fetch(url);
const body = await result.text();
$ = cheerio.load(body);
let novels = [];
$(".a_item").each((i, el) => {
const novelName = $(el).find(".name_views > a").attr("title");
const novelCover = baseUrl + $(el).find("a > img").attr("src");
let novelUrl = $(el).find(".name_views > a").attr("href");
novelId = novelUrl.split("/");
novelUrl = novelId[2] + "/";
const novel = {
sourceId: 12,
novelName,
novelCover,
novelUrl,
};
novels.push(novel);
});
return novels;
};
const WuxiaWorldSiteScraper = {
popularNovels,
parseNovelAndChapters,
parseChapter,
searchNovels,
};
export default WuxiaWorldSiteScraper;
+235
View File
@@ -1,6 +1,26 @@
import ComradeMaoScraper from "./en/comrademao";
import ReadLightNovelScraper from "./en/readlightnovel";
import BoxNovelScraper from "./en/boxnovel";
import fastNovelScraper from "./en/fastnovel";
import readNovelFullScraper from "./en/readnovelfull";
import mtlNovelScraper from "./en/mtlnovel";
import novelhallScraper from "./en/novelhall";
import WuxiaWorldScraper from "./en/wuxiaworld";
import novelFullScraper from "./en/novelfull";
import novelTrenchScraper from "./en/noveltrench";
import vipNovelScraper from "./en/vipnovel";
import kissLightNovelScraper from "./en/kisslightnovels";
import WuxiaWorldSiteScraper from "./en/wuxiaworldsite";
import FreeWebNovelScraper from "./en/freewebnovel";
import JPMTLScraper from "./en/jpmtl";
import lightNovelPubScraper from "./en/lightnovelpub";
import WuxiaWorldCoScraper from "./en/wuxiaworldco";
import novelUpdatesCcScraper from "./en/novelupdatescc";
import readLightNovelCcScraper from "./en/readlightnovelcc";
import tapReadScraper from "./en/tapread";
import WuxiaWorldCloudScraper from "./en/wuxiaworldcloud";
import WoopReadScraper from "./en/woopread";
import FoxaHolicScraper from "./en/foxaholic";
export const sources = [
{
@@ -17,6 +37,146 @@ export const sources = [
lang: "English",
icon: "https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/readlightnovel/icon.png?raw=true",
},
{
sourceId: 3,
url: "https://fastnovel.net/",
sourceName: "FastNovel",
icon: "https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/fastnovel/icon.png?raw=true",
lang: "English",
},
{
sourceId: 4,
url: "https://readnovelfull.com/",
sourceName: "ReadNovelFull",
icon: "https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/readnovelfull/icon.png?raw=true",
lang: "English",
},
{
sourceId: 5,
url: "https://www.mtlnovel.com/",
sourceName: "MTLNovel",
icon: "https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/mtlnovel/icon.png?raw=true",
lang: "English",
},
{
sourceId: 6,
url: "https://www.novelhall.com/",
sourceName: "NovelHall",
icon: "https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/novelhall/icon.png?raw=true",
lang: "English",
},
{
sourceId: 7,
url: "https://www.wuxiaworld.com/",
sourceName: "WuxiaWorld",
icon: "https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/wuxiaworld/icon.png?raw=true",
lang: "English",
},
{
sourceId: 8,
url: "https://novelfull.com/",
sourceName: "NovelFull",
icon: "https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/novelfull/icon.png?raw=true",
lang: "English",
},
{
sourceId: 9,
url: "https://noveltrench.com/",
sourceName: "NovelTrench",
icon: "https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/noveltrench/icon.png?raw=true",
lang: "English",
},
{
sourceId: 10,
url: "https://vipnovel.com/",
sourceName: "VipNovel",
icon: "https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/vipnovel/icon.png?raw=true",
lang: "English",
},
{
sourceId: 11,
url: "https://kisslightnovels.info/",
sourceName: "KissLightNovels",
icon: "https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/kisslightnovels/icon.png?raw=true",
lang: "English",
},
{
sourceId: 12,
url: "https://wuxiaworldsite.co/",
sourceName: "WuxiaWorldSite",
icon: "https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/wuxiaworldsite/icon.png?raw=true",
lang: "English",
},
{
sourceId: 13,
url: "https://freewebnovel.com/",
sourceName: "FreeWebNovel",
icon: "https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/freewebnovel/icon.png?raw=true",
lang: "English",
},
{
sourceId: 14,
url: "https://jpmtl.com/",
sourceName: "JPMTL",
icon: "https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/jpmtl/icon.png?raw=true",
lang: "English",
},
{
sourceId: 15,
url: "https://www.lightnovelpub.com/",
sourceName: "LightNovelPub",
icon: "https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/lightnovelpub/icon.png?raw=true",
lang: "English",
},
{
sourceId: 16,
url: "https://www.wuxiaworld.co/",
sourceName: "WuxiaWorld.co",
icon: "https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/wuxiaworldco/icon.png?raw=true",
lang: "English",
},
{
sourceId: 17,
url: "http://www.tapread.com/",
sourceName: "TapRead",
icon: "https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/tapread/icon.png?raw=true",
lang: "English",
},
{
sourceId: 18,
url: "https://www.novelupdates.cc/",
sourceName: "NovelUpdates.cc",
icon: "https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/novelupdatescc/icon.png?raw=true",
lang: "English",
},
{
sourceId: 19,
url: "https://m.readlightnovel.cc/",
sourceName: "ReadLightNovel.cc",
icon: "https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/readlightnovelcc/icon.png?raw=true",
lang: "English",
},
{
sourceId: 20,
url: "http://wuxiaworld.cloud/",
sourceName: "WuxiaWorld.cloud",
icon: "https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/wuxiaworldcloud/icon.png?raw=true",
lang: "English",
},
{
sourceId: 21,
url: "https://woopread.com/",
sourceName: "WoopRead",
icon: "https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/woopread/icon.png?raw=true",
lang: "English",
},
{
sourceId: 22,
url: "https://foxaholic.com/",
sourceName: "Foxaholic",
icon: "https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/foxaholic/icon.png?raw=true",
lang: "English",
},
{
sourceId: 27,
url: "https://comrademao.com/",
@@ -34,8 +194,83 @@ export const getSource = (sourceId) => {
case 2: {
return ReadLightNovelScraper;
}
case 3: {
return fastNovelScraper;
}
case 4: {
return readNovelFullScraper;
}
case 5: {
return mtlNovelScraper;
}
case 6: {
return novelhallScraper;
}
case 7: {
return WuxiaWorldScraper;
}
case 8: {
return novelFullScraper;
}
case 9: {
return novelTrenchScraper;
}
case 10: {
return vipNovelScraper;
}
case 11: {
return kissLightNovelScraper;
}
case 12: {
return WuxiaWorldSiteScraper;
}
case 13: {
return FreeWebNovelScraper;
}
case 14: {
return JPMTLScraper;
}
case 15: {
return lightNovelPubScraper;
}
case 16: {
return WuxiaWorldCoScraper;
}
case 17: {
return tapReadScraper;
}
case 18: {
return novelUpdatesCcScraper;
}
case 19: {
return readLightNovelCcScraper;
}
case 20: {
return WuxiaWorldCloudScraper;
}
case 21: {
return WoopReadScraper;
}
case 22: {
return FoxaHolicScraper;
}
case 23: {
return;
}
case 24: {
return;
}
case 25: {
return;
}
case 26: {
return;
}
case 27: {
return ComradeMaoScraper;
}
case 3: {
return;
}
}
};