Add more sources and fix sources 1-9
This commit is contained in:
@@ -66,7 +66,8 @@ const Extension = ({ navigation, route }) => {
|
||||
};
|
||||
|
||||
const clearSearchbar = () => {
|
||||
getNovels();
|
||||
setNovels([]);
|
||||
getNovels(1);
|
||||
setLoading(true);
|
||||
setSearchText("");
|
||||
};
|
||||
|
||||
@@ -0,0 +1,177 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
import { htmlToText } from "../helpers/htmlToText";
|
||||
|
||||
const baseUrl = "https://fastnovel.net";
|
||||
const searchUrl = `https://fastnovel.net/search/`;
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
const url = `${baseUrl}/list/most-popular.html?page=${page}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".film-item").each(function (result) {
|
||||
const novelName = $(this).find(".name").text();
|
||||
const novelCover = $(this).find(".img").attr("data-original");
|
||||
const novelUrl = $(this).find("a").attr("href").substring(1);
|
||||
|
||||
const novel = {
|
||||
sourceId: 3,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = `${baseUrl}/${novelUrl}/`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.sourceId = 3;
|
||||
|
||||
novel.sourceName = "FastNovel";
|
||||
|
||||
novel.url = url;
|
||||
|
||||
novel.novelUrl = novelUrl;
|
||||
|
||||
novel.novelName = $("h1").text();
|
||||
|
||||
novel.novelCover = $(".book-cover").attr("data-original");
|
||||
|
||||
novel.summary = $("div.film-content > p").text();
|
||||
|
||||
novel.author = $("li > label")
|
||||
.filter(function () {
|
||||
return $(this).text().trim() === "Author:";
|
||||
})
|
||||
.next()
|
||||
.text();
|
||||
|
||||
novel.genre = $("li")
|
||||
.filter(function () {
|
||||
return $(this).find("label").text().trim() === "Genre:";
|
||||
})
|
||||
.text()
|
||||
.replace("Genre:", "");
|
||||
|
||||
novel.artist = null;
|
||||
|
||||
novel.Status = null;
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
$(".chapter").each(function (result) {
|
||||
const chapterName = $(this).text();
|
||||
const releaseDate = null;
|
||||
let chapterUrl = $(this).attr("href");
|
||||
chapterUrl = chapterUrl.replace(`/${novelUrl}/`, "");
|
||||
|
||||
const novel = {
|
||||
chapterName,
|
||||
releaseDate,
|
||||
chapterUrl,
|
||||
};
|
||||
|
||||
novelChapters.push(novel);
|
||||
});
|
||||
|
||||
novel.chapters = novelChapters;
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = `${baseUrl}/${novelUrl}/${chapterUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
const chapterName = $(".episode-name").text();
|
||||
let chapterText = $("#chapter-body").html();
|
||||
chapterText = htmlToText(chapterText);
|
||||
|
||||
let nextChapter = null;
|
||||
|
||||
nextChapter = $("div.btn-group > a")
|
||||
.filter(function () {
|
||||
return $(this).text().trim() === "Next Chapter";
|
||||
})
|
||||
.attr("href")
|
||||
.replace("/", "");
|
||||
|
||||
let prevChapter = null;
|
||||
|
||||
prevChapter = $("div.btn-group > a")
|
||||
.filter(function () {
|
||||
return $(this).text().trim() === "Prev Chapter";
|
||||
})
|
||||
.attr("href")
|
||||
.replace("/", "");
|
||||
|
||||
const chapter = {
|
||||
sourceId: 3,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
nextChapter,
|
||||
prevChapter,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const url = `${searchUrl}${searchTerm}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".film-item").each(function (result) {
|
||||
const novelName = $(this).find("div.title > p.name").text();
|
||||
const novelCover = $(this).find(".img").attr("data-original");
|
||||
const novelUrl = $(this).find("a").attr("href").substring(1);
|
||||
|
||||
const novel = {
|
||||
sourceId: 3,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const fastNovelScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default fastNovelScraper;
|
||||
@@ -0,0 +1,212 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
|
||||
import { htmlToText } from "../helpers/htmlToText";
|
||||
|
||||
const baseUrl = "https://foxaholic.com/";
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
let url = baseUrl + "novel/?m_orderby=rating";
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".page-item-detail").each(function (result) {
|
||||
const novelName = $(this).find(".h5 > a").text();
|
||||
const novelCover = $(this).find("img").attr("data-src");
|
||||
|
||||
let novelUrl = $(this).find(".h5 > a").attr("href").split("/")[4];
|
||||
novelUrl += "/";
|
||||
|
||||
const novel = {
|
||||
sourceId: 22,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = `${baseUrl}novel/${novelUrl}/`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.sourceId = 22;
|
||||
|
||||
novel.sourceName = "Foxaholic";
|
||||
|
||||
novel.url = url;
|
||||
|
||||
novel.novelUrl = novelUrl;
|
||||
|
||||
novel.novelName = $(".post-title > h1")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
novel.novelCover = $(".summary_image > a > img").attr("data-src");
|
||||
|
||||
$(".post-content_item").each(function (result) {
|
||||
detailName = $(this)
|
||||
.find(".summary-heading > h5")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
detail = $(this)
|
||||
.find(".summary-content")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
novel[detailName] = detail;
|
||||
});
|
||||
|
||||
novel.genre = novel.Genre.replace(/, /g, ",");
|
||||
novel.author = novel.Author;
|
||||
novel.Status = null;
|
||||
novel.artist = null;
|
||||
|
||||
delete novel.Genre;
|
||||
delete novel.Author;
|
||||
|
||||
$(".description-summary > div.summary__content").find("em").remove();
|
||||
|
||||
novel.summary = $(".description-summary > div.summary__content")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "");
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
const novelId = $("#manga-chapters-holder").attr("data-id");
|
||||
|
||||
let formData = new FormData();
|
||||
formData.append("action", "manga_get_chapters");
|
||||
formData.append("manga", novelId);
|
||||
|
||||
const data = await fetch("https://foxaholic.com/wp-admin/admin-ajax.php", {
|
||||
method: "POST",
|
||||
body: formData,
|
||||
});
|
||||
const text = await data.text();
|
||||
|
||||
$ = cheerio.load(text);
|
||||
|
||||
$(".wp-manga-chapter").each(function (result) {
|
||||
chapterName = $(this)
|
||||
.find("a")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
releaseDate = $(this).find("span").text().trim();
|
||||
|
||||
chapterUrl = $(this).find("a").attr("href").split("/");
|
||||
|
||||
chapterUrl[6]
|
||||
? (chapterUrl = chapterUrl[5] + "/" + chapterUrl[6])
|
||||
: (chapterUrl = chapterUrl[5]);
|
||||
|
||||
novelChapters.push({ chapterName, releaseDate, chapterUrl });
|
||||
});
|
||||
|
||||
novel.chapters = novelChapters.reverse();
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = `${baseUrl}novel/${novelUrl}/${chapterUrl}/`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
const chapterName = $("h1#chapter-heading").text();
|
||||
let chapterText = $(".reading-content").html();
|
||||
chapterText = htmlToText(chapterText);
|
||||
|
||||
let nextChapter = null;
|
||||
|
||||
if ($(".nav-next").length) {
|
||||
nextChapter = $(".nav-next").find("a").attr("href").split("/");
|
||||
nextChapter[6]
|
||||
? (nextChapter = nextChapter[5] + "/" + nextChapter[6])
|
||||
: (nextChapter = nextChapter[5]);
|
||||
}
|
||||
|
||||
let prevChapter = null;
|
||||
|
||||
if ($(".nav-previous").length) {
|
||||
prevChapter = $(".nav-previous").find("a").attr("href").split("/");
|
||||
prevChapter[6]
|
||||
? (prevChapter = prevChapter[5] + "/" + prevChapter[6])
|
||||
: (prevChapter = prevChapter[5]);
|
||||
}
|
||||
|
||||
novelUrl += "/";
|
||||
chapterUrl = optionalUrl + chapterUrl;
|
||||
|
||||
const chapter = {
|
||||
sourceId: 22,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
nextChapter,
|
||||
prevChapter,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const url = `${baseUrl}?s=${searchTerm}&post_type=wp-manga`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".c-tabs-item__content").each(function (result) {
|
||||
const novelName = $(this).find(".h4 > a").text();
|
||||
const novelCover = $(this).find("img").attr("data-src");
|
||||
|
||||
let novelUrl = $(this).find(".h4 > a").attr("href").split("/")[4];
|
||||
novelUrl += "/";
|
||||
const novel = {
|
||||
sourceId: 22,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const FoxaHolicScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default FoxaHolicScraper;
|
||||
@@ -0,0 +1,213 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
import { htmlToText } from "../helpers/htmlToText";
|
||||
|
||||
const baseUrl = "https://freewebnovel.com/";
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
let url = baseUrl + "most-popular-novel/";
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".li-row").each(function (result) {
|
||||
const novelName = $(this).find(".tit").text();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
|
||||
let novelUrl = $(this)
|
||||
.find("h3 > a")
|
||||
.attr("href")
|
||||
.replace(".html", "")
|
||||
.slice(1);
|
||||
|
||||
novelUrl += "/";
|
||||
|
||||
const novel = {
|
||||
sourceId: 13,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = `${baseUrl}${novelUrl}.html`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.sourceId = 13;
|
||||
|
||||
novel.sourceName = "FreeWebNovel";
|
||||
|
||||
novel.url = url;
|
||||
|
||||
novel.novelUrl = `${novelUrl.replace(".html", "")}/`;
|
||||
|
||||
novel.novelName = $("h1.tit").text();
|
||||
|
||||
novel.novelCover = $(".pic > img").attr("src");
|
||||
|
||||
novel.genre = $("[title=Genre]")
|
||||
.next()
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "");
|
||||
|
||||
novel.author = $("[title=Author]")
|
||||
.next()
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "");
|
||||
|
||||
novel.artist = null;
|
||||
|
||||
novel.Status = $("[title=Status]")
|
||||
.next()
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "");
|
||||
|
||||
novel.Alternative = $("[title='Alternative names']")
|
||||
.next()
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "");
|
||||
|
||||
let novelSummary = $(".inner").html();
|
||||
novel.summary = htmlToText(novelSummary);
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
let latestChapter;
|
||||
|
||||
$("h3.tit").each(function (res) {
|
||||
if ($(this).find("a").text() === novel.novelName) {
|
||||
// console.log($(this).find("a").text());
|
||||
// console.log(novel.novelName);
|
||||
latestChapter = $(this).next().find("span.s3").text().match(/\d+/);
|
||||
}
|
||||
});
|
||||
|
||||
latestChapter = latestChapter[0];
|
||||
|
||||
for (let i = 1; i <= parseInt(latestChapter); i++) {
|
||||
const chapterName = "Chapter " + i;
|
||||
|
||||
const releaseDate = null;
|
||||
|
||||
const chapterUrl = "chapter-" + i;
|
||||
|
||||
const chapter = { chapterName, releaseDate, chapterUrl };
|
||||
|
||||
novelChapters.push(chapter);
|
||||
}
|
||||
|
||||
novel.chapters = novelChapters;
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = `${baseUrl}${novelUrl}/${chapterUrl}.html`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let chapterName = $("h1.tit").text();
|
||||
|
||||
let chapterText = $("div.txt").html();
|
||||
chapterText = htmlToText(chapterText);
|
||||
|
||||
let nextChapter = null;
|
||||
|
||||
if ($("[title='Read Next chapter']").length) {
|
||||
nextChapter = $("[title='Read Next chapter']")
|
||||
.attr("href")
|
||||
.replace("/" + novelUrl + "/", "")
|
||||
.replace(".html", "");
|
||||
}
|
||||
|
||||
let prevChapter = null;
|
||||
|
||||
if ($("[title='Read Privious Chapter']").length) {
|
||||
prevChapter = $("[title='Read Privious Chapter']")
|
||||
.attr("href")
|
||||
.replace("/" + novelUrl + "/", "")
|
||||
.replace(".html", "");
|
||||
}
|
||||
|
||||
chapterUrl = chapterUrl;
|
||||
|
||||
const chapter = {
|
||||
sourceId: 13,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
nextChapter,
|
||||
prevChapter,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const url = baseUrl + "search/";
|
||||
|
||||
const formData = new FormData();
|
||||
formData.append("searchkey", searchTerm);
|
||||
|
||||
const result = await fetch(url, {
|
||||
method: "POST",
|
||||
body: formData,
|
||||
});
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".li-row > .li > .con").each(function (result) {
|
||||
const novelName = $(this).find(".tit").text();
|
||||
const novelCover = $(this).find(".pic > a > img").attr("data-cfsrc");
|
||||
|
||||
let novelUrl = $(this)
|
||||
.find("h3 > a")
|
||||
.attr("href")
|
||||
.replace(".html", "")
|
||||
.slice(1);
|
||||
|
||||
novelUrl += "/";
|
||||
|
||||
const novel = {
|
||||
sourceId: 13,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const FreeWebNovelScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default FreeWebNovelScraper;
|
||||
@@ -0,0 +1,179 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
|
||||
import { htmlToText } from "../helpers/htmlToText";
|
||||
|
||||
const baseUrl = "https://jpmtl.com/";
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
let url =
|
||||
"https://jpmtl.com/v2/book/show/browse?query=&categories=&content_type=0&direction=0&page=2&limit=20&type=5&status=all&language=3&exclude_categories=";
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.json();
|
||||
|
||||
let novels = [];
|
||||
|
||||
body.map((item) => {
|
||||
const novelName = item.title;
|
||||
const novelCover = item.cover;
|
||||
const novelUrl = item.id + "/";
|
||||
|
||||
const novel = { sourceId: 14, novelName, novelCover, novelUrl };
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = `${baseUrl}books/${novelUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.sourceId = 14;
|
||||
|
||||
novel.sourceName = "JPMTL";
|
||||
|
||||
novel.url = url;
|
||||
|
||||
novel.novelUrl = novelUrl;
|
||||
|
||||
novel.novelName = $("h1.book-sidebar__title").text();
|
||||
|
||||
novel.novelCover = $(".book-sidebar").find("img").attr("src");
|
||||
|
||||
$(".post-content_item").each(function (result) {
|
||||
detailName = $(this)
|
||||
.find(".summary-heading > h5")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
detail = $(this)
|
||||
.find(".summary-content")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
novel[detailName] = detail;
|
||||
});
|
||||
|
||||
novel.summary = $(".main-book__synopsis").text();
|
||||
|
||||
novel.genre = "";
|
||||
|
||||
$("a.main-book__category").each(function (result) {
|
||||
novel.genre += $(this).text();
|
||||
});
|
||||
|
||||
novel.author = $(".book-sidebar__author > .book-sidebar__info")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
const chapterListUrl = `https://jpmtl.com/v2/chapter/${novelUrl}/list?state=published&structured=true&d`;
|
||||
|
||||
const chapterResult = await fetch(chapterListUrl);
|
||||
const volumes = await chapterResult.json();
|
||||
|
||||
volumes.map((volume) => {
|
||||
volume.chapters.map((chapter) => {
|
||||
const chapterName = chapter.title;
|
||||
const releaseDate = chapter.created_at;
|
||||
const chapterUrl = chapter.id;
|
||||
|
||||
const obj = {
|
||||
sourceId: 14,
|
||||
chapterName,
|
||||
releaseDate,
|
||||
chapterUrl,
|
||||
};
|
||||
|
||||
novelChapters.push(obj);
|
||||
});
|
||||
});
|
||||
|
||||
novel.chapters = novelChapters;
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = `${baseUrl}books/${novelUrl}/${chapterUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
console.log(url);
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
const chapterName = $(".chapter-content__title").text();
|
||||
let chapterText = $(".chapter-content__content").html();
|
||||
chapterText = htmlToText(chapterText);
|
||||
|
||||
let nextChapter = null;
|
||||
let prevChapter = null;
|
||||
|
||||
$(".chapter-wrapper__nav").each(function (result) {
|
||||
const chId = $(this)
|
||||
.attr("href")
|
||||
.replace("/books/" + novelUrl + "/", "");
|
||||
if (chId < chapterUrl) {
|
||||
prevChapter = chId;
|
||||
} else {
|
||||
nextChapter = chId;
|
||||
}
|
||||
});
|
||||
|
||||
novelUrl += "/";
|
||||
|
||||
const chapter = {
|
||||
sourceId: 14,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
nextChapter,
|
||||
prevChapter,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const url = `https://jpmtl.com/v2/book/show/browse?query=${searchTerm}&categories=&content_type=2&direction=0&page=1&limit=20&type=5&status=all&language=3&exclude_categories=`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.json();
|
||||
|
||||
const novels = [];
|
||||
|
||||
body.map((item) => {
|
||||
const novelName = item.title;
|
||||
const novelCover = item.cover;
|
||||
const novelUrl = item.id + "/";
|
||||
|
||||
const novel = { sourceId: 14, novelName, novelCover, novelUrl };
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const JPMTLScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default JPMTLScraper;
|
||||
@@ -0,0 +1,188 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
|
||||
import { htmlToText } from "../helpers/htmlToText";
|
||||
|
||||
const baseUrl = "https://kisslightnovels.info/";
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
const result = await fetch(baseUrl);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".page-item-detail").each(function (result) {
|
||||
const novelName = $(this).find("h5 > a").text();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
|
||||
let novelUrl = $(this).find("h5 > a").attr("href");
|
||||
novelUrl = novelUrl.replace(`${baseUrl}novel/`, "");
|
||||
|
||||
const novel = {
|
||||
sourceId: 11,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = `${baseUrl}novel/${novelUrl}/`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.sourceId = 11;
|
||||
|
||||
novel.sourceName = "KissLightNovels";
|
||||
|
||||
novel.url = url;
|
||||
|
||||
novel.novelUrl = novelUrl;
|
||||
|
||||
novel.novelName = $(".post-title > h1")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
novel.novelCover = $(".summary_image > a > img").attr("src");
|
||||
|
||||
$(".post-content_item").each(function (result) {
|
||||
detailName = $(this)
|
||||
.find(".summary-heading > h5")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
detail = $(this)
|
||||
.find(".summary-content")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
novel[detailName] = detail;
|
||||
});
|
||||
|
||||
$(".description-summary > div.summary__content").find("em").remove();
|
||||
|
||||
novel.summary = $(".description-summary > div.summary__content")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "");
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
$(".wp-manga-chapter").each(function (result) {
|
||||
chapterName = $(this)
|
||||
.find("a")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
releaseDate = $(this)
|
||||
.find("span")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
chapterUrl = $(this).find("a").attr("href").replace(url, "");
|
||||
|
||||
novelChapters.push({ chapterName, releaseDate, chapterUrl });
|
||||
});
|
||||
|
||||
novel.chapters = novelChapters.reverse();
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = `${baseUrl}novel/${novelUrl}/${chapterUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
$("#snippet-box").remove();
|
||||
|
||||
const chapterName = $("div.text-left > h4").text();
|
||||
let chapterText = $(".reading-content").html();
|
||||
chapterText = htmlToText(chapterText);
|
||||
|
||||
let nextChapter = null;
|
||||
|
||||
if ($(".nav-next").length) {
|
||||
nextChapter = $(".nav-next")
|
||||
.find("a")
|
||||
.attr("href")
|
||||
.replace(baseUrl + "novel/" + novelUrl + "/", "");
|
||||
}
|
||||
|
||||
let prevChapter = null;
|
||||
|
||||
if ($(".nav-previous").length) {
|
||||
prevChapter = $(".nav-previous")
|
||||
.find("a")
|
||||
.attr("href")
|
||||
.replace(baseUrl + "novel/" + novelUrl + "/", "");
|
||||
}
|
||||
|
||||
const chapter = {
|
||||
sourceId: 11,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
nextChapter,
|
||||
prevChapter,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const url = `${baseUrl}?s=${searchTerm}&post_type=wp-manga&m_orderby=rating`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".c-tabs-item__content").each(function (result) {
|
||||
const novelName = $(this).find("h4 > a").text();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
|
||||
let novelUrl = $(this).find("h4 > a").attr("href");
|
||||
novelUrl = novelUrl.replace(`${baseUrl}novel/`, "");
|
||||
|
||||
const novel = {
|
||||
sourceId: 11,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const kissLightNovelScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default kissLightNovelScraper;
|
||||
@@ -0,0 +1,198 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
|
||||
import { htmlToText } from "../helpers/htmlToText";
|
||||
|
||||
const baseUrl = "https://www.lightnovelpub.com/";
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
let url = baseUrl + "browse/all/popular/all/1";
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".novel-item.ads").remove();
|
||||
|
||||
$(".novel-item").each(function (result) {
|
||||
const novelName = $(this)
|
||||
.find(".novel-title")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "");
|
||||
|
||||
const novelCover = $(this).find("img").attr("data-src");
|
||||
|
||||
let novelUrl = $(this)
|
||||
.find(".novel-title > a")
|
||||
.attr("href")
|
||||
.replace(`/novel/`, "");
|
||||
novelUrl += "/";
|
||||
|
||||
const novel = {
|
||||
sourceId: 15,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = `${baseUrl}novel/${novelUrl}/`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.sourceId = 15;
|
||||
|
||||
novel.sourceName = "LightNovelPub";
|
||||
|
||||
novel.url = url;
|
||||
|
||||
novel.novelUrl = novelUrl;
|
||||
|
||||
novel.novelName = $("h1.novel-title")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
novel.novelCover = $("figure.cover > img").attr("src");
|
||||
|
||||
novel.genre = "";
|
||||
|
||||
$("div.categories > ul > li").each(function (result) {
|
||||
novel.genre +=
|
||||
$(this)
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "") + ",";
|
||||
});
|
||||
|
||||
$("div.header-stats > span").each(function (result) {
|
||||
if ($(this).find("small").text() === "Status") {
|
||||
novel.Status = $(this).find("strong").text();
|
||||
}
|
||||
});
|
||||
|
||||
novel.genre = novel.genre.slice(0, -1);
|
||||
|
||||
novel.author = $(".author > a > span").text();
|
||||
|
||||
novelSummary = $(".summary > .content").html();
|
||||
novel.summary = htmlToText(novelSummary);
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
let firstChapterNo, totalChapters;
|
||||
|
||||
totalChapters = $(".header-stats > span").first().text().match(/\d+/)[0];
|
||||
firstChapterNo = $(".chapter-no").first().text().match(/\d+/)[0];
|
||||
|
||||
for (let i = firstChapterNo; i <= totalChapters; i++) {
|
||||
const chapterName = "Chapter " + i;
|
||||
|
||||
const releaseDate = null;
|
||||
|
||||
const chapterUrl = "chapter-" + i;
|
||||
|
||||
const chapter = { chapterName, releaseDate, chapterUrl };
|
||||
|
||||
novelChapters.push(chapter);
|
||||
}
|
||||
|
||||
novel.chapters = novelChapters;
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = `${baseUrl}novel/${novelUrl}/${chapterUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
const chapterName = $("h2").text();
|
||||
let chapterText = $("#chapter-container").html();
|
||||
chapterText = htmlToText(chapterText);
|
||||
|
||||
let nextChapter = null;
|
||||
|
||||
if (!$("a.nextchap.isDisabled").length) {
|
||||
nextChapter = $("a.nextchap")
|
||||
.attr("href")
|
||||
.replace("/novel/" + novelUrl + "/", "");
|
||||
}
|
||||
|
||||
let prevChapter = null;
|
||||
|
||||
if (!$("a.prevchap.isDisabled").length) {
|
||||
prevChapter = $("a.prevchap")
|
||||
.attr("href")
|
||||
.replace("/novel/" + novelUrl + "/", "");
|
||||
}
|
||||
|
||||
const chapter = {
|
||||
sourceId: 15,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
nextChapter,
|
||||
prevChapter,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const url = `${baseUrl}lnwsearchlive?inputContent=${searchTerm}`;
|
||||
|
||||
const result = await scraper(url);
|
||||
|
||||
$ = cheerio.load(result);
|
||||
|
||||
let novels = [];
|
||||
|
||||
let results = JSON.parse($("body").text());
|
||||
|
||||
$ = cheerio.load(results.resultview);
|
||||
|
||||
$(".novel-item").each(function (result) {
|
||||
const novelName = $(this).find("h4.novel-title").text();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
|
||||
let novelUrl = $(this).find("a").attr("href").replace(`/novel/`, "");
|
||||
novelUrl += "/";
|
||||
|
||||
const novel = {
|
||||
sourceId: 15,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const lightNovelPubScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default lightNovelPubScraper;
|
||||
@@ -0,0 +1,196 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
import { htmlToText } from "../helpers/htmlToText";
|
||||
|
||||
const baseUrl = "https://www.mtlnovel.com";
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
const url = `${baseUrl}/alltime-rank/page/${page}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$("div.box.wide").each(function (result) {
|
||||
const novelName = $(this).find("a.list-title").text().slice(4);
|
||||
const novelCover = $(this).find("amp-img").attr("src");
|
||||
|
||||
let novelUrl = $(this).find("a.list-title").attr("href");
|
||||
novelUrl = novelUrl.replace("https://www.mtlnovel.com/", "");
|
||||
|
||||
const novel = {
|
||||
sourceId: 5,
|
||||
novelUrl,
|
||||
novelName,
|
||||
novelCover,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = `${baseUrl}/${novelUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.sourceId = 5;
|
||||
|
||||
novel.sourceName = "MTLNovel";
|
||||
|
||||
novel.url = url;
|
||||
|
||||
novel.novelUrl = novelUrl;
|
||||
|
||||
novel.novelName = $("h1.entry-title").text();
|
||||
|
||||
novel.novelCover = $(".nov-head > amp-img").attr("src");
|
||||
|
||||
novel.summary = $("div.desc > h2").next().text();
|
||||
|
||||
novel.author = $("tr > td")
|
||||
.filter(function () {
|
||||
return $(this).prev().text().trim() === "Author";
|
||||
})
|
||||
.next()
|
||||
.text()
|
||||
.replace("Auhtor:", "");
|
||||
|
||||
novel.Status = $("tr > td")
|
||||
.filter(function () {
|
||||
return $(this).prev().text().trim() === "Status";
|
||||
})
|
||||
.next()
|
||||
.text()
|
||||
.replace("Status:", "");
|
||||
|
||||
novel.genre = $("td")
|
||||
.filter(function () {
|
||||
return $(this).prev().text().trim() === "Genre";
|
||||
})
|
||||
.next()
|
||||
.text()
|
||||
.replace("Genre:", "");
|
||||
|
||||
novel.artist = null;
|
||||
|
||||
const chapterListUrl = url + "/chapter-list/";
|
||||
|
||||
const getChapters = async () => {
|
||||
const result = await fetch(chapterListUrl);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
$("div.ch-list")
|
||||
.find("a.ch-link")
|
||||
.each(function (result) {
|
||||
const chapterName = $(this).text().replace("~ ", "");
|
||||
const releaseDate = null;
|
||||
|
||||
let chapterUrl = $(this).attr("href");
|
||||
chapterUrl = chapterUrl.replace(
|
||||
`https://www.mtlnovel.com/${novelUrl}/`,
|
||||
""
|
||||
);
|
||||
|
||||
novelChapters.push({
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
releaseDate,
|
||||
});
|
||||
});
|
||||
return novelChapters.reverse();
|
||||
};
|
||||
|
||||
novel.chapters = await getChapters();
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = chapterUrl;
|
||||
console.log(url);
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
const chapterName = $("h1.main-title").text();
|
||||
let chapterText = $("div.par").html();
|
||||
chapterText = htmlToText(chapterText);
|
||||
|
||||
let nextChapter = null;
|
||||
if ($("a.next").attr("href")) {
|
||||
nextChapter = $("a.next").attr("href").replace(`/${novelUrl}/`, "");
|
||||
}
|
||||
|
||||
let prevChapter = null;
|
||||
if ($("a.prev").attr("href")) {
|
||||
prevChapter = $("a.prev").attr("href").replace(`/${novelUrl}/`, "");
|
||||
}
|
||||
|
||||
chapter = {
|
||||
sourceId: 5,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
nextChapter,
|
||||
prevChapter,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const searchUrl =
|
||||
"https://www.mtlnovel.com/wp-admin/admin-ajax.php?action=autosuggest&q=" +
|
||||
searchTerm +
|
||||
"&__amp_source_origin=https%3A%2F%2Fwww.mtlnovel.com";
|
||||
|
||||
const res = await fetch(searchUrl);
|
||||
const result = await res.text();
|
||||
|
||||
$ = cheerio.load(result);
|
||||
|
||||
let body = JSON.parse($("body").text());
|
||||
|
||||
let novels = [];
|
||||
|
||||
body.items[0].results.map((item) => {
|
||||
const novelName = item.title.replace(/<\/?strong>/g, "");
|
||||
const novelCover = item.thumbnail;
|
||||
const novelUrl = item.permalink.replace(
|
||||
"https://www.mtlnovel.com/",
|
||||
""
|
||||
);
|
||||
|
||||
const novel = { sourceId: 5, novelName, novelCover, novelUrl };
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const mtlNovelScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default mtlNovelScraper;
|
||||
@@ -0,0 +1,206 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
|
||||
import { htmlToText } from "../helpers/htmlToText";
|
||||
|
||||
const baseUrl = "https://novelfull.com";
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
const url = `${baseUrl}/most-popular?page=${page}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$("div.col-truyen-main > div.list-truyen > .row").each(function (result) {
|
||||
const novelName = $(this).find("h3.truyen-title > a").text();
|
||||
|
||||
let novelCover = $(this).find("img").attr("src");
|
||||
novelCover = baseUrl + novelCover;
|
||||
|
||||
let novelUrl = $(this)
|
||||
.find("h3.truyen-title > a")
|
||||
.attr("href")
|
||||
.replace(".html", "")
|
||||
.substring(1);
|
||||
novelUrl = `${novelUrl}/`;
|
||||
|
||||
const novel = {
|
||||
sourceId: 8,
|
||||
novelUrl,
|
||||
novelName,
|
||||
novelCover,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = `${baseUrl}/${novelUrl.slice(0, -1)}.html`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.sourceId = 8;
|
||||
|
||||
novel.sourceName = "NovelFull";
|
||||
|
||||
novel.url = url;
|
||||
|
||||
novel.novelUrl = novelUrl;
|
||||
|
||||
novel.novelName = $("div.book > img").attr("alt");
|
||||
|
||||
novel.novelCover = baseUrl + $("div.book > img").attr("src");
|
||||
|
||||
novel.summary = $("div.desc-text").text();
|
||||
|
||||
novel.author = $("div.info > div > h3")
|
||||
.filter(function () {
|
||||
return $(this).text().trim() === "Author:";
|
||||
})
|
||||
.siblings()
|
||||
.text();
|
||||
|
||||
novel.genre = $("div.info > div")
|
||||
.filter(function () {
|
||||
return $(this).find("h3").text().trim() === "Genre:";
|
||||
})
|
||||
.text()
|
||||
.replace("Genre:", "");
|
||||
|
||||
novel.artist = null;
|
||||
|
||||
novel.Status = $("div.info > div > h3")
|
||||
.filter(function () {
|
||||
return $(this).text().trim() === "Status:";
|
||||
})
|
||||
.next()
|
||||
.text();
|
||||
|
||||
const novelId = $("#rating").attr("data-novel-id");
|
||||
|
||||
const getChapters = async (novelId) => {
|
||||
const chapterListUrl =
|
||||
baseUrl + "/ajax/chapter-option?novelId=" + novelId;
|
||||
|
||||
const data = await fetch(chapterListUrl);
|
||||
const chapters = await data.text();
|
||||
|
||||
$ = cheerio.load(chapters);
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
$("select > option").each(function (result) {
|
||||
let chapterName = $(this).text();
|
||||
let releaseDate = null;
|
||||
let chapterUrl = $(this).attr("value");
|
||||
chapterUrl = chapterUrl.replace(`/${novelUrl}/`, "");
|
||||
|
||||
novelChapters.push({
|
||||
chapterName,
|
||||
releaseDate,
|
||||
chapterUrl,
|
||||
});
|
||||
});
|
||||
return novelChapters;
|
||||
};
|
||||
|
||||
novel.chapters = await getChapters(novelId);
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = `${baseUrl}${chapterUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
const chapterName = $(".chapter-title").attr("title");
|
||||
let chapterText = $("#chapter-content").html();
|
||||
chapterText = htmlToText(chapterText);
|
||||
|
||||
let nextChapter = null;
|
||||
if ($("a#next_chap").attr("href")) {
|
||||
nextChapter = $("a#next_chap")
|
||||
.attr("href")
|
||||
.replace(`/${novelUrl}/`, "");
|
||||
}
|
||||
|
||||
let prevChapter = null;
|
||||
if ($("a#prev_chap").attr("href")) {
|
||||
prevChapter = $("a#prev_chap")
|
||||
.attr("href")
|
||||
.replace(`/${novelUrl}/`, "");
|
||||
}
|
||||
|
||||
const chapter = {
|
||||
sourceId: 8,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
nextChapter,
|
||||
prevChapter,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const searchUrl = `https://novelfull.com/search?keyword=`;
|
||||
|
||||
const url = `${searchUrl}${searchTerm}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$("div.col-truyen-main > div.list-truyen > .row").each(function (result) {
|
||||
let novelUrl = $(this)
|
||||
.find("h3.truyen-title > a")
|
||||
.attr("href")
|
||||
.replace(".html", "")
|
||||
.substring(1);
|
||||
novelUrl = `${novelUrl}/`;
|
||||
|
||||
const novelName = $(this).find("h3.truyen-title > a").text();
|
||||
let novelCover = $(this).find("img").attr("src");
|
||||
novelCover = baseUrl + novelCover;
|
||||
|
||||
const novel = {
|
||||
sourceId: 8,
|
||||
novelUrl,
|
||||
novelName,
|
||||
novelCover,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const novelFullScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default novelFullScraper;
|
||||
@@ -0,0 +1,178 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
|
||||
import { htmlToText } from "../helpers/htmlToText";
|
||||
|
||||
const baseUrl = "https://www.novelhall.com/";
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
const result = await fetch(baseUrl);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$("div.section1")
|
||||
.find("li")
|
||||
.each(function (result) {
|
||||
const novelName = $(this).find(".book-info > h2 > a").text();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
const novelUrl = $(this).find("a").attr("href").substring(1);
|
||||
|
||||
const novel = {
|
||||
sourceId: 6,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = `${baseUrl}${novelUrl}/`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.sourceId = 6;
|
||||
|
||||
novel.sourceName = "NovelHall";
|
||||
|
||||
novel.url = url;
|
||||
|
||||
novel.novelUrl = novelUrl;
|
||||
|
||||
novel.novelName = $("h1").text();
|
||||
|
||||
novel.novelCover = $("div.book-img > img").attr("src");
|
||||
|
||||
novel.summary = $("div.intro")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "");
|
||||
|
||||
novel.author = $("span.blue").first().text().replace("Author:", "");
|
||||
|
||||
novel.genre = $("a.red").text();
|
||||
|
||||
novel.artist = null;
|
||||
|
||||
novel.Status = $("span.blue").first().next().text().replace("Status:", "");
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
$("div.book-catalog.hidden-xs#morelist")
|
||||
.find("li.post-11")
|
||||
.each(function (result) {
|
||||
let chapterName = $(this).find("a").text();
|
||||
|
||||
let releaseDate = null;
|
||||
|
||||
let chapterUrl = $(this).find("a").attr("href");
|
||||
|
||||
if (chapterUrl) {
|
||||
chapterUrl = chapterUrl.replace(`/${novelUrl}/`, "");
|
||||
}
|
||||
|
||||
const chapter = {
|
||||
chapterName,
|
||||
releaseDate,
|
||||
chapterUrl,
|
||||
};
|
||||
|
||||
novelChapters.push(chapter);
|
||||
});
|
||||
|
||||
novel.chapters = novelChapters;
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = `${baseUrl}${novelUrl}/${chapterUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
const chapterName = $("h1").text();
|
||||
let chapterText = $("div.entry-content").html();
|
||||
chapterText = htmlToText(chapterText);
|
||||
|
||||
let nextChapter = null;
|
||||
nextChapter = $('a[rel="next"]').attr("href");
|
||||
if (nextChapter) {
|
||||
nextChapter = nextChapter.replace(`/${novelUrl}/`, "");
|
||||
}
|
||||
|
||||
let prevChapter = null;
|
||||
prevChapter = $('a[rel="prev"]').attr("href");
|
||||
if (prevChapter) {
|
||||
prevChapter = prevChapter.replace(`/${novelUrl}/`, "");
|
||||
}
|
||||
|
||||
chapter = {
|
||||
sourceId: 6,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
nextChapter,
|
||||
prevChapter,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const searchUrl = `${baseUrl}index.php?s=so&module=book&keyword=`;
|
||||
|
||||
const url = `${searchUrl}${searchTerm}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$("tr").each(function (result) {
|
||||
let novelName = $(this).find("td:nth-child(2)").text();
|
||||
novelName = novelName.replace(/[\t\n]/g, "");
|
||||
|
||||
const novelCover = "https://cdn.novelupdates.com/imgmid/noimagemid.jpg";
|
||||
|
||||
let novelUrl = $(this).find("td:nth-child(2) >").attr("href");
|
||||
if (novelUrl) {
|
||||
novelUrl = novelUrl.slice(1);
|
||||
}
|
||||
|
||||
novel = {
|
||||
sourceId: 6,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const novelhallScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default novelhallScraper;
|
||||
@@ -0,0 +1,200 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
|
||||
import { htmlToText } from "../helpers/htmlToText";
|
||||
|
||||
const baseUrl = "https://noveltrench.com/";
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
let url = baseUrl + "manga/page/" + page;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".page-item-detail").each(function (result) {
|
||||
const novelName = $(this).find(".h5 > a").text();
|
||||
const novelCover = $(this).find("img").attr("data-src");
|
||||
|
||||
let novelUrl = $(this).find(".h5 > a").attr("href");
|
||||
novelUrl = novelUrl.replace(`${baseUrl}manga/`, "");
|
||||
|
||||
const novel = {
|
||||
sourceId: 9,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = `${baseUrl}manga/${novelUrl}/`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.sourceId = 9;
|
||||
|
||||
novel.sourceName = "NovelTrench";
|
||||
|
||||
novel.url = url;
|
||||
|
||||
novel.novelUrl = novelUrl;
|
||||
|
||||
novel.novelName = $(".post-title > h1")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
novel.novelCover = $(".summary_image > a > img").attr("data-src");
|
||||
|
||||
$(".post-content_item").each(function (result) {
|
||||
detailName = $(this)
|
||||
.find(".summary-heading > h5")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
detail = $(this)
|
||||
.find(".summary-content")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
switch (detailName) {
|
||||
case "Genre(s)":
|
||||
novel.genre = detail.trim().replace(/[\t\n]/g, ",");
|
||||
break;
|
||||
case "Author(s)":
|
||||
novel.author = detail.trim();
|
||||
break;
|
||||
case "Artist(s)":
|
||||
novel.status = detail.trim();
|
||||
break;
|
||||
}
|
||||
});
|
||||
|
||||
$(".description-summary > div.summary__content").find("em").remove();
|
||||
|
||||
novel.summary = $(".description-summary > div.summary__content")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "");
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
$(".wp-manga-chapter").each(function (result) {
|
||||
chapterName = $(this)
|
||||
.find("a")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
releaseDate = $(this)
|
||||
.find("span")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
chapterUrl = $(this).find("a").attr("href").replace(url, "");
|
||||
|
||||
novelChapters.push({ chapterName, releaseDate, chapterUrl });
|
||||
});
|
||||
|
||||
novel.chapters = novelChapters.reverse();
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = chapterUrl;
|
||||
|
||||
console.log(url);
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
const chapterName = $("h1#chapter-heading").text();
|
||||
let chapterText = $(".reading-content").html();
|
||||
chapterText = htmlToText(chapterText);
|
||||
|
||||
let nextChapter = null;
|
||||
|
||||
if ($(".nav-next").length) {
|
||||
nextChapter = $(".nav-next")
|
||||
.find("a")
|
||||
.attr("href")
|
||||
.replace(baseUrl + "manga/" + novelUrl + "/", "");
|
||||
}
|
||||
|
||||
let prevChapter = null;
|
||||
|
||||
if ($(".nav-previous").length) {
|
||||
prevChapter = $(".nav-previous")
|
||||
.find("a")
|
||||
.attr("href")
|
||||
.replace(baseUrl + "manga/" + novelUrl + "/", "");
|
||||
}
|
||||
|
||||
const chapter = {
|
||||
sourceId: 9,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
nextChapter,
|
||||
prevChapter,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const url = `${baseUrl}?s=${searchTerm}&post_type=wp-manga`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".c-tabs-item__content").each(function (result) {
|
||||
const novelName = $(this).find(".h4 > a").text();
|
||||
const novelCover = $(this).find("img").attr("data-src");
|
||||
|
||||
let novelUrl = $(this).find(".h4 > a").attr("href");
|
||||
novelUrl = novelUrl.replace(`${baseUrl}manga/`, "");
|
||||
|
||||
const novel = {
|
||||
sourceId: 9,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const novelTrenchScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default novelTrenchScraper;
|
||||
@@ -0,0 +1,159 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
|
||||
import { htmlToText } from "../helpers/htmlToText";
|
||||
|
||||
const baseUrl = "https://www.novelupdates.cc/";
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
const result = await fetch(baseUrl);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".section-item").each(function (result) {
|
||||
const novelName = $(this).find(".book-name").text();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
|
||||
let novelUrl = $(this).find("a").attr("href").slice(1);
|
||||
|
||||
const novel = {
|
||||
sourceId: 18,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = `${baseUrl}${novelUrl}/`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.sourceId = 18;
|
||||
|
||||
novel.sourceName = "NovelUpdates.cc";
|
||||
|
||||
novel.url = url;
|
||||
|
||||
novel.novelUrl = novelUrl;
|
||||
|
||||
novel.novelName = $(".book-name").text();
|
||||
|
||||
novel.novelCover = $("div.book-img > img").attr("src");
|
||||
|
||||
novel.genre = $("div.book-catalog > span.txt").text();
|
||||
|
||||
novel.Status = $("div.book-state > span.txt").text();
|
||||
|
||||
novel.author = $("div.author > span.name").text();
|
||||
|
||||
novelSummary = $("div.content > p.desc").html();
|
||||
novel.summary = htmlToText(novelSummary);
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
$(".chapter-item").each(function (result) {
|
||||
const chapterName = $(this).find(".chapter-name").text();
|
||||
const releaseDate = null;
|
||||
const chapterUrl = $(this).attr("href").slice(1);
|
||||
|
||||
novelChapters.push({
|
||||
chapterName,
|
||||
releaseDate,
|
||||
chapterUrl,
|
||||
});
|
||||
});
|
||||
|
||||
novel.chapters = novelChapters;
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = `${baseUrl}${chapterUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
const chapterName = $("h1.chapter-title").text();
|
||||
let chapterText = $("div.chapter-entity").html();
|
||||
chapterText = htmlToText(chapterText);
|
||||
|
||||
let nextChapter = null;
|
||||
|
||||
if ($("a.next").length) {
|
||||
nextChapter = $("a.next").attr("href").slice(1);
|
||||
}
|
||||
|
||||
let prevChapter = null;
|
||||
|
||||
if ($("a.prev").length) {
|
||||
prevChapter = $("a.prev").attr("href").slice(1);
|
||||
}
|
||||
|
||||
const chapter = {
|
||||
sourceId: 18,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
nextChapter,
|
||||
prevChapter,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const url = `${baseUrl}search/${searchTerm}/1`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".list-item").each(function (result) {
|
||||
$(this).find("font").remove();
|
||||
|
||||
const novelName = $(this).find(".book-name").text();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
|
||||
let novelUrl = $(this).find("a").attr("href").slice(1);
|
||||
|
||||
const novel = {
|
||||
sourceId: 18,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const novelUpdatesCcScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default novelUpdatesCcScraper;
|
||||
@@ -0,0 +1,159 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
|
||||
import { htmlToText } from "../helpers/htmlToText";
|
||||
|
||||
const baseUrl = "https://m.readlightnovel.cc/";
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
const result = await fetch(baseUrl);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".section-item").each(function (result) {
|
||||
const novelName = $(this).find(".book-name").text();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
|
||||
let novelUrl = $(this).find("a").attr("href").slice(1);
|
||||
|
||||
const novel = {
|
||||
sourceId: 19,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = `${baseUrl}${novelUrl}/`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.sourceId = 19;
|
||||
|
||||
novel.sourceName = "ReadLightNovel.cc";
|
||||
|
||||
novel.url = url;
|
||||
|
||||
novel.novelUrl = novelUrl;
|
||||
|
||||
novel.novelName = $(".book-name").text();
|
||||
|
||||
novel.novelCover = $("div.book-img > img").attr("src");
|
||||
|
||||
novel.genre = $("div.book-catalog > span.txt").text();
|
||||
|
||||
novel.Status = $("div.book-state > span.txt").text();
|
||||
|
||||
novel.author = $("div.author > span.name").text();
|
||||
|
||||
novelSummary = $("div.content > p.desc").html();
|
||||
novel.summary = htmlToText(novelSummary);
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
$(".chapter-item").each(function (result) {
|
||||
const chapterName = $(this).find(".chapter-name").text();
|
||||
const releaseDate = null;
|
||||
const chapterUrl = $(this).attr("href").slice(1);
|
||||
|
||||
novelChapters.push({
|
||||
chapterName,
|
||||
releaseDate,
|
||||
chapterUrl,
|
||||
});
|
||||
});
|
||||
|
||||
novel.chapters = novelChapters;
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = `${baseUrl}${chapterUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
const chapterName = $("h1.chapter-title").text();
|
||||
let chapterText = $("div.chapter-entity").html();
|
||||
chapterText = htmlToText(chapterText);
|
||||
|
||||
let nextChapter = null;
|
||||
|
||||
if ($("a.next").length) {
|
||||
nextChapter = $("a.next").attr("href").slice(1);
|
||||
}
|
||||
|
||||
let prevChapter = null;
|
||||
|
||||
if ($("a.prev").length) {
|
||||
prevChapter = $("a.prev").attr("href").slice(1);
|
||||
}
|
||||
|
||||
const chapter = {
|
||||
sourceId: 19,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
nextChapter,
|
||||
prevChapter,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const url = `${baseUrl}search/${searchTerm}/1`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".list-item").each(function (result) {
|
||||
$(this).find("font").remove();
|
||||
|
||||
const novelName = $(this).find(".book-name").text();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
|
||||
let novelUrl = $(this).find("a").attr("href").slice(1);
|
||||
|
||||
const novel = {
|
||||
sourceId: 19,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const readLightNovelCcScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default readLightNovelCcScraper;
|
||||
@@ -0,0 +1,205 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
|
||||
import { htmlToText } from "../helpers/htmlToText";
|
||||
|
||||
const baseUrl = "https://readnovelfull.com";
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
const url = `${baseUrl}/most-popular-novel?page=${page}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$("div.col-novel-main > div.list-novel > .row").each(function (result) {
|
||||
let novelUrl = $(this)
|
||||
.find("h3.novel-title > a")
|
||||
.attr("href")
|
||||
.replace(".html", "")
|
||||
.substring(1);
|
||||
novelUrl = `${novelUrl}/`;
|
||||
|
||||
const novelName = $(this).find("h3.novel-title > a").text();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
|
||||
const novel = {
|
||||
sourceId: 4,
|
||||
novelUrl,
|
||||
novelName,
|
||||
novelCover,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = `${baseUrl}/${novelUrl.slice(0, -1)}.html`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.sourceId = 4;
|
||||
|
||||
novel.sourceName = "ReadNovelFull";
|
||||
|
||||
novel.url = url;
|
||||
|
||||
novel.novelUrl = novelUrl;
|
||||
|
||||
novel.novelName = $("div.book > img").attr("alt");
|
||||
|
||||
novel.novelCover = $("div.book > img").attr("src");
|
||||
|
||||
novel.summary = $("div.desc-text").text();
|
||||
|
||||
novel.author = $("li > h3")
|
||||
.filter(function () {
|
||||
return $(this).text().trim() === "Author:";
|
||||
})
|
||||
.siblings()
|
||||
.text();
|
||||
|
||||
novel.genre = $("li")
|
||||
.filter(function () {
|
||||
return $(this).find("h3").text().trim() === "Genre:";
|
||||
})
|
||||
.text()
|
||||
.replace("Genre:", "");
|
||||
|
||||
novel.artist = null;
|
||||
|
||||
novel.Status = $("li > h3")
|
||||
.filter(function () {
|
||||
return $(this).text().trim() === "Status:";
|
||||
})
|
||||
.siblings()
|
||||
.text();
|
||||
|
||||
const novelId = $("#rating").attr("data-novel-id");
|
||||
|
||||
const getChapters = async (novelId) => {
|
||||
const chapterListUrl =
|
||||
"https://readnovelfull.com/ajax/chapter-archive?novelId=" + novelId;
|
||||
|
||||
const data = await fetch(chapterListUrl);
|
||||
const chapters = await data.text();
|
||||
|
||||
$ = cheerio.load(chapters);
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
$(".panel-body")
|
||||
.find("li")
|
||||
.each(function (result) {
|
||||
let chapterName = $(this).find("a").attr("title");
|
||||
let releaseDate = null;
|
||||
let chapterUrl = $(this).find("a").attr("href");
|
||||
chapterUrl = chapterUrl.replace(`/${novelUrl}/`, "");
|
||||
|
||||
novelChapters.push({
|
||||
chapterName,
|
||||
releaseDate,
|
||||
chapterUrl,
|
||||
});
|
||||
});
|
||||
return novelChapters;
|
||||
};
|
||||
|
||||
novel.chapters = await getChapters(novelId);
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = `${baseUrl}${chapterUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
const chapterName = $(".chr-title").attr("title");
|
||||
let chapterText = $("#chr-content").html();
|
||||
chapterText = htmlToText(chapterText);
|
||||
|
||||
let nextChapter = null;
|
||||
if ($("a#next_chap").attr("href")) {
|
||||
nextChapter = $("a#next_chap")
|
||||
.attr("href")
|
||||
.replace(`/${novelUrl}/`, "");
|
||||
}
|
||||
|
||||
let prevChapter = null;
|
||||
if ($("a#prev_chap").attr("href")) {
|
||||
prevChapter = $("a#prev_chap")
|
||||
.attr("href")
|
||||
.replace(`/${novelUrl}/`, "");
|
||||
}
|
||||
|
||||
const chapter = {
|
||||
sourceId: 4,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
nextChapter,
|
||||
prevChapter,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const searchUrl = `https://readnovelfull.com/search?keyword=`;
|
||||
|
||||
const url = `${searchUrl}${searchTerm}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$("div.col-novel-main > div.list-novel > .row").each(function (result) {
|
||||
let novelUrl = $(this)
|
||||
.find("h3.novel-title > a")
|
||||
.attr("href")
|
||||
.replace(".html", "")
|
||||
.substring(1);
|
||||
novelUrl = `${novelUrl}/`;
|
||||
|
||||
const novelName = $(this).find("h3.novel-title > a").text();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
|
||||
const novel = {
|
||||
sourceId: 4,
|
||||
novelUrl,
|
||||
novelName,
|
||||
novelCover,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const readNovelFullScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default readNovelFullScraper;
|
||||
@@ -0,0 +1,175 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
import { htmlToText } from "../helpers/htmlToText";
|
||||
|
||||
const baseUrl = "http://www.tapread.com";
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
const result = await fetch(baseUrl);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$('.section-item[data-classify="1"]').each(function (result) {
|
||||
const novelName = $(this).find(".book-name").text();
|
||||
const novelCover = "http:" + $(this).find("img").attr("src");
|
||||
let novelUrl = $(this).attr("data-id");
|
||||
novelUrl += "/";
|
||||
|
||||
const novel = {
|
||||
sourceId: 17,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = `${baseUrl}/book/detail/${novelUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.sourceId = 17;
|
||||
|
||||
novel.sourceName = "TapRead";
|
||||
|
||||
novel.url = url;
|
||||
|
||||
novel.novelUrl = novelUrl;
|
||||
|
||||
novel.novelName = $(".book-name").text();
|
||||
|
||||
novel.novelCover = "http:" + $("div.book-img > img").attr("src");
|
||||
|
||||
novel.genre = $("div.book-catalog > span.txt").text();
|
||||
|
||||
novel.Status = $("div.book-state > span.txt").text();
|
||||
|
||||
novel.author = $("div.author > span.name").text();
|
||||
|
||||
novelSummary = $("div.content > p.desc").text();
|
||||
|
||||
const getChapters = async (novelId) => {
|
||||
const chapterListUrl = "http://www.tapread.com/ajax/book/contents";
|
||||
|
||||
let formData = new FormData();
|
||||
formData.append("bookId", novelId);
|
||||
|
||||
const data = await fetch(chapterListUrl, {
|
||||
method: "POST",
|
||||
body: formData,
|
||||
});
|
||||
|
||||
const chapters = await data.json();
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
chapters.result.chapterList.map((chapter) => {
|
||||
let chapterName = chapter.chapterName;
|
||||
const releaseDate = chapter.pubTime;
|
||||
const chapterUrl = chapter.chapterId.toString();
|
||||
|
||||
if (chapter.lock) {
|
||||
chapterName = "🔒 " + chapterName;
|
||||
}
|
||||
|
||||
novelChapters.push({
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
releaseDate,
|
||||
});
|
||||
});
|
||||
|
||||
return novelChapters;
|
||||
};
|
||||
|
||||
novel.chapters = await getChapters(novelUrl);
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
let formData = new FormData();
|
||||
formData.append("bookId", novelUrl);
|
||||
formData.append("chapterId", chapterUrl);
|
||||
|
||||
const url = "http://www.tapread.com/ajax/book/chapter";
|
||||
|
||||
const result = await fetch(url, {
|
||||
method: "POST",
|
||||
body: formData,
|
||||
});
|
||||
const body = await result.json();
|
||||
|
||||
const chapterName = body.result.chapterName;
|
||||
const chapterText = htmlToText(body.result.content);
|
||||
const nextChapter =
|
||||
body.result.nextChapterId === 0
|
||||
? null
|
||||
: body.result.nextChapterId.toString();
|
||||
const prevChapter = body.result.preChapterId.toString();
|
||||
novelUrl += "/";
|
||||
|
||||
const chapter = {
|
||||
sourceId: 17,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
nextChapter,
|
||||
prevChapter,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
let formData = new FormData();
|
||||
formData.append("storyType", 1);
|
||||
formData.append("pageNo", 1);
|
||||
formData.append("searchText", searchTerm);
|
||||
|
||||
const url = "http://www.tapread.com/ajax/search/story";
|
||||
|
||||
const data = await fetch(url, {
|
||||
method: "POST",
|
||||
body: formData,
|
||||
});
|
||||
|
||||
const body = await data.json();
|
||||
|
||||
const novels = [];
|
||||
|
||||
body.result.storyList.map((novel) => {
|
||||
const novelName = novel.storyName.replace(
|
||||
'<font color="#FFCE2E">Demon</font>',
|
||||
""
|
||||
);
|
||||
const novelUrl = novel.storyId + "/";
|
||||
const novelCover = "http://static.tapread.com" + novel.coverUrl;
|
||||
|
||||
novels.push({ sourceId: 17, novelName, novelCover, novelUrl });
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const tapReadScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default tapReadScraper;
|
||||
@@ -0,0 +1,204 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
|
||||
import { htmlToText } from "../helpers/htmlToText";
|
||||
|
||||
const baseUrl = "https://vipnovel.com/";
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
let url = baseUrl;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".page-item-detail").each(function (result) {
|
||||
const novelName = $(this).find("h5 > a").text();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
|
||||
let novelUrl = $(this).find("h5 > a").attr("href");
|
||||
novelUrl = novelUrl.replace(`${baseUrl}vipnovel/`, "");
|
||||
|
||||
const novel = {
|
||||
sourceId: 10,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = `${baseUrl}vipnovel/${novelUrl}/`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.sourceId = 10;
|
||||
|
||||
novel.sourceName = "VipNovel";
|
||||
|
||||
novel.url = url;
|
||||
|
||||
novel.novelUrl = novelUrl;
|
||||
|
||||
novel.novelName = $(".post-title > h3")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
novel.novelCover = $(".summary_image > a > img").attr("src");
|
||||
$(".post-content_item").each(function (result) {
|
||||
detailName = $(this)
|
||||
.find(".summary-heading > h5")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
detail = $(this)
|
||||
.find(".summary-content")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
switch (detailName) {
|
||||
case "Genre(s)":
|
||||
novel.genre = detail.trim().replace(/[\t\n]/g, ",");
|
||||
break;
|
||||
case "Author(s)":
|
||||
novel.author = detail.trim();
|
||||
break;
|
||||
case "Artist(s)":
|
||||
novel.status = detail.trim();
|
||||
break;
|
||||
}
|
||||
});
|
||||
|
||||
// $(".description-summary > div.summary__content").find("em").remove();
|
||||
|
||||
let novelSummary = $(".description-summary > div.summary__content").html();
|
||||
|
||||
novel.summary = htmlToText(novelSummary);
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
$(".wp-manga-chapter").each(function (result) {
|
||||
const chapterName = $(this)
|
||||
.find("a")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
const releaseDate = $(this)
|
||||
.find("span")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
const chapterUrl = $(this).find("a").attr("href").replace(url, "");
|
||||
|
||||
const chapter = { chapterName, releaseDate, chapterUrl };
|
||||
|
||||
novelChapters.push(chapter);
|
||||
});
|
||||
|
||||
novel.chapters = novelChapters.reverse();
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = `${baseUrl}vipnovel/${novelUrl}/${chapterUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let chapterName = $(".text-center").text();
|
||||
|
||||
if (!chapterName) {
|
||||
chapterName = $(".text-left > div> h3").text();
|
||||
}
|
||||
|
||||
let chapterText = $(".text-left").html();
|
||||
chapterText = htmlToText(chapterText);
|
||||
|
||||
let nextChapter = null;
|
||||
|
||||
if ($(".nav-next").length) {
|
||||
nextChapter = $(".nav-next")
|
||||
.find("a")
|
||||
.attr("href")
|
||||
.replace(baseUrl + "vipnovel/" + novelUrl + "/", "");
|
||||
}
|
||||
|
||||
let prevChapter = null;
|
||||
|
||||
if ($(".nav-previous").length) {
|
||||
prevChapter = $(".nav-previous")
|
||||
.find("a")
|
||||
.attr("href")
|
||||
.replace(baseUrl + "vipnovel/" + novelUrl + "/", "");
|
||||
}
|
||||
|
||||
const chapter = {
|
||||
sourceId: 10,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
nextChapter,
|
||||
prevChapter,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const url = `${baseUrl}?s=${searchTerm}&post_type=wp-manga`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".c-tabs-item__content").each(function (result) {
|
||||
const novelName = $(this).find("h4 > a").text();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
|
||||
let novelUrl = $(this).find("h4 > a").attr("href");
|
||||
novelUrl = novelUrl.replace(`${baseUrl}vipnovel/`, "");
|
||||
|
||||
const novel = {
|
||||
sourceId: 10,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const vipNovelScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default vipNovelScraper;
|
||||
@@ -0,0 +1,202 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
|
||||
import { htmlToText } from "../helpers/htmlToText";
|
||||
|
||||
const baseUrl = "https://woopread.com/";
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
let url = baseUrl + "novellist/?m_orderby=rating";
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".page-item-detail").each(function (result) {
|
||||
const novelName = $(this).find(".h5 > a").text();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
|
||||
let novelUrl = $(this).find(".h5 > a").attr("href");
|
||||
novelUrl = novelUrl.replace(baseUrl + "series/", "");
|
||||
|
||||
const novel = {
|
||||
sourceId: 21,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = `${baseUrl}series/${novelUrl}/`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.sourceId = 21;
|
||||
|
||||
novel.sourceName = "WoopRead";
|
||||
|
||||
novel.url = url;
|
||||
|
||||
novel.novelUrl = novelUrl;
|
||||
|
||||
novel.novelName = $(".post-title > h1")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
novel.novelCover = $(".summary_image > a > img").attr("src");
|
||||
|
||||
$(".post-content_item").each(function (result) {
|
||||
detailName = $(this)
|
||||
.find(".summary-heading > h5")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
detail = $(this)
|
||||
.find(".summary-content")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
novel[detailName] = detail;
|
||||
});
|
||||
|
||||
$(".description-summary > div.summary__content").find("em").remove();
|
||||
|
||||
novel.summary = $(".description-summary > div.summary__content")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "");
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
const novelId = $(".rating-post-id").attr("value");
|
||||
|
||||
let formData = new FormData();
|
||||
formData.append("action", "manga_get_chapters");
|
||||
formData.append("manga", novelId);
|
||||
|
||||
const data = await fetch("https://woopread.com/wp-admin/admin-ajax.php", {
|
||||
method: "POST",
|
||||
body: formData,
|
||||
});
|
||||
const text = await data.text();
|
||||
|
||||
$ = cheerio.load(text);
|
||||
|
||||
$(".wp-manga-chapter.free-chap").each(function (result) {
|
||||
chapterName = $(this)
|
||||
.find("a")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
releaseDate = $(this).find("span").text().replace("Free", "").trim();
|
||||
|
||||
chapterUrl = $(this).find("a").attr("href").replace(url, "");
|
||||
|
||||
novelChapters.push({ chapterName, releaseDate, chapterUrl });
|
||||
});
|
||||
|
||||
novel.chapters = novelChapters.reverse();
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = `${baseUrl}series/${novelUrl}/${chapterUrl}/`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
const chapterName = $("h1#chapter-heading").text();
|
||||
let chapterText = $(".reading-content").html();
|
||||
chapterText = htmlToText(chapterText);
|
||||
chapterText = chapterText.replace(/(?<=[[])[\n](?=[h])/g, "");
|
||||
|
||||
let nextChapter = null;
|
||||
|
||||
if ($(".nav-next").length) {
|
||||
nextChapter = $(".nav-next")
|
||||
.find("a")
|
||||
.attr("href")
|
||||
.replace(baseUrl + "series/" + novelUrl + "/", "");
|
||||
}
|
||||
|
||||
let prevChapter = null;
|
||||
|
||||
if ($(".nav-previous").length) {
|
||||
prevChapter = $(".nav-previous")
|
||||
.find("a")
|
||||
.attr("href")
|
||||
.replace(baseUrl + "series/" + novelUrl + "/", "");
|
||||
}
|
||||
|
||||
novelUrl += "/";
|
||||
chapterUrl = optionalUrl + chapterUrl + "/";
|
||||
|
||||
const chapter = {
|
||||
sourceId: 21,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
nextChapter,
|
||||
prevChapter,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const url = `${baseUrl}?s=${searchTerm}&post_type=wp-manga&op=&author=&artist=&release=&adult=`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".c-tabs-item__content").each(function (result) {
|
||||
const novelName = $(this).find(".h4 > a").text();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
|
||||
let novelUrl = $(this).find(".h4 > a").attr("href");
|
||||
novelUrl = novelUrl.replace(`${baseUrl}series/`, "");
|
||||
|
||||
const novel = {
|
||||
sourceId: 21,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const WoopReadScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default WoopReadScraper;
|
||||
@@ -0,0 +1,179 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
|
||||
import { htmlToText } from "../helpers/htmlToText";
|
||||
|
||||
const baseUrl = "https://www.wuxiaworld.com/";
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
const url = `${baseUrl}api/novels`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const data = await result.json();
|
||||
|
||||
let novels = [];
|
||||
|
||||
data.items.map((novel) => {
|
||||
let novelName = novel.name;
|
||||
let novelCover = novel.coverUrl;
|
||||
let novelUrl = novel.slug;
|
||||
|
||||
novels.push({
|
||||
sourceId: 7,
|
||||
novelUrl,
|
||||
novelName,
|
||||
novelCover,
|
||||
});
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = `${baseUrl}novel/${novelUrl}/`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.sourceId = 7;
|
||||
|
||||
novel.sourceName = "WuxiaWorld";
|
||||
|
||||
novel.url = url;
|
||||
|
||||
novel.novelUrl = novelUrl;
|
||||
|
||||
novel.novelName = $("h2").text();
|
||||
|
||||
novel.novelCover = $("img.img-thumbnail").attr("src");
|
||||
|
||||
novel.summary = $(".fr-view > p").text();
|
||||
|
||||
novel.author = $("div > dt")
|
||||
.filter(function () {
|
||||
return $(this).text().trim() === "Author:";
|
||||
})
|
||||
.next()
|
||||
.text();
|
||||
|
||||
let genres = [];
|
||||
|
||||
$(".genres")
|
||||
.find("div")
|
||||
.each(function (res) {
|
||||
genres.push($(this).find("a").text());
|
||||
});
|
||||
|
||||
novel.genre = genres.join(",");
|
||||
|
||||
novel.artist = null;
|
||||
|
||||
novel.Status = null;
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
$(".chapter-item").each(function (result) {
|
||||
let chapterName = $(this).text();
|
||||
chapterName = chapterName.replace(/[\t\n]/g, "");
|
||||
|
||||
const releaseDate = null;
|
||||
|
||||
let chapterUrl = $(this).find("a").attr("href");
|
||||
chapterUrl = chapterUrl.replace(`/novel/${novelUrl}/`, "");
|
||||
|
||||
const chapter = {
|
||||
chapterName,
|
||||
releaseDate,
|
||||
chapterUrl,
|
||||
};
|
||||
|
||||
novelChapters.push(chapter);
|
||||
});
|
||||
|
||||
novel.chapters = novelChapters;
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
let url = `${baseUrl}novel/${novelUrl}/${chapterUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let chapterName = $("#sidebar-toggler-container").next().text();
|
||||
chapterName = chapterName.replace(/[\t\n]/g, "");
|
||||
|
||||
$("#chapter-content > script").remove();
|
||||
|
||||
let chapterText = $("#chapter-content").html();
|
||||
chapterText = htmlToText(chapterText);
|
||||
|
||||
let nextChapter = null;
|
||||
|
||||
if ($("li.next > a").attr("href") !== "#") {
|
||||
nextChapter = $("li.next > a")
|
||||
.attr("href")
|
||||
.replace(`/novel/${novelUrl}`, "");
|
||||
}
|
||||
|
||||
let prevChapter = null;
|
||||
|
||||
if ($("li.prev > a").attr("href") !== "#") {
|
||||
prevChapter = $("li.prev > a")
|
||||
.attr("href")
|
||||
.replace(`/novel/${novelUrl}`, "");
|
||||
}
|
||||
|
||||
const chapter = {
|
||||
sourceId: 7,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
nextChapter,
|
||||
prevChapter,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const searchUrl = `https://www.wuxiaworld.com/api/novels/search?query=`;
|
||||
|
||||
const url = `${searchUrl}${searchTerm}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const data = await result.json();
|
||||
|
||||
let novels = [];
|
||||
|
||||
data.items.map((novel) => {
|
||||
let novelName = novel.name;
|
||||
let novelCover = novel.coverUrl;
|
||||
let novelUrl = novel.slug;
|
||||
|
||||
novels.push({
|
||||
sourceId: 7,
|
||||
novelUrl,
|
||||
novelName,
|
||||
novelCover,
|
||||
});
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const WuxiaWorldScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default WuxiaWorldScraper;
|
||||
@@ -0,0 +1,200 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
|
||||
import { htmlToText } from "../helpers/htmlToText";
|
||||
|
||||
const baseUrl = "http://wuxiaworld.cloud/";
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
const url = `${baseUrl}/popular-novel`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$("div.col-novel-main > div.list-novel > .row").each(function (result) {
|
||||
let novelUrl = $(this)
|
||||
.find("h3.novel-title > a")
|
||||
.attr("href")
|
||||
.replace(baseUrl + "novel/", "");
|
||||
novelUrl = `${novelUrl}/`;
|
||||
|
||||
const novelName = $(this).find("h3.novel-title > a").text();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
|
||||
const novel = {
|
||||
sourceId: 20,
|
||||
novelUrl,
|
||||
novelName,
|
||||
novelCover,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = `${baseUrl}novel/${novelUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.sourceId = 20;
|
||||
|
||||
novel.sourceName = "WuxiaWorld.cloud";
|
||||
|
||||
novel.url = url;
|
||||
|
||||
novel.novelUrl = novelUrl;
|
||||
|
||||
novel.novelName = $("div.book > img").attr("alt");
|
||||
|
||||
novel.novelCover = $("div.book > img").attr("src");
|
||||
|
||||
novel.summary = $("div.desc-text").text();
|
||||
|
||||
novel.author = $("div.info > div > h3")
|
||||
.filter(function () {
|
||||
return $(this).text().trim() === "Author:";
|
||||
})
|
||||
.siblings()
|
||||
.text();
|
||||
|
||||
novel.genre = $("div.info > div")
|
||||
.filter(function () {
|
||||
return $(this).find("h3").text().trim() === "Genre:";
|
||||
})
|
||||
.text()
|
||||
.replace("Genre:", "")
|
||||
.replace(/\s/g, "");
|
||||
|
||||
novel.artist = null;
|
||||
|
||||
novel.Status = $("li > h3")
|
||||
.filter(function () {
|
||||
return $(this).text().trim() === "Status:";
|
||||
})
|
||||
.siblings()
|
||||
.text();
|
||||
|
||||
const novelId = $("#rating").attr("data-novel-id");
|
||||
|
||||
const getChapters = async (novelId) => {
|
||||
const chapterListUrl =
|
||||
baseUrl + "ajax/chapter-option?novelId=" + novelId;
|
||||
|
||||
const data = await fetch(chapterListUrl);
|
||||
const chapters = await data.text();
|
||||
|
||||
$ = cheerio.load(chapters);
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
$("select > option").each(function (result) {
|
||||
let chapterName = $(this).text();
|
||||
let releaseDate = null;
|
||||
let chapterUrl = $(this).attr("value");
|
||||
chapterUrl = chapterUrl.replace(baseUrl, "");
|
||||
|
||||
novelChapters.push({
|
||||
chapterName,
|
||||
releaseDate,
|
||||
chapterUrl,
|
||||
});
|
||||
});
|
||||
return novelChapters;
|
||||
};
|
||||
|
||||
novel.chapters = await getChapters(novelId);
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
chapterUrl = encodeURI(chapterUrl);
|
||||
|
||||
const url = `${baseUrl}${chapterUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
const chapterName = $(".chr-title").attr("title");
|
||||
let chapterText = $("#chr-content").html();
|
||||
chapterText = htmlToText(chapterText);
|
||||
|
||||
let nextChapter = null;
|
||||
if ($("a#next_chap").attr("href")) {
|
||||
nextChapter = $("a#next_chap").attr("href").replace(baseUrl, "");
|
||||
}
|
||||
|
||||
let prevChapter = null;
|
||||
if ($("a#prev_chap").attr("href")) {
|
||||
prevChapter = $("a#prev_chap").attr("href").replace(baseUrl, "");
|
||||
}
|
||||
|
||||
const chapter = {
|
||||
sourceId: 20,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
nextChapter,
|
||||
prevChapter,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const searchUrl = `http://wuxiaworld.cloud/search?keyword=`;
|
||||
|
||||
const url = `${searchUrl}${searchTerm}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$("div.col-novel-main > div.list-novel > .row").each(function (result) {
|
||||
let novelUrl = $(this)
|
||||
.find("h3.novel-title > a")
|
||||
.attr("href")
|
||||
.replace(baseUrl + "novel/", "");
|
||||
novelUrl = `${novelUrl}/`;
|
||||
|
||||
const novelName = $(this).find("h3.novel-title > a").text();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
|
||||
const novel = {
|
||||
sourceId: 20,
|
||||
novelUrl,
|
||||
novelName,
|
||||
novelCover,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const WuxiaWorldCloudScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default WuxiaWorldCloudScraper;
|
||||
@@ -0,0 +1,163 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
|
||||
import { htmlToText } from "../helpers/htmlToText";
|
||||
|
||||
const baseUrl = "https://www.wuxiaworld.co/";
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
const result = await fetch(baseUrl);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".section-item").each(function (result) {
|
||||
const novelName = $(this).find(".book-name").text();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
|
||||
let novelUrl = $(this).find("a").attr("href").slice(1);
|
||||
|
||||
const novel = {
|
||||
sourceId: 16,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = `${baseUrl}${novelUrl}/`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.sourceId = 16;
|
||||
|
||||
novel.sourceName = "WuxiaWorldCo";
|
||||
|
||||
novel.url = url;
|
||||
|
||||
novel.novelUrl = novelUrl;
|
||||
|
||||
novel.novelName = $(".book-name").text();
|
||||
|
||||
novel.novelCover = $("div.book-img > img").attr("src");
|
||||
|
||||
novel.genre = $("div.book-catalog > span.txt").text();
|
||||
|
||||
novel.Status = $("div.book-state > span.txt").text();
|
||||
|
||||
novel.author = $("div.author > span.name").text();
|
||||
|
||||
novelSummary = $("div.content > p.desc").html();
|
||||
novel.summary = htmlToText(novelSummary);
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
$(".chapter-item").each(function (result) {
|
||||
const chapterName = $(this).find(".chapter-name").text();
|
||||
const releaseDate = null;
|
||||
const chapterUrl = $(this).attr("href").split("/")[2];
|
||||
|
||||
novelChapters.push({
|
||||
chapterName,
|
||||
releaseDate,
|
||||
chapterUrl,
|
||||
});
|
||||
});
|
||||
|
||||
novel.chapters = novelChapters;
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = `${baseUrl}${novelUrl}/${chapterUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
const chapterName = $("h1.chapter-title").text();
|
||||
let chapterText = $("div.chapter-entity").html();
|
||||
chapterText = htmlToText(chapterText);
|
||||
|
||||
let nextChapter = null;
|
||||
|
||||
if ($("a.next").length) {
|
||||
nextChapter = $("a.next")
|
||||
.attr("href")
|
||||
.replace("/" + novelUrl + "/", "");
|
||||
}
|
||||
|
||||
let prevChapter = null;
|
||||
|
||||
if ($("a.prev").length) {
|
||||
prevChapter = $("a.prev")
|
||||
.attr("href")
|
||||
.replace("/" + novelUrl + "/", "");
|
||||
}
|
||||
|
||||
const chapter = {
|
||||
sourceId: 16,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
nextChapter,
|
||||
prevChapter,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const url = `${baseUrl}search/${searchTerm}/1`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".list-item").each(function (result) {
|
||||
$(this).find("font").remove();
|
||||
|
||||
const novelName = $(this).find(".book-name").text();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
|
||||
let novelUrl = $(this).find("a").attr("href").slice(1);
|
||||
|
||||
const novel = {
|
||||
sourceId: 16,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const WuxiaWorldCoScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default WuxiaWorldCoScraper;
|
||||
@@ -0,0 +1,200 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
|
||||
const baseUrl = "https://wuxiaworldsite.co/";
|
||||
const searchUrl = "https://wuxiaworldsite.co/search/";
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
const url = `${baseUrl}/power-ranking`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".a_item").each((i, el) => {
|
||||
const novelName = $(el).find("a").attr("title");
|
||||
const novelCover = baseUrl + $(el).find("a > img").attr("src");
|
||||
|
||||
let novelUrl = $(el).find(".name_views > a").attr("href");
|
||||
novelId = novelUrl.split("/");
|
||||
novelUrl = novelId[2] + "/";
|
||||
|
||||
const novel = {
|
||||
sourceId: 12,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = `${baseUrl}${novelUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
$(".category_list").remove();
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.sourceId = 12;
|
||||
|
||||
novel.sourceName = "WuxiaWorldSite";
|
||||
|
||||
novel.url = url;
|
||||
|
||||
novel.novelUrl = novelUrl;
|
||||
|
||||
novel.novelName = $(".content-reading > h1").text().trim();
|
||||
|
||||
novel.novelCover = baseUrl + $(".img-read> img").attr("src");
|
||||
|
||||
novel.summary = $(".story-introduction-content").text();
|
||||
|
||||
novel.author = $(".content-reading > p").text();
|
||||
|
||||
novel.artist = null;
|
||||
|
||||
novel.genre = "";
|
||||
|
||||
$(".a_tag_item").each((i, el) => {
|
||||
novel.genre += $(el).text() + ",";
|
||||
});
|
||||
|
||||
novel.genre = novel.genre.split(",");
|
||||
novel.genre.pop();
|
||||
|
||||
novel.Status = novel.genre.pop();
|
||||
|
||||
novel.genre = novel.genre.join(",");
|
||||
|
||||
const novelID = $(".show-more-list").attr("data-id");
|
||||
|
||||
const getChapters = async (novelID) => {
|
||||
const chapterListUrl = baseUrl + "/get-full-list.ajax?id=" + novelID;
|
||||
|
||||
const data = await fetch(chapterListUrl);
|
||||
const chapters = await data.text();
|
||||
|
||||
$ = cheerio.load(chapters);
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
$(".new-update-content").each((i, el) => {
|
||||
let chapterName = $(el).text().split(/\t+/);
|
||||
const releaseDate = chapterName.pop();
|
||||
chapterName = chapterName[0];
|
||||
let chapterUrl = $(el).attr("href");
|
||||
chapterUrl = chapterUrl.split("/").pop();
|
||||
|
||||
const novel = {
|
||||
chapterName,
|
||||
releaseDate,
|
||||
chapterUrl,
|
||||
};
|
||||
novelChapters.push(novel);
|
||||
});
|
||||
return novelChapters;
|
||||
};
|
||||
|
||||
novel.chapters = await getChapters(novelID);
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = `${baseUrl}/${novelUrl}/${chapterUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let chapterText = $("#content").html() || "";
|
||||
let chapterName = "";
|
||||
if (chapterText) {
|
||||
chapterText = chapterText.split("<br>");
|
||||
chapterName = chapterText.shift();
|
||||
} else {
|
||||
$("p").each((i, el) => {
|
||||
if ($(el).css("display") !== "none") {
|
||||
chapterText += $(el).text() + "\n";
|
||||
}
|
||||
});
|
||||
chapterText = chapterText.split("\n");
|
||||
chapterName = chapterText.shift();
|
||||
}
|
||||
|
||||
chapterText = chapterText.join("\n");
|
||||
|
||||
const prevChapter = $(".pre").attr("href")
|
||||
? $(".pre").attr("href").split("/").pop()
|
||||
: null;
|
||||
|
||||
const nextChapter = $(".next_sesction > a").attr("href")
|
||||
? $(".next_sesction > a").attr("href").split("/").pop()
|
||||
: null;
|
||||
|
||||
novelUrl += "/";
|
||||
chapterUrl += "/";
|
||||
|
||||
const chapter = {
|
||||
sourceId: 12,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
nextChapter,
|
||||
prevChapter,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const url = `${searchUrl}${searchTerm}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".a_item").each((i, el) => {
|
||||
const novelName = $(el).find(".name_views > a").attr("title");
|
||||
const novelCover = baseUrl + $(el).find("a > img").attr("src");
|
||||
let novelUrl = $(el).find(".name_views > a").attr("href");
|
||||
novelId = novelUrl.split("/");
|
||||
novelUrl = novelId[2] + "/";
|
||||
|
||||
const novel = {
|
||||
sourceId: 12,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const WuxiaWorldSiteScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default WuxiaWorldSiteScraper;
|
||||
@@ -1,6 +1,26 @@
|
||||
import ComradeMaoScraper from "./en/comrademao";
|
||||
import ReadLightNovelScraper from "./en/readlightnovel";
|
||||
import BoxNovelScraper from "./en/boxnovel";
|
||||
import fastNovelScraper from "./en/fastnovel";
|
||||
import readNovelFullScraper from "./en/readnovelfull";
|
||||
import mtlNovelScraper from "./en/mtlnovel";
|
||||
import novelhallScraper from "./en/novelhall";
|
||||
import WuxiaWorldScraper from "./en/wuxiaworld";
|
||||
import novelFullScraper from "./en/novelfull";
|
||||
import novelTrenchScraper from "./en/noveltrench";
|
||||
import vipNovelScraper from "./en/vipnovel";
|
||||
import kissLightNovelScraper from "./en/kisslightnovels";
|
||||
import WuxiaWorldSiteScraper from "./en/wuxiaworldsite";
|
||||
import FreeWebNovelScraper from "./en/freewebnovel";
|
||||
import JPMTLScraper from "./en/jpmtl";
|
||||
import lightNovelPubScraper from "./en/lightnovelpub";
|
||||
import WuxiaWorldCoScraper from "./en/wuxiaworldco";
|
||||
import novelUpdatesCcScraper from "./en/novelupdatescc";
|
||||
import readLightNovelCcScraper from "./en/readlightnovelcc";
|
||||
import tapReadScraper from "./en/tapread";
|
||||
import WuxiaWorldCloudScraper from "./en/wuxiaworldcloud";
|
||||
import WoopReadScraper from "./en/woopread";
|
||||
import FoxaHolicScraper from "./en/foxaholic";
|
||||
|
||||
export const sources = [
|
||||
{
|
||||
@@ -17,6 +37,146 @@ export const sources = [
|
||||
lang: "English",
|
||||
icon: "https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/readlightnovel/icon.png?raw=true",
|
||||
},
|
||||
{
|
||||
sourceId: 3,
|
||||
url: "https://fastnovel.net/",
|
||||
sourceName: "FastNovel",
|
||||
icon: "https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/fastnovel/icon.png?raw=true",
|
||||
lang: "English",
|
||||
},
|
||||
{
|
||||
sourceId: 4,
|
||||
url: "https://readnovelfull.com/",
|
||||
sourceName: "ReadNovelFull",
|
||||
icon: "https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/readnovelfull/icon.png?raw=true",
|
||||
lang: "English",
|
||||
},
|
||||
{
|
||||
sourceId: 5,
|
||||
url: "https://www.mtlnovel.com/",
|
||||
sourceName: "MTLNovel",
|
||||
icon: "https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/mtlnovel/icon.png?raw=true",
|
||||
lang: "English",
|
||||
},
|
||||
{
|
||||
sourceId: 6,
|
||||
url: "https://www.novelhall.com/",
|
||||
sourceName: "NovelHall",
|
||||
icon: "https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/novelhall/icon.png?raw=true",
|
||||
lang: "English",
|
||||
},
|
||||
{
|
||||
sourceId: 7,
|
||||
url: "https://www.wuxiaworld.com/",
|
||||
sourceName: "WuxiaWorld",
|
||||
icon: "https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/wuxiaworld/icon.png?raw=true",
|
||||
lang: "English",
|
||||
},
|
||||
{
|
||||
sourceId: 8,
|
||||
url: "https://novelfull.com/",
|
||||
sourceName: "NovelFull",
|
||||
icon: "https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/novelfull/icon.png?raw=true",
|
||||
lang: "English",
|
||||
},
|
||||
{
|
||||
sourceId: 9,
|
||||
url: "https://noveltrench.com/",
|
||||
sourceName: "NovelTrench",
|
||||
icon: "https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/noveltrench/icon.png?raw=true",
|
||||
lang: "English",
|
||||
},
|
||||
{
|
||||
sourceId: 10,
|
||||
url: "https://vipnovel.com/",
|
||||
sourceName: "VipNovel",
|
||||
icon: "https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/vipnovel/icon.png?raw=true",
|
||||
lang: "English",
|
||||
},
|
||||
{
|
||||
sourceId: 11,
|
||||
url: "https://kisslightnovels.info/",
|
||||
sourceName: "KissLightNovels",
|
||||
icon: "https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/kisslightnovels/icon.png?raw=true",
|
||||
lang: "English",
|
||||
},
|
||||
{
|
||||
sourceId: 12,
|
||||
url: "https://wuxiaworldsite.co/",
|
||||
sourceName: "WuxiaWorldSite",
|
||||
icon: "https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/wuxiaworldsite/icon.png?raw=true",
|
||||
lang: "English",
|
||||
},
|
||||
{
|
||||
sourceId: 13,
|
||||
url: "https://freewebnovel.com/",
|
||||
sourceName: "FreeWebNovel",
|
||||
icon: "https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/freewebnovel/icon.png?raw=true",
|
||||
lang: "English",
|
||||
},
|
||||
{
|
||||
sourceId: 14,
|
||||
url: "https://jpmtl.com/",
|
||||
sourceName: "JPMTL",
|
||||
icon: "https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/jpmtl/icon.png?raw=true",
|
||||
lang: "English",
|
||||
},
|
||||
{
|
||||
sourceId: 15,
|
||||
url: "https://www.lightnovelpub.com/",
|
||||
sourceName: "LightNovelPub",
|
||||
icon: "https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/lightnovelpub/icon.png?raw=true",
|
||||
lang: "English",
|
||||
},
|
||||
{
|
||||
sourceId: 16,
|
||||
url: "https://www.wuxiaworld.co/",
|
||||
sourceName: "WuxiaWorld.co",
|
||||
icon: "https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/wuxiaworldco/icon.png?raw=true",
|
||||
lang: "English",
|
||||
},
|
||||
{
|
||||
sourceId: 17,
|
||||
url: "http://www.tapread.com/",
|
||||
sourceName: "TapRead",
|
||||
icon: "https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/tapread/icon.png?raw=true",
|
||||
lang: "English",
|
||||
},
|
||||
{
|
||||
sourceId: 18,
|
||||
url: "https://www.novelupdates.cc/",
|
||||
sourceName: "NovelUpdates.cc",
|
||||
icon: "https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/novelupdatescc/icon.png?raw=true",
|
||||
lang: "English",
|
||||
},
|
||||
{
|
||||
sourceId: 19,
|
||||
url: "https://m.readlightnovel.cc/",
|
||||
sourceName: "ReadLightNovel.cc",
|
||||
icon: "https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/readlightnovelcc/icon.png?raw=true",
|
||||
lang: "English",
|
||||
},
|
||||
{
|
||||
sourceId: 20,
|
||||
url: "http://wuxiaworld.cloud/",
|
||||
sourceName: "WuxiaWorld.cloud",
|
||||
icon: "https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/wuxiaworldcloud/icon.png?raw=true",
|
||||
lang: "English",
|
||||
},
|
||||
{
|
||||
sourceId: 21,
|
||||
url: "https://woopread.com/",
|
||||
sourceName: "WoopRead",
|
||||
icon: "https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/woopread/icon.png?raw=true",
|
||||
lang: "English",
|
||||
},
|
||||
{
|
||||
sourceId: 22,
|
||||
url: "https://foxaholic.com/",
|
||||
sourceName: "Foxaholic",
|
||||
icon: "https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/foxaholic/icon.png?raw=true",
|
||||
lang: "English",
|
||||
},
|
||||
{
|
||||
sourceId: 27,
|
||||
url: "https://comrademao.com/",
|
||||
@@ -34,8 +194,83 @@ export const getSource = (sourceId) => {
|
||||
case 2: {
|
||||
return ReadLightNovelScraper;
|
||||
}
|
||||
case 3: {
|
||||
return fastNovelScraper;
|
||||
}
|
||||
case 4: {
|
||||
return readNovelFullScraper;
|
||||
}
|
||||
case 5: {
|
||||
return mtlNovelScraper;
|
||||
}
|
||||
case 6: {
|
||||
return novelhallScraper;
|
||||
}
|
||||
case 7: {
|
||||
return WuxiaWorldScraper;
|
||||
}
|
||||
case 8: {
|
||||
return novelFullScraper;
|
||||
}
|
||||
case 9: {
|
||||
return novelTrenchScraper;
|
||||
}
|
||||
case 10: {
|
||||
return vipNovelScraper;
|
||||
}
|
||||
case 11: {
|
||||
return kissLightNovelScraper;
|
||||
}
|
||||
case 12: {
|
||||
return WuxiaWorldSiteScraper;
|
||||
}
|
||||
case 13: {
|
||||
return FreeWebNovelScraper;
|
||||
}
|
||||
case 14: {
|
||||
return JPMTLScraper;
|
||||
}
|
||||
case 15: {
|
||||
return lightNovelPubScraper;
|
||||
}
|
||||
case 16: {
|
||||
return WuxiaWorldCoScraper;
|
||||
}
|
||||
case 17: {
|
||||
return tapReadScraper;
|
||||
}
|
||||
case 18: {
|
||||
return novelUpdatesCcScraper;
|
||||
}
|
||||
case 19: {
|
||||
return readLightNovelCcScraper;
|
||||
}
|
||||
case 20: {
|
||||
return WuxiaWorldCloudScraper;
|
||||
}
|
||||
case 21: {
|
||||
return WoopReadScraper;
|
||||
}
|
||||
case 22: {
|
||||
return FoxaHolicScraper;
|
||||
}
|
||||
case 23: {
|
||||
return;
|
||||
}
|
||||
case 24: {
|
||||
return;
|
||||
}
|
||||
case 25: {
|
||||
return;
|
||||
}
|
||||
case 26: {
|
||||
return;
|
||||
}
|
||||
case 27: {
|
||||
return ComradeMaoScraper;
|
||||
}
|
||||
case 3: {
|
||||
return;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user