From cf5d9e18bbd0a3f611ea8d1b2dd4c8176752acf3 Mon Sep 17 00:00:00 2001 From: Rajarshee Chatterjee Date: Thu, 10 Jun 2021 21:53:59 +0530 Subject: [PATCH] Update all english sources --- src/Components/NovelList.js | 2 ++ src/Screens/Extension/Extension.js | 8 ++++- src/redux/novel/novel.actions.js | 2 -- src/sources/en/foxaholic.js | 47 ++++++++++++++----------- src/sources/en/freewebnovel.js | 22 ++++++------ src/sources/en/jpmtl.js | 8 +++-- src/sources/en/kisslightnovels.js | 19 +++++++--- src/sources/en/lightnovelpub.js | 56 +++++++++++++++++++++++++----- src/sources/en/novelupdatescc.js | 4 +-- src/sources/en/readlightnovelcc.js | 4 +-- src/sources/en/tapread.js | 9 ++--- src/sources/en/vipnovel.js | 12 ++++--- src/sources/en/woopread.js | 23 +++++++----- src/sources/en/wuxiaworldcloud.js | 2 +- src/sources/en/wuxiaworldco.js | 2 +- src/sources/en/wuxiaworldsite.js | 13 +++++-- 16 files changed, 160 insertions(+), 73 deletions(-) diff --git a/src/Components/NovelList.js b/src/Components/NovelList.js index b7dd937a..232e6f9e 100644 --- a/src/Components/NovelList.js +++ b/src/Components/NovelList.js @@ -7,6 +7,7 @@ import { getDeviceOrientation } from "../Services/utils/helpers"; const NovelList = ({ data, onScroll, + onEndReached, renderItem, refreshControl, ListEmptyComponent, @@ -40,6 +41,7 @@ const NovelList = ({ ListEmptyComponent={ListEmptyComponent} ListFooterComponent={ListFooterComponent} onScroll={onScroll} + onEndReached={onEndReached} /> ); }; diff --git a/src/Screens/Extension/Extension.js b/src/Screens/Extension/Extension.js index 759fbcd4..9a4d33ad 100644 --- a/src/Screens/Extension/Extension.js +++ b/src/Screens/Extension/Extension.js @@ -204,7 +204,13 @@ const Extension = ({ navigation, route }) => { data={novels} renderItem={renderItem} ListEmptyComponent={listEmptyComponent()} - onScroll={onScroll} + // onScroll={onScroll} + onEndReached={() => { + if (!searchText) { + getNovels(page + 1); + setPage((page) => page + 1); + } + }} ListFooterComponent={ !searchText && ( diff --git a/src/redux/novel/novel.actions.js b/src/redux/novel/novel.actions.js index c90cb398..267cbe38 100644 --- a/src/redux/novel/novel.actions.js +++ b/src/redux/novel/novel.actions.js @@ -57,7 +57,6 @@ export const setNovel = (novel) => async (dispatch) => { export const getNovelAction = (followed, sourceId, novelUrl, novelId, sort, filter) => async (dispatch) => { - console.log(followed, sourceId, novelUrl, novelId, sort, filter); try { dispatch({ type: LOADING_NOVEL }); @@ -102,7 +101,6 @@ export const getNovelAction = * Insert novel in db. */ const fetchedNovelId = await insertNovel(fetchedNovel); - console.log(fetchedNovelId); await insertChapters(fetchedNovelId, fetchedNovel.chapters); /** diff --git a/src/sources/en/foxaholic.js b/src/sources/en/foxaholic.js index 399d5b2b..b925c85b 100644 --- a/src/sources/en/foxaholic.js +++ b/src/sources/en/foxaholic.js @@ -5,7 +5,7 @@ import { htmlToText } from "../helpers/htmlToText"; const baseUrl = "https://foxaholic.com/"; const popularNovels = async (page) => { - let url = baseUrl + "novel/?m_orderby=rating"; + let url = baseUrl + "novel/page/" + page + "/?m_orderby=rating"; const result = await fetch(url); const body = await result.text(); @@ -35,7 +35,9 @@ const popularNovels = async (page) => { }; const parseNovelAndChapters = async (novelUrl) => { - const url = `${baseUrl}novel/${novelUrl}/`; + const url = `${baseUrl}novel/${novelUrl}`; + + console.log(url); const result = await fetch(url); const body = await result.text(); @@ -71,36 +73,44 @@ const parseNovelAndChapters = async (novelUrl) => { .replace(/[\t\n]/g, "") .trim(); - novel[detailName] = detail; + switch (detailName) { + case "Genre": + novel.genre = detail.trim().replace(/[\t\n]/g, ","); + break; + case "Author": + novel.author = detail.trim(); + break; + case "Artist": + novel.status = detail.trim(); + break; + } }); - novel.genre = novel.Genre.replace(/, /g, ","); - novel.author = novel.Author; - novel.Status = null; - novel.artist = null; - - delete novel.Genre; - delete novel.Author; - $(".description-summary > div.summary__content").find("em").remove(); novel.summary = $(".description-summary > div.summary__content") .text() - .replace(/[\t\n]/g, ""); + .trim(); let novelChapters = []; const novelId = $("#manga-chapters-holder").attr("data-id"); + console.log(novelId); + let formData = new FormData(); formData.append("action", "manga_get_chapters"); formData.append("manga", novelId); - const data = await fetch("https://foxaholic.com/wp-admin/admin-ajax.php", { - method: "POST", - body: formData, - }); + const data = await fetch( + "https://www.foxaholic.com/wp-admin/admin-ajax.php", + { + method: "POST", + body: formData, + } + ); const text = await data.text(); + console.log(text); $ = cheerio.load(text); @@ -128,7 +138,7 @@ const parseNovelAndChapters = async (novelUrl) => { }; const parseChapter = async (novelUrl, chapterUrl) => { - const url = `${baseUrl}novel/${novelUrl}/${chapterUrl}/`; + const url = `${baseUrl}novel/${novelUrl}${chapterUrl}/`; const result = await fetch(url); const body = await result.text(); @@ -157,9 +167,6 @@ const parseChapter = async (novelUrl, chapterUrl) => { : (prevChapter = prevChapter[5]); } - novelUrl += "/"; - chapterUrl = optionalUrl + chapterUrl; - const chapter = { sourceId: 22, novelUrl, diff --git a/src/sources/en/freewebnovel.js b/src/sources/en/freewebnovel.js index 010efdb7..7109002a 100644 --- a/src/sources/en/freewebnovel.js +++ b/src/sources/en/freewebnovel.js @@ -4,7 +4,7 @@ import { htmlToText } from "../helpers/htmlToText"; const baseUrl = "https://freewebnovel.com/"; const popularNovels = async (page) => { - let url = baseUrl + "most-popular-novel/"; + let url = baseUrl + "completed-novel/" + page; const result = await fetch(url); const body = await result.text(); @@ -23,8 +23,6 @@ const popularNovels = async (page) => { .replace(".html", "") .slice(1); - novelUrl += "/"; - const novel = { sourceId: 13, novelName, @@ -41,6 +39,8 @@ const popularNovels = async (page) => { const parseNovelAndChapters = async (novelUrl) => { const url = `${baseUrl}${novelUrl}.html`; + console.log(url); + const result = await fetch(url); const body = await result.text(); @@ -54,7 +54,7 @@ const parseNovelAndChapters = async (novelUrl) => { novel.url = url; - novel.novelUrl = `${novelUrl.replace(".html", "")}/`; + novel.novelUrl = novelUrl; novel.novelName = $("h1.tit").text(); @@ -72,18 +72,18 @@ const parseNovelAndChapters = async (novelUrl) => { novel.artist = null; - novel.Status = $("[title=Status]") + novel.status = $("[title=Status]") .next() .text() .replace(/[\t\n]/g, ""); - novel.Alternative = $("[title='Alternative names']") - .next() - .text() - .replace(/[\t\n]/g, ""); + // novel.Alternative = $("[title='Alternative names']") + // .next() + // .text() + // .replace(/[\t\n]/g, ""); - let novelSummary = $(".inner").html(); - novel.summary = htmlToText(novelSummary); + let novelSummary = $(".inner").text(); + novel.summary = novelSummary; let novelChapters = []; diff --git a/src/sources/en/jpmtl.js b/src/sources/en/jpmtl.js index 41ebbc53..2de38d11 100644 --- a/src/sources/en/jpmtl.js +++ b/src/sources/en/jpmtl.js @@ -6,7 +6,9 @@ const baseUrl = "https://jpmtl.com/"; const popularNovels = async (page) => { let url = - "https://jpmtl.com/v2/book/show/browse?query=&categories=&content_type=0&direction=0&page=2&limit=20&type=5&status=all&language=3&exclude_categories="; + "https://jpmtl.com/v2/book/show/browse?query=&categories=&content_type=0&direction=0&page=" + + page + + "&limit=20&type=5&status=all&language=3&exclude_categories="; const result = await fetch(url); const body = await result.json(); @@ -106,7 +108,9 @@ const parseNovelAndChapters = async (novelUrl) => { }; const parseChapter = async (novelUrl, chapterUrl) => { - const url = `${baseUrl}books/${novelUrl}/${chapterUrl}`; + const url = `${baseUrl}books/${novelUrl}${chapterUrl}`; + + console.log(url); const result = await fetch(url); const body = await result.text(); diff --git a/src/sources/en/kisslightnovels.js b/src/sources/en/kisslightnovels.js index 2ffd5dca..76b675ca 100644 --- a/src/sources/en/kisslightnovels.js +++ b/src/sources/en/kisslightnovels.js @@ -1,11 +1,12 @@ import cheerio from "react-native-cheerio"; - import { htmlToText } from "../helpers/htmlToText"; const baseUrl = "https://kisslightnovels.info/"; const popularNovels = async (page) => { - const result = await fetch(baseUrl); + const url = `${baseUrl}light-novel/page/${page}`; + + const result = await fetch(url); const body = await result.text(); $ = cheerio.load(body); @@ -33,7 +34,7 @@ const popularNovels = async (page) => { }; const parseNovelAndChapters = async (novelUrl) => { - const url = `${baseUrl}novel/${novelUrl}/`; + const url = `${baseUrl}novel/${novelUrl}`; const result = await fetch(url); const body = await result.text(); @@ -69,7 +70,17 @@ const parseNovelAndChapters = async (novelUrl) => { .replace(/[\t\n]/g, "") .trim(); - novel[detailName] = detail; + switch (detailName) { + case "Genre(s)": + novel.genre = detail.trim().replace(/[\t\n]/g, ","); + break; + case "Author(s)": + novel.author = detail.trim(); + break; + case "Artist(s)": + novel.status = detail.trim(); + break; + } }); $(".description-summary > div.summary__content").find("em").remove(); diff --git a/src/sources/en/lightnovelpub.js b/src/sources/en/lightnovelpub.js index 1c1d7753..298b6db5 100644 --- a/src/sources/en/lightnovelpub.js +++ b/src/sources/en/lightnovelpub.js @@ -5,9 +5,19 @@ import { htmlToText } from "../helpers/htmlToText"; const baseUrl = "https://www.lightnovelpub.com/"; const popularNovels = async (page) => { - let url = baseUrl + "browse/all/popular/all/1"; + let url = baseUrl + "browse/all/popular/all/" + page; - const result = await fetch(url); + let headers = new Headers({ + Accept: "application/json", + "Content-Type": "application/json", + "User-Agent": + "'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36", + }); + + const result = await fetch(url, { + method: "GET", + headers: headers, + }); const body = await result.text(); $ = cheerio.load(body); @@ -46,7 +56,17 @@ const popularNovels = async (page) => { const parseNovelAndChapters = async (novelUrl) => { const url = `${baseUrl}novel/${novelUrl}/`; - const result = await fetch(url); + let headers = new Headers({ + Accept: "application/json", + "Content-Type": "application/json", + "User-Agent": + "'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36", + }); + + const result = await fetch(url, { + method: "GET", + headers: headers, + }); const body = await result.text(); $ = cheerio.load(body); @@ -88,7 +108,7 @@ const parseNovelAndChapters = async (novelUrl) => { novel.author = $(".author > a > span").text(); novelSummary = $(".summary > .content").html(); - novel.summary = htmlToText(novelSummary); + novel.summary = htmlToText(novelSummary).trim(); let novelChapters = []; @@ -117,9 +137,18 @@ const parseNovelAndChapters = async (novelUrl) => { const parseChapter = async (novelUrl, chapterUrl) => { const url = `${baseUrl}novel/${novelUrl}/${chapterUrl}`; - const result = await fetch(url); - const body = await result.text(); + let headers = new Headers({ + Accept: "application/json", + "Content-Type": "application/json", + "User-Agent": + "'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36", + }); + const result = await fetch(url, { + method: "GET", + headers: headers, + }); + const body = await result.text(); $ = cheerio.load(body); const chapterName = $("h2").text(); @@ -158,9 +187,20 @@ const parseChapter = async (novelUrl, chapterUrl) => { const searchNovels = async (searchTerm) => { const url = `${baseUrl}lnwsearchlive?inputContent=${searchTerm}`; - const result = await scraper(url); + let headers = new Headers({ + Accept: "application/json", + "Content-Type": "application/json", + "User-Agent": + "'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36", + }); - $ = cheerio.load(result); + const result = await fetch(url, { + method: "GET", + headers: headers, + }); + const body = await result.text(); + + $ = cheerio.load(body); let novels = []; diff --git a/src/sources/en/novelupdatescc.js b/src/sources/en/novelupdatescc.js index 3b46aadc..8eba7d63 100644 --- a/src/sources/en/novelupdatescc.js +++ b/src/sources/en/novelupdatescc.js @@ -59,8 +59,8 @@ const parseNovelAndChapters = async (novelUrl) => { novel.author = $("div.author > span.name").text(); - novelSummary = $("div.content > p.desc").html(); - novel.summary = htmlToText(novelSummary); + novelSummary = $("div.content > p.desc").text(); + novel.summary = novelSummary.trim(); let novelChapters = []; diff --git a/src/sources/en/readlightnovelcc.js b/src/sources/en/readlightnovelcc.js index 95c4dc54..8a1f4185 100644 --- a/src/sources/en/readlightnovelcc.js +++ b/src/sources/en/readlightnovelcc.js @@ -59,8 +59,8 @@ const parseNovelAndChapters = async (novelUrl) => { novel.author = $("div.author > span.name").text(); - novelSummary = $("div.content > p.desc").html(); - novel.summary = htmlToText(novelSummary); + novelSummary = $("div.content > p.desc").text(); + novel.summary = novelSummary.trim(); let novelChapters = []; diff --git a/src/sources/en/tapread.js b/src/sources/en/tapread.js index 98388e7c..006998de 100644 --- a/src/sources/en/tapread.js +++ b/src/sources/en/tapread.js @@ -15,7 +15,6 @@ const popularNovels = async (page) => { const novelName = $(this).find(".book-name").text(); const novelCover = "http:" + $(this).find("img").attr("src"); let novelUrl = $(this).attr("data-id"); - novelUrl += "/"; const novel = { sourceId: 17, @@ -33,6 +32,8 @@ const popularNovels = async (page) => { const parseNovelAndChapters = async (novelUrl) => { const url = `${baseUrl}/book/detail/${novelUrl}`; + console.log(url); + const result = await fetch(url); const body = await result.text(); @@ -58,7 +59,7 @@ const parseNovelAndChapters = async (novelUrl) => { novel.author = $("div.author > span.name").text(); - novelSummary = $("div.content > p.desc").text(); + novel.summary = $("div.content > p.desc").text(); const getChapters = async (novelId) => { const chapterListUrl = "http://www.tapread.com/ajax/book/contents"; @@ -153,10 +154,10 @@ const searchNovels = async (searchTerm) => { body.result.storyList.map((novel) => { const novelName = novel.storyName.replace( - 'Demon', + /|<\/font>/g, "" ); - const novelUrl = novel.storyId + "/"; + const novelUrl = novel.storyId; const novelCover = "http://static.tapread.com" + novel.coverUrl; novels.push({ sourceId: 17, novelName, novelCover, novelUrl }); diff --git a/src/sources/en/vipnovel.js b/src/sources/en/vipnovel.js index 03800f5b..23a5009f 100644 --- a/src/sources/en/vipnovel.js +++ b/src/sources/en/vipnovel.js @@ -5,7 +5,7 @@ import { htmlToText } from "../helpers/htmlToText"; const baseUrl = "https://vipnovel.com/"; const popularNovels = async (page) => { - let url = baseUrl; + let url = `${baseUrl}vipnovel/page/${page}/?m_orderby=rating`; const result = await fetch(url); const body = await result.text(); @@ -31,6 +31,8 @@ const popularNovels = async (page) => { novels.push(novel); }); + console.log(novels[0]); + return novels; }; @@ -85,9 +87,9 @@ const parseNovelAndChapters = async (novelUrl) => { // $(".description-summary > div.summary__content").find("em").remove(); - let novelSummary = $(".description-summary > div.summary__content").html(); + let novelSummary = $(".description-summary > div.summary__content").text(); - novel.summary = htmlToText(novelSummary); + novel.summary = novelSummary.replace(/[\t\n]/g, ""); let novelChapters = []; @@ -117,7 +119,9 @@ const parseNovelAndChapters = async (novelUrl) => { }; const parseChapter = async (novelUrl, chapterUrl) => { - const url = `${baseUrl}vipnovel/${novelUrl}/${chapterUrl}`; + const url = chapterUrl; + + console.log(url); const result = await fetch(url); const body = await result.text(); diff --git a/src/sources/en/woopread.js b/src/sources/en/woopread.js index 56bb848e..77fc2167 100644 --- a/src/sources/en/woopread.js +++ b/src/sources/en/woopread.js @@ -1,11 +1,10 @@ import cheerio from "react-native-cheerio"; - import { htmlToText } from "../helpers/htmlToText"; const baseUrl = "https://woopread.com/"; const popularNovels = async (page) => { - let url = baseUrl + "novellist/?m_orderby=rating"; + let url = baseUrl + "novellist/page/" + page + "/?m_orderby=rating"; const result = await fetch(url); const body = await result.text(); @@ -35,7 +34,7 @@ const popularNovels = async (page) => { }; const parseNovelAndChapters = async (novelUrl) => { - const url = `${baseUrl}series/${novelUrl}/`; + const url = `${baseUrl}series/${novelUrl}`; const result = await fetch(url); const body = await result.text(); @@ -71,14 +70,23 @@ const parseNovelAndChapters = async (novelUrl) => { .replace(/[\t\n]/g, "") .trim(); - novel[detailName] = detail; + switch (detailName) { + case "Genre(s)": + novel.genre = detail.trim().replace(/[\t\n]/g, ","); + break; + case "Author(s)": + novel.author = detail.trim(); + break; + case "Artist(s)": + novel.status = detail.trim(); + break; + } }); - $(".description-summary > div.summary__content").find("em").remove(); novel.summary = $(".description-summary > div.summary__content") .text() - .replace(/[\t\n]/g, ""); + .trim(); let novelChapters = []; @@ -146,9 +154,6 @@ const parseChapter = async (novelUrl, chapterUrl) => { .replace(baseUrl + "series/" + novelUrl + "/", ""); } - novelUrl += "/"; - chapterUrl = optionalUrl + chapterUrl + "/"; - const chapter = { sourceId: 21, novelUrl, diff --git a/src/sources/en/wuxiaworldcloud.js b/src/sources/en/wuxiaworldcloud.js index 9e5e895a..0026705c 100644 --- a/src/sources/en/wuxiaworldcloud.js +++ b/src/sources/en/wuxiaworldcloud.js @@ -5,7 +5,7 @@ import { htmlToText } from "../helpers/htmlToText"; const baseUrl = "http://wuxiaworld.cloud/"; const popularNovels = async (page) => { - const url = `${baseUrl}/popular-novel`; + const url = `${baseUrl}/popular-novel?page=${page}`; const result = await fetch(url); const body = await result.text(); diff --git a/src/sources/en/wuxiaworldco.js b/src/sources/en/wuxiaworldco.js index 727886db..954cf454 100644 --- a/src/sources/en/wuxiaworldco.js +++ b/src/sources/en/wuxiaworldco.js @@ -60,7 +60,7 @@ const parseNovelAndChapters = async (novelUrl) => { novel.author = $("div.author > span.name").text(); novelSummary = $("div.content > p.desc").html(); - novel.summary = htmlToText(novelSummary); + novel.summary = novelSummary.trim(); let novelChapters = []; diff --git a/src/sources/en/wuxiaworldsite.js b/src/sources/en/wuxiaworldsite.js index 2f13e0e2..0ddb3712 100644 --- a/src/sources/en/wuxiaworldsite.js +++ b/src/sources/en/wuxiaworldsite.js @@ -4,9 +4,18 @@ const baseUrl = "https://wuxiaworldsite.co/"; const searchUrl = "https://wuxiaworldsite.co/search/"; const popularNovels = async (page) => { - const url = `${baseUrl}/power-ranking`; + const url = `https://wuxiaworldsite.co/ajax-story-power.ajax`; - const result = await fetch(url); + const formData = new FormData(); + formData.append("page", page); + formData.append("keyword", ""); + formData.append("count", 18); + formData.append("genres_include", ""); + formData.append("limit", 18); + formData.append("order_type", "DESC"); + formData.append("order_by", "views"); + + const result = await fetch(url, { method: "POST", body: formData }); const body = await result.text(); $ = cheerio.load(body);