From d8d231cb90653904a032a086876d50ef14873cdf Mon Sep 17 00:00:00 2001 From: Rajarshee Chatterjee Date: Thu, 22 Jul 2021 09:08:40 +0530 Subject: [PATCH] Add source WLNUpdates --- src/screens/Chapter/TextToComponent.js | 1 - .../Novel/components/Info/GenreChip.js | 1 + .../components/Info/NovelDetailsComponents.js | 1 + src/sources/en/royalroad.js | 16 +- src/sources/en/wlnupdates.js | 297 ++++++++++++++++++ src/sources/sources.js | 2 + src/sources/sources.json | 7 + 7 files changed, 319 insertions(+), 6 deletions(-) create mode 100644 src/sources/en/wlnupdates.js diff --git a/src/screens/Chapter/TextToComponent.js b/src/screens/Chapter/TextToComponent.js index d5609266..700a2eb5 100644 --- a/src/screens/Chapter/TextToComponent.js +++ b/src/screens/Chapter/TextToComponent.js @@ -241,7 +241,6 @@ const TextToComponents = ({ text={text} textStyle={textStyle} textSelectable={textSelectable} - key={index} /> ); } diff --git a/src/screens/Novel/components/Info/GenreChip.js b/src/screens/Novel/components/Info/GenreChip.js index 146c1d2f..8f67dc58 100644 --- a/src/screens/Novel/components/Info/GenreChip.js +++ b/src/screens/Novel/components/Info/GenreChip.js @@ -26,5 +26,6 @@ const styles = StyleSheet.create({ fontSize: 13, borderWidth: 1, borderRadius: 50, + textTransform: "capitalize", }, }); diff --git a/src/screens/Novel/components/Info/NovelDetailsComponents.js b/src/screens/Novel/components/Info/NovelDetailsComponents.js index d392a48e..21e190a7 100644 --- a/src/screens/Novel/components/Info/NovelDetailsComponents.js +++ b/src/screens/Novel/components/Info/NovelDetailsComponents.js @@ -229,5 +229,6 @@ const styles = StyleSheet.create({ fontSize: 13, borderWidth: 1, borderRadius: 50, + textTransform: "capitalize", }, }); diff --git a/src/sources/en/royalroad.js b/src/sources/en/royalroad.js index 91712316..1511408e 100644 --- a/src/sources/en/royalroad.js +++ b/src/sources/en/royalroad.js @@ -1,6 +1,7 @@ import cheerio from "react-native-cheerio"; import { htmlToText } from "../helpers/htmlToText"; import moment from "moment"; +import { parseMadaraDate } from "../helpers/parseDate"; const baseUrl = "https://www.royalroad.com/"; @@ -15,10 +16,14 @@ const popularNovels = async (page) => { let novels = []; - $("div.fiction-list-item.row").each(function (result) { + $("div.fiction-list-item.row").each(function () { const novelName = $(this).find("h2.fiction-title").text().trim(); - const novelCover = $(this).find("img").attr("src"); + let novelCover = $(this).find("img").attr("src"); + if (novelCover === "/Content/Images/nocover-new-min.png") { + novelCover = + "https://github.com/LNReader/lnreader-sources/blob/main/src/coverNotAvailable.jpg?raw=true"; + } let novelUrl = $(this) .find("h2.fiction-title > a") .attr("href") @@ -76,9 +81,10 @@ const parseNovelAndChapters = async (novelUrl) => { $("table#chapters > tbody") .find("tr") - .each(function (result) { + .each(function () { const chapterName = $(this).find("td").first().text().trim(); - const releaseDate = $(this).find("td").first().next().text().trim(); + let releaseDate = $(this).find("td").first().next().text().trim(); + releaseDate = releaseDate && parseMadaraDate(releaseDate); const chapterUrl = $(this) .find("td") @@ -135,7 +141,7 @@ const searchNovels = async (searchTerm) => { let novels = []; - $("div.fiction-list-item.row").each(function (result) { + $("div.fiction-list-item.row").each(function () { const novelName = $(this).find("h2.fiction-title").text().trim(); let novelCover = $(this).find("img").attr("src"); diff --git a/src/sources/en/wlnupdates.js b/src/sources/en/wlnupdates.js new file mode 100644 index 00000000..6a91eeaa --- /dev/null +++ b/src/sources/en/wlnupdates.js @@ -0,0 +1,297 @@ +import cheerio from "react-native-cheerio"; +import { Status } from "../helpers/constants"; +import { htmlToText } from "../helpers/htmlToText"; + +const sourceId = 62; + +const sourceName = "WLNUpdates"; + +const baseUrl = "https://www.wlnupdates.com/"; + +let headers = new Headers({ + "User-Agent": + "'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36", +}); + +const popularNovels = async (page) => { + const totalPages = 100; + let url = `${baseUrl}highest-rated/` + page; + + const result = await fetch(url, { + method: "GET", + headers: headers, + }); + const body = await result.text(); + + const $ = cheerio.load(body); + + const novels = []; + + $("table").find("tr").first().remove(); + + $("tr").each(function (res) { + const novelCover = + "https://github.com/LNReader/lnreader-sources/blob/main/src/coverNotAvailable.jpg?raw=true"; + + const novelName = $(this).find("td > a").text(); + const novelUrl = + "https://www.wlnupdates.com" + $(this).find("td > a").attr("href"); + + const novel = { + sourceId, + novelName, + novelCover, + novelUrl, + }; + + novels.push(novel); + }); + + return { totalPages, novels }; +}; + +const parseNovelAndChapters = async (novelUrl) => { + const url = novelUrl; + + const result = await fetch(url, { + method: "GET", + headers: headers, + }); + const body = await result.text(); + + $ = cheerio.load(body); + + let novel = { + sourceId, + sourceName, + url, + novelUrl, + }; + + novel.novelName = $("h2").text(); + + novel.novelCover = + "https://github.com/LNReader/lnreader-sources/blob/main/src/coverNotAvailable.jpg?raw=true"; + + novel.author = $(".multiitem#author") + .text() + .trim() + .replace(/[\t\n]/g, ""); + + novel.genre = $(".multiitem#genre") + .text() + .trim() + .replace(/[\t\n]/g, ""); + + novel.status = Status.UNKNOWN; + + let summary = $("#description").text().trim(); + + novel.summary = summary; + + let novelChapters = []; + + $("#release-entry").each(function () { + let chapterName; + + if ($(this).find("td.numeric").length > 1) { + chapterName = + "Volume " + + $(this).find(" td:nth-child(3)").text().trim() + + " Chapter " + + $(this).find(" td:nth-child(4)").text().trim(); + } else if ($(this).find("td.numeric").length > 0) { + chapterName = "Chapter " + $(this).find("td.numeric").text().trim(); + } else { + chapterName = $(this).find("td.postfix").text().trim(); + } + + const releaseDate = $(this).find(".release-entry-cell").text().trim(); + + const chapterUrl = $(this).find("td:nth-child(1) > a").attr("href"); + + novelChapters.push({ chapterName, releaseDate, chapterUrl }); + }); + + novel.chapters = novelChapters.reverse(); + + return novel; +}; + +const parseChapter = async (novelUrl, chapterUrl) => { + const url = chapterUrl; + let result, body; + + let chapterName = ""; + + try { + result = await fetch(url, { + method: "GET", + headers: headers, + }); + body = await result.text(); + + $ = cheerio.load(body); + + let isWuxiaWorld = result.url.toLowerCase().includes("wuxiaworld"); + + let isBlogspot = result.url.toLowerCase().includes("blogspot"); + + let isTumblr = result.url.toLowerCase().includes("tumblr"); + + /** + * Checks if its a wwordpress site + */ + let isWordPress = + $('meta[name="generator"]').attr("content") || $("footer").text(); + + if (isWordPress) { + isWordPress = + isWordPress.toLowerCase().includes("wordpress") || + isWordPress.includes("Site Kit by Google") || + $(".powered-by").text().toLowerCase().includes("wordpress") || + $(".entry-content").html() || + $("section.content").html() + ? true + : false; + } + + let isRainOfSnow = result.url.toLowerCase().includes("rainofsnow"); + let isRoyalRoad = result.url.toLowerCase().includes("royalroad"); + + let isWebNovel = result.url.toLowerCase().includes("webnovel"); + + let isHostedNovel = result.url.toLowerCase().includes("hostednovel"); + + let isScribbleHub = result.url.toLowerCase().includes("scribblehub"); + + if (isWuxiaWorld) { + chapterText = $("#chapter-content").html(); + } else if (isRainOfSnow) { + chapterText = $("div.content").html(); + } else if (isTumblr) { + chapterText = $(".post").html(); + } else if (isBlogspot) { + chapterText = $(".entry-content").html(); + } else if (isHostedNovel) { + chapterText = $(".chapter").html(); + } else if (isScribbleHub) { + chapterText = $("div.chp_raw").html(); + } else if (isRoyalRoad) { + chapterText = $("div.chapter-content").html(); + } else if (isWordPress) { + /** + * Remove wordpress bloat tags + */ + + const bloatClasses = [ + ".c-ads", + "#madara-comments", + "#comments", + ".sharedaddy", + ".wp-dark-mode-switcher", + ".wp-next-post-navi", + ".wp-block-buttons", + ".wp-block-columns", + ".post-cats", + ".sidebar", + ".author-avatar", + ]; + + bloatClasses.map((tag) => $(tag).remove()); + + chapterText = + $(".entry-content").html() || + $(".single_post").html() || + $(".post-entry").html() || + $("article.post").html() || + $(".content").html() || + $("#content").html(); + } else if (isWebNovel) { + chapterText = $(".cha-words").html(); + + if (!chapterText) { + chapterText = $("._content").html(); + } + } else { + /** + * Remove unnecessary tags + */ + const tags = ["nav", "header", "footer", ".hidden"]; + + tags.map((tag) => $(tag).remove()); + + chapterText = $("body").html(); + } + + if (chapterText) { + /** + * Remove patreon/discord links + */ + chapterText = chapterText.replace( + /<\s*a[^>]*href=['"](https:\/\/(?:www.|patreon|discord).*?)['"][^>]*>([\s\S]*?)<\/\s*a\s*>/gi, + "" + ); + + chapterText = htmlToText(chapterText); + } else { + if (!chapterText) { + chapterText = + "Chapter not available.\n\nReport if it's available in webview."; + } + } + } catch (error) { + chapterText = `Chapter not available (Error: ${error.message}).\n\nReport if it's available in webview.`; + } + + const chapter = { + sourceId, + novelUrl, + chapterUrl, + chapterName, + chapterText, + }; + + return chapter; +}; + +const searchNovels = async (searchTerm) => { + const url = baseUrl + "search?title=" + searchTerm; + + const res = await fetch(url, { + method: "GET", + headers: headers, + }); + const body = await res.text(); + + $ = cheerio.load(body); + + let novels = []; + + $("tr").each(function (res) { + const novelCover = + "https://github.com/LNReader/lnreader-sources/blob/main/src/coverNotAvailable.jpg?raw=true"; + const novelName = $(this).find("td > a").text(); + const novelUrl = + "https://www.wlnupdates.com" + $(this).find("td > a").attr("href"); + + const novel = { + sourceId, + novelName, + novelCover, + novelUrl, + }; + + novels.push(novel); + }); + return novels; +}; + +const WLNUpdatesScraper = { + popularNovels, + parseNovelAndChapters, + parseChapter, + searchNovels, +}; + +export default WLNUpdatesScraper; diff --git a/src/sources/sources.js b/src/sources/sources.js index f5f2fb56..4eccae6e 100644 --- a/src/sources/sources.js +++ b/src/sources/sources.js @@ -62,6 +62,7 @@ import { KolNovelScraper, RewayatArScraper, } from "./multisrc/wpmangastream/WPMangaStreamGenerator"; +import WLNUpdatesScraper from "./en/wlnupdates"; export const getSource = (sourceId) => { const scrapers = { @@ -125,6 +126,7 @@ export const getSource = (sourceId) => { 59: ArNovelScraper, 60: MeionNovelScraper, 61: WebNovelLoverScraper, + 62: WLNUpdatesScraper, }; return scrapers[sourceId]; diff --git a/src/sources/sources.json b/src/sources/sources.json index 76e016e3..b3d6f01d 100644 --- a/src/sources/sources.json +++ b/src/sources/sources.json @@ -418,5 +418,12 @@ "sourceName": "WebNovelLover", "icon": "https://github.com/LNReader/lnreader-sources/blob/main/multisrc/madara/icons/webnovellovers.png?raw=true", "lang": "English" + }, + { + "sourceId": 62, + "url": "https://www.wlnupdates.com/", + "sourceName": "WLNUpdates", + "icon": "https://github.com/LNReader/lnreader-sources/blob/main/src/en/wlnupdates/icon.png?raw=true", + "lang": "English" } ]