From e274feda6e8fc66c12460bf993a2a569aaeb6ded Mon Sep 17 00:00:00 2001 From: Rajarshee Chatterjee Date: Tue, 13 Jul 2021 17:07:55 +0530 Subject: [PATCH] Minor fixes --- src/sources/ch/yushubo.js | 6 +- src/sources/en/novelupdates.js | 182 +++++++++++++++++------------- src/sources/en/tapread.js | 28 +++-- src/sources/helpers/htmlToText.js | 1 + 4 files changed, 128 insertions(+), 89 deletions(-) diff --git a/src/sources/ch/yushubo.js b/src/sources/ch/yushubo.js index c7cf056a..8a2be7ba 100644 --- a/src/sources/ch/yushubo.js +++ b/src/sources/ch/yushubo.js @@ -30,7 +30,7 @@ const popularNovels = async (page) => { : `${baseUrl}${novelCover}`; const novel = { - extensionId: 52, + sourceId: 52, novelUrl, novelName, novelCover, @@ -126,7 +126,7 @@ const parseChapter = async (novelUrl, chapterUrl) => { const prevChapter = null; const chapter = { - extensionId: 52, + sourceId: 52, novelUrl, chapterUrl, chapterName, @@ -169,7 +169,7 @@ const searchNovels = async (searchTerm) => { : `${baseUrl}${novelCover}`; const novel = { - extensionId: 52, + sourceId: 52, novelUrl, novelName, novelCover, diff --git a/src/sources/en/novelupdates.js b/src/sources/en/novelupdates.js index 962f85cc..c629f16b 100644 --- a/src/sources/en/novelupdates.js +++ b/src/sources/en/novelupdates.js @@ -127,93 +127,119 @@ const parseChapter = async (novelUrl, chapterUrl) => { let result, body; + let chapterName = ""; + try { result = await fetch(url, { method: "GET", headers: headers, }); body = await result.text(); + + // console.log("Redirected URL: ", result.url); + + $ = cheerio.load(body); + + let isWuxiaWorld = result.url.toLowerCase().includes("wuxiaworld"); + + let isBlogspot = result.url.toLowerCase().includes("blogspot"); + + let isTumblr = result.url.toLowerCase().includes("tumblr"); + + /** + * Checks if its a wwordpress site + */ + let isWordPress = + $('meta[name="generator"]').attr("content") || $("footer").text(); + + if (isWordPress) { + isWordPress = + isWordPress.toLowerCase().includes("wordpress") || + isWordPress + .toLowerCase() + .includes("Site Kit by Google 1.36.0") || + $(".powered-by").text().toLowerCase().includes("wordpress") || + $(".entry-content").html() || + $("section.content").html() + ? true + : false; + + // console.log(isWordPress); + } + + let isRainOfSnow = result.url.toLowerCase().includes("rainofsnow"); + + let isWebNovel = result.url.toLowerCase().includes("webnovel"); + + let isHostedNovel = result.url.toLowerCase().includes("hostednovel"); + + let isScribbleHub = result.url.toLowerCase().includes("scribblehub"); + + if (isWuxiaWorld) { + chapterText = $("#chapter-content").html(); + } else if (isRainOfSnow) { + chapterText = $("div.content").html(); + } else if (isTumblr) { + chapterText = $(".post").html(); + } else if (isBlogspot) { + chapterText = $(".entry-content").html(); + } else if (isHostedNovel) { + chapterText = $(".chapter").html(); + } else if (isScribbleHub) { + chapterText = $("div.chp_raw").html(); + } else if (isWordPress) { + /** + * Remove wordpress bloat tags + */ + $(".c-ads").remove(); + $("#madara-comments").remove(); + $(".sharedaddy").remove(); + + chapterText = $(".entry-content").html(); + + if (!chapterText) { + chapterText = $(".single_post").html(); + } + if (!chapterText) { + chapterText = $("article.post").html(); + } + + if (!chapterText) { + chapterText = $(".content").html() || $("#content").html(); + } + + /** + * Format anchor tags properly Eg. [title](https://www.example.com) + */ + } else if (isWebNovel) { + chapterText = $(".cha-words").html(); + + if (!chapterText) { + chapterText = $("._content").html(); + } + } else { + /** + * Remove unnecessary tags + */ + const tags = ["nav", "header", "footer", ".hidden"]; + + tags.map((tag) => $(tag).remove()); + + chapterText = $("body").html(); + } + + if (chapterText) { + chapterText = htmlToText(chapterText); + } else { + if (!chapterText) { + chapterText = + "Chapter not available.\n\nReport if it's available in webview."; + } + } } catch (error) { - chapterText = - "Chapter not available.\n\nReport if it's available in webview."; + chapterText = `Chapter not available (Error: ${error.message}).\n\nReport if it's available in webview.`; } - // console.log("Redirected URL: ", result.url); - - $ = cheerio.load(body); - - let chapterName = ""; - - let isWuxiaWorld = result.url.toLowerCase().includes("wuxiaworld"); - - /** - * Checks if its a wwordpress site - */ - let isWordPress = - $('meta[name="generator"]').attr("content") || $("footer").text(); - - if (isWordPress) { - isWordPress = - isWordPress.toLowerCase().includes("wordpress") || - isWordPress.toLowerCase().includes("Site Kit by Google 1.36.0") || - $(".powered-by").text().toLowerCase().includes("wordpress") || - $(".entry-content").html() || - $("section.content").html() - ? true - : false; - - // console.log(isWordPress); - } - - let isWebNovel = result.url.toLowerCase().includes("webnovel"); - - if (isWuxiaWorld) { - chapterText = $("#chapter-content").html(); - } else if (isWordPress) { - /** - * Remove wordpress bloat tags - */ - $(".c-ads").remove(); - $("#madara-comments").remove(); - $(".sharedaddy").remove(); - - chapterText = $(".entry-content").html(); - - if (!chapterText) { - chapterText = $("section.content").html(); - } - - if (!chapterText) { - chapterText = $(".single_post").html(); - } - - /** - * Format anchor tags properly Eg. [title](https://www.example.com) - */ - // .replace( - // /<\s*a[^>]*href=['"](.*?)['"][^>]*>([\s\S]*?)<\/\s*a\s*>/gi, - // "[$2]($1)" - // ); - } else if (isWebNovel) { - chapterText = $(".cha-words").html(); - } else { - /** - * Remove unnecessary tags - */ - const tags = ["nav", "header", "footer", ".hidden"]; - - tags.map((tag) => $(tag).remove()); - - chapterText = $("body").html(); - } - - if (!chapterText) { - chapterText = - "Chapter not available.\n\nReport if it's available in webview."; - } - - chapterText = htmlToText(chapterText); - let nextChapter = null; let prevChapter = null; diff --git a/src/sources/en/tapread.js b/src/sources/en/tapread.js index 57e22506..fed350a9 100644 --- a/src/sources/en/tapread.js +++ b/src/sources/en/tapread.js @@ -3,19 +3,31 @@ import { htmlToText } from "../helpers/htmlToText"; const baseUrl = "http://www.tapread.com"; +let headers = new Headers({ + "User-Agent": + "'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36", +}); + const popularNovels = async (page) => { - let totalPages = 1; - const result = await fetch(baseUrl); - const body = await result.text(); + let totalPages = 10; - $ = cheerio.load(body); + let url = baseUrl + "/ajax/category/detail"; + let formData = new FormData(); + formData.append("cateId", "0"); + formData.append("pageNo", page); + + const result = await fetch(url, { + method: "POST", + headers: headers, + }); + const data = await result.json(); let novels = []; - $('.section-item[data-classify="1"]').each(function (result) { - const novelName = $(this).find(".book-name").text(); - const novelCover = "http:" + $(this).find("img").attr("src"); - let novelUrl = $(this).attr("data-id"); + data.result.bookList.map((book) => { + const novelName = book.bookName; + const novelCover = "http://static.tapread.com" + book.coverUrl; + let novelUrl = book.bookId + "/"; const novel = { sourceId: 17, diff --git a/src/sources/helpers/htmlToText.js b/src/sources/helpers/htmlToText.js index 8d79fc76..a5241d8d 100644 --- a/src/sources/helpers/htmlToText.js +++ b/src/sources/helpers/htmlToText.js @@ -12,6 +12,7 @@ const htmlToText = (html, options = {}) => { .replace(/<\s*p[^>]*>/gi, "\n\n") // Remove content in script tags. .replace(/<\s*script[^>]*>[\s\S]*?<\/script>/gim, "") + .replace(/<\s*noscript[^>]*>[\s\S]*?<\/noscript>/gim, "") // Remove content in style tags. .replace(/<\s*style[^>]*>[\s\S]*?<\/style>/gim, "") // Remove content in comments.