's into double line breaks.
- .replace(/<\s*p[^>]*>/gi, "\n\n")
- // Remove content in script tags.
- .replace(/<\s*script[^>]*>[\s\S]*?<\/script>/gim, "")
- // Remove content in style tags.
- .replace(/<\s*style[^>]*>[\s\S]*?<\/style>/gim, "")
- // Remove content in comments.
- .replace(//gim, "")
- // Format anchor tags properly.
- .replace(
- /<\s*a[^>]*href=['"](.*?)['"][^>]*>([\s\S]*?)<\/\s*a\s*>/gi,
- "$2"
- )
- .replace(/<\s*img[^>]*src=['"](.*?)['"][^>]*>/gi, "[$1]")
- // Remove all remaining tags.
- .replace(/(<([^>]+)>)/gi, "")
- // Make sure there are never more than two
- // consecutive linebreaks.
- .replace(/\n{2,}/g, "\n\n")
- // Remove tabs.
- .replace(/\t/g, "")
- // Remove newlines at the beginning of the text.
- .replace(/^\n+/m, "")
- // Replace multiple spaces with a single space.
- .replace(/ {2,}/g, " ")
- // Decode HTML entities.
- .replace(/&([^;]+);/g, decodeHtmlEntity)
- );
+const htmlToText = (html, options = {}) => {
+ const { removeLineBreaks = true } = options;
+
+ text = removeLineBreaks && html.replace(/(?:\n|\r\n|\r)/gi, "");
+ text = html
+ // Turn
's into single line breaks.
+ .replace(/<\s*br[^>]*>/gi, "\n")
+ // Turn 's into line breaks.
+ .replace(/<\s*\/li[^>]*>/gi, "\n")
+ // Turn
's into double line breaks. + .replace(/<\s*p[^>]*>/gi, "\n\n") + // Remove content in script tags. + .replace(/<\s*script[^>]*>[\s\S]*?<\/script>/gim, "") + // Remove content in style tags. + .replace(/<\s*style[^>]*>[\s\S]*?<\/style>/gim, "") + // Remove content in comments. + .replace(//gim, "") + // Format anchor tags properly. + .replace( + /<\s*a[^>]*href=['"](.*?)['"][^>]*>([\s\S]*?)<\/\s*a\s*>/gi, + "$2" + ) + .replace(/<\s*img[^>]*src=['"](.*?)['"][^>]*>/gi, "[$1]") + // Remove all remaining tags. + .replace(/(<([^>]+)>)/gi, "") + // Make sure there are never more than two + // consecutive linebreaks. + .replace(/\n{2,}/g, "\n\n") + // Remove tabs. + .replace(/\t/g, "") + // Remove newlines at the beginning of the text. + .replace(/^\n+/m, "") + // Replace multiple spaces with a single space. + .replace(/ {2,}/g, " ") + // Decode HTML entities. + .replace(/&([^;]+);/g, decodeHtmlEntity); + + return text; }; const decodeHtmlEntity = (m, n) => { diff --git a/src/sources/multisrc/MadaraGenerator.js b/src/sources/multisrc/MadaraGenerator.js new file mode 100644 index 00000000..7693f34e --- /dev/null +++ b/src/sources/multisrc/MadaraGenerator.js @@ -0,0 +1,72 @@ +import MadaraScraper from "./madara/MadaraScraper"; + +const getPath = (extensionId) => { + const path = { + 38: { novels: "manga", novel: "manga", chapter: "manga" }, + 39: { novels: "series", novel: "series", chapter: "series" }, + 40: { novels: "novels", novel: "novel", chapter: "novel" }, + 41: { novels: "manga", novel: "manga", chapter: "manga" }, + 42: { novels: "all-novels", novel: "manga", chapter: "manga" }, + 43: { novels: "series", novel: "series", chapter: "series" }, + 44: { novels: "series", novel: "series", chapter: "series" }, + 45: { novels: "novel", novel: "novel", chapter: "novel" }, + }; + + return path[extensionId]; +}; + +export const SkyNovelScraper = new MadaraScraper( + 38, + "https://skynovel.org/", + "SkyNovel", + getPath(38) +); + +export const NovelCakeScraper = new MadaraScraper( + 39, + "https://novelcake.com/", + "NovelCake", + getPath(39) +); + +export const NovelsRockScraper = new MadaraScraper( + 40, + "https://novelsrock.com/", + "NovelsRock", + getPath(40) +); + +export const ZinnNovelScraper = new MadaraScraper( + 41, + "https://zinnovel.com/", + "ZinnNovel", + getPath(41) +); + +export const NovelTranslateScraper = new MadaraScraper( + 42, + "https://noveltranslate.com/", + "NovelTranslate", + getPath(42) +); + +export const LunarLettersScraper = new MadaraScraper( + 43, + "https://www.lunarletters.com/", + "LunarLetters", + getPath(43) +); + +export const SleepyTranslationsScraper = new MadaraScraper( + 44, + "https://sleepytranslations.com/", + "SleepyTranslations", + getPath(44) +); + +export const FreeNovelScraper = new MadaraScraper( + 45, + "https://freenovel.me/", + "FreeNovelMe", + getPath(45) +); diff --git a/src/sources/multisrc/madara/MadaraScraper.js b/src/sources/multisrc/madara/MadaraScraper.js new file mode 100644 index 00000000..bdee0eb1 --- /dev/null +++ b/src/sources/multisrc/madara/MadaraScraper.js @@ -0,0 +1,220 @@ +import cheerio from "react-native-cheerio"; +import { htmlToText } from "../../helpers/htmlToText"; + +class MadaraScraper { + constructor(sourceId, baseUrl, sourceName, path) { + this.sourceId = sourceId; + this.baseUrl = baseUrl; + this.sourceName = sourceName; + this.path = path; + + // this.novelsScraper = this.novelsScraper.bind(this); + // this.novelScraper = this.novelScraper.bind(this); + // this.chapterScraper = this.chapterScraper.bind(this); + // this.searchScraper = this.searchScraper.bind(this); + } + + async popularNovels(page) { + const totalPages = 100; + let url = + this.baseUrl + + this.path.novels + + "/page/" + + page + + "/?m_orderby=rating"; + let sourceId = this.sourceId; + + const result = await fetch(url); + const body = await result.text(); + + const $ = cheerio.load(body); + + let novels = []; + + $(".manga-title-badges").remove(); + + $(".page-item-detail").each(function (result) { + const novelName = $(this).find(".post-title").text().trim(); + let image = $(this).find("img"); + const novelCover = image.attr("data-src") || image.attr("src"); + + let novelUrl = $(this) + .find(".post-title") + .find("a") + .attr("href") + .split("/")[4]; + + const novel = { + sourceId, + novelName, + novelCover, + novelUrl, + }; + + novels.push(novel); + }); + + return { totalPages, novels }; + } + + async parseNovelAndChapters(novelUrl) { + const url = `${this.baseUrl}${this.path.novel}/${novelUrl}/`; + + const result = await fetch(url); + const body = await result.text(); + + let $ = cheerio.load(body); + + let novel = {}; + + novel.sourceId = this.sourceId; + + novel.sourceName = this.sourceName; + + novel.url = url; + + novel.novelUrl = novelUrl; + + $(".manga-title-badges").remove(); + + novel.novelName = $(".post-title > h1").text().trim(); + + novel.novelCover = + $(".summary_image > a > img").attr("data-src") || + $(".summary_image > a > img").attr("src"); + + $(".post-content_item").each(function (result) { + const detailName = $(this) + .find(".summary-heading > h5") + .text() + .trim(); + const detail = $(this).find(".summary-content").text().trim(); + + switch (detailName) { + case "Genre(s)": + novel.genre = detail.replace(/[\t\n]/g, ","); + break; + case "Author(s)": + novel.author = detail; + break; + case "Status": + novel.status = detail; + break; + } + }); + + novel.summary = $("div.summary__content").text().trim(); + + let novelChapters = []; + + const novelId = + $(".rating-post-id").attr("value") || + $("#manga-chapters-holder").attr("data-id"); + + let formData = new FormData(); + formData.append("action", "manga_get_chapters"); + formData.append("manga", novelId); + + const data = await fetch(this.baseUrl + "wp-admin/admin-ajax.php", { + method: "POST", + body: formData, + }); + const text = await data.text(); + + $ = cheerio.load(text); + + $(".wp-manga-chapter").each(function (result) { + const chapterName = $(this) + .find("a") + .text() + .replace(/[\t\n]/g, "") + .trim(); + + const releaseDate = $(this) + .find("span") + .text() + .replace("Free", "") + .trim(); + + const chapterUrl = $(this).find("a").attr("href").split("/")[5]; + + novelChapters.push({ chapterName, releaseDate, chapterUrl }); + }); + + console.log(novel); + + novel.chapters = novelChapters.reverse(); + + return novel; + } + + async parseChapter(novelUrl, chapterUrl) { + let sourceId = this.sourceId; + + const url = `${this.baseUrl}${this.path.chapter}/${novelUrl}/${chapterUrl}`; + + const result = await fetch(url); + const body = await result.text(); + + const $ = cheerio.load(body); + + let chapterName = + $(".text-center").text() || $("#chapter-heading").text(); + + let chapterText = $(".text-left").html(); + chapterText = htmlToText(chapterText); + + let nextChapter = null; + let prevChapter = null; + + const chapter = { + sourceId, + novelUrl, + chapterUrl, + chapterName, + chapterText, + nextChapter, + prevChapter, + }; + + return chapter; + } + + async searchNovels(searchTerm) { + const url = `${this.baseUrl}?s=${searchTerm}&post_type=wp-manga`; + + const result = await fetch(url); + const body = await result.text(); + + const $ = cheerio.load(body); + + let novels = []; + let sourceId = this.sourceId; + + $(".c-tabs-item__content").each(function (result) { + const novelName = $(this).find(".post-title").text().trim(); + + let image = $(this).find("img"); + const novelCover = image.attr("data-src") || image.attr("src"); + + let novelUrl = $(this) + .find(".post-title") + .find("a") + .attr("href") + .split("/")[4]; + + const novel = { + sourceId, + novelName, + novelCover, + novelUrl, + }; + + novels.push(novel); + }); + + return novels; + } +} + +export default MadaraScraper; diff --git a/src/sources/sources.js b/src/sources/sources.js index f9834069..1c0efe91 100644 --- a/src/sources/sources.js +++ b/src/sources/sources.js @@ -34,6 +34,16 @@ import RoyalRoadScraper from "./en/royalroad"; import ScribbleHubScraper from "./en/scribblehub"; import SyosetuScraper from "./jp/syosetu"; import LNMTLScraper from "./en/lnmtl"; +import { + FreeNovelScraper, + LunarLettersScraper, + NovelCakeScraper, + NovelsRockScraper, + NovelTranslateScraper, + SkyNovelScraper, + SleepyTranslationsScraper, + ZinnNovelScraper, +} from "./multisrc/MadaraGenerator"; export const sources = [ /** @@ -230,6 +240,62 @@ export const sources = [ icon: "https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/lnmtl/icon.png?raw=true", lang: "English", }, + { + sourceId: 38, + sourceUrl: "https://skynovel.org/", + sourceName: "SkyNovel", + icon: "https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/multisrc/madara/icons/skynovel.png?raw=true", + lang: "English", + }, + { + sourceId: 39, + sourceUrl: "https://novelcake.com/", + sourceName: "NovelCake", + icon: "https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/multisrc/madara/icons/novelcake.png?raw=true", + lang: "English", + }, + { + sourceId: 40, + sourceUrl: "https://novelsrock.com/", + sourceName: "NovelsRock", + icon: "https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/multisrc/madara/icons/novelsrock.png?raw=true", + lang: "English", + }, + { + sourceId: 41, + sourceUrl: "https://zinnovel.com/", + sourceName: "ZinnNovel", + icon: "https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/multisrc/madara/icons/zinnovel.png?raw=true", + lang: "English", + }, + { + sourceId: 42, + sourceUrl: "https://noveltranslate.com/", + sourceName: "NovelTranslate", + icon: "https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/multisrc/madara/icons/noveltranslate.png?raw=true", + lang: "English", + }, + { + sourceId: 43, + sourceUrl: "https://www.lunarletters.com/", + sourceName: "LunarLetters", + icon: "https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/multisrc/madara/icons/lunarletters.png?raw=true", + lang: "English", + }, + { + sourceId: 44, + sourceUrl: "https://sleepytranslations.com/", + sourceName: "SleepyTranslations", + icon: "https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/multisrc/madara/icons/sleepytranslations.png?raw=true", + lang: "English", + }, + { + sourceId: 45, + sourceUrl: "https://freenovel.me/", + sourceName: "FreeNovelMe", + icon: "https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/multisrc/madara/icons/freenovel.png?raw=true", + lang: "English", + }, /** * Spanish @@ -414,5 +480,29 @@ export const getSource = (sourceId) => { case 37: { return LNMTLScraper; } + case 38: { + return SkyNovelScraper; + } + case 39: { + return NovelCakeScraper; + } + case 40: { + return NovelsRockScraper; + } + case 41: { + return ZinnNovelScraper; + } + case 42: { + return NovelTranslateScraper; + } + case 43: { + return LunarLettersScraper; + } + case 44: { + return SleepyTranslationsScraper; + } + case 45: { + return FreeNovelScraper; + } } };