diff --git a/src/en/boxnovel/BoxNovelScraper.js b/src/en/boxnovel/BoxNovelScraper.js index 4433541..0ca5b11 100644 --- a/src/en/boxnovel/BoxNovelScraper.js +++ b/src/en/boxnovel/BoxNovelScraper.js @@ -1,6 +1,6 @@ const cheerio = require("cheerio"); const fetch = require("node-fetch"); -const { htmlToText } = require("html-to-text"); +const { parseHtml } = require("../../helper"); const baseUrl = "https://boxnovel.com/novel"; const searchUrl = "https://boxnovel.com/"; @@ -127,7 +127,7 @@ const chapterScraper = async (req, res) => { const chapterName = $("div.text-left > h3").text(); let chapterText = $(".reading-content").html(); - chapterText = htmlToText(chapterText); + chapterText = parseHtml(chapterText); let nextChapter = null; diff --git a/src/en/freewebnovel/FreeWebNovelScraper.js b/src/en/freewebnovel/FreeWebNovelScraper.js index 00040ac..6b6ca68 100644 --- a/src/en/freewebnovel/FreeWebNovelScraper.js +++ b/src/en/freewebnovel/FreeWebNovelScraper.js @@ -2,6 +2,7 @@ const cheerio = require("cheerio"); const fetch = require("node-fetch"); const request = require("request"); const { htmlToText } = require("html-to-text"); +const { parseHtml } = require("../../helper"); const baseUrl = "https://freewebnovel.com/"; @@ -133,7 +134,7 @@ const chapterScraper = async (req, res) => { let chapterName = $("h1.tit").text(); let chapterText = $("div.txt").html(); - chapterText = htmlToText(chapterText); + chapterText = parseHtml(chapterText); let nextChapter = null; diff --git a/src/en/jpmtl/JPMTLScraper.js b/src/en/jpmtl/JPMTLScraper.js index 105eeb5..a31ee7e 100644 --- a/src/en/jpmtl/JPMTLScraper.js +++ b/src/en/jpmtl/JPMTLScraper.js @@ -1,6 +1,6 @@ const cheerio = require("cheerio"); const fetch = require("node-fetch"); -const { htmlToText } = require("html-to-text"); +const { parseHtml } = require("../../helper"); const baseUrl = "https://jpmtl.com/"; @@ -121,7 +121,7 @@ const chapterScraper = async (req, res) => { const chapterName = $(".chapter-content__title").text(); let chapterText = $(".chapter-content__content").html(); - chapterText = htmlToText(chapterText); + chapterText = parseHtml(chapterText); let nextChapter = null; let prevChapter = null; diff --git a/src/en/kisslightnovels/KissLightNovelsScraper.js b/src/en/kisslightnovels/KissLightNovelsScraper.js index d59aa90..f2e581f 100644 --- a/src/en/kisslightnovels/KissLightNovelsScraper.js +++ b/src/en/kisslightnovels/KissLightNovelsScraper.js @@ -1,6 +1,6 @@ const cheerio = require("cheerio"); const fetch = require("node-fetch"); -const { htmlToText } = require("html-to-text"); +const { parseHtml } = require("../../helper"); const baseUrl = "https://kisslightnovels.info/"; @@ -119,7 +119,7 @@ const chapterScraper = async (req, res) => { const chapterName = $("div.text-left > h4").text(); let chapterText = $(".reading-content").html(); - chapterText = htmlToText(chapterText); + chapterText = parseHtml(chapterText); let nextChapter = null; diff --git a/src/en/lightnovelpub/LightNovelPubScraper.js b/src/en/lightnovelpub/LightNovelPubScraper.js index 595f1bb..ef41cef 100644 --- a/src/en/lightnovelpub/LightNovelPubScraper.js +++ b/src/en/lightnovelpub/LightNovelPubScraper.js @@ -1,6 +1,7 @@ const cheerio = require("cheerio"); const fetch = require("node-fetch"); const { htmlToText } = require("html-to-text"); +const { parseHtml } = require("../../helper"); const { scraper } = require("../../helper"); const baseUrl = "https://www.lightnovelpub.com/"; @@ -126,7 +127,7 @@ const chapterScraper = async (req, res) => { const chapterName = $("h2").text(); let chapterText = $("#chapter-container").html(); - chapterText = htmlToText(chapterText); + chapterText = parseHtml(chapterText); let nextChapter = null; diff --git a/src/en/novelfull/NovelFullScraper.js b/src/en/novelfull/NovelFullScraper.js index 6bde106..fe73abf 100644 --- a/src/en/novelfull/NovelFullScraper.js +++ b/src/en/novelfull/NovelFullScraper.js @@ -1,6 +1,6 @@ const cheerio = require("cheerio"); const fetch = require("node-fetch"); -const { htmlToText } = require("html-to-text"); +const { parseHtml } = require("../../helper"); const baseUrl = "https://novelfull.com"; @@ -134,7 +134,7 @@ const chapterScraper = async (req, res) => { const chapterName = $(".chapter-title").attr("title"); let chapterText = $("#chapter-content").html(); - chapterText = htmlToText(chapterText); + chapterText = parseHtml(chapterText); let nextChapter = null; if ($("a#next_chap").attr("href")) { diff --git a/src/en/novelhall/NovelhallScraper.js b/src/en/novelhall/NovelhallScraper.js index 3f3e37e..fe1f8b8 100644 --- a/src/en/novelhall/NovelhallScraper.js +++ b/src/en/novelhall/NovelhallScraper.js @@ -1,6 +1,6 @@ const cheerio = require("cheerio"); const fetch = require("node-fetch"); -const { htmlToText } = require("html-to-text"); +const { parseHtml } = require("../../helper"); const baseUrl = "https://www.novelhall.com/"; @@ -109,7 +109,7 @@ const chapterScraper = async (req, res) => { const chapterName = $("h1").text(); let chapterText = $("div.entry-content").html(); - chapterText = htmlToText(chapterText); + chapterText = parseHtml(chapterText); let nextChapter = null; nextChapter = $('a[rel="next"]').attr("href"); diff --git a/src/en/noveltrench/NovelTrenchScraper.js b/src/en/noveltrench/NovelTrenchScraper.js index 67932a8..3e2a7df 100644 --- a/src/en/noveltrench/NovelTrenchScraper.js +++ b/src/en/noveltrench/NovelTrenchScraper.js @@ -1,6 +1,6 @@ const cheerio = require("cheerio"); const fetch = require("node-fetch"); -const { htmlToText } = require("html-to-text"); +const { parseHtml } = require("../../helper"); const baseUrl = "https://noveltrench.com/"; @@ -119,7 +119,7 @@ const chapterScraper = async (req, res) => { const chapterName = $("h1#chapter-heading").text(); let chapterText = $(".reading-content").html(); - chapterText = htmlToText(chapterText); + chapterText = parseHtml(chapterText); let nextChapter = null; diff --git a/src/en/readlightnovel/ReadLightNovelScraper.js b/src/en/readlightnovel/ReadLightNovelScraper.js index 48c6b0d..3b04b84 100644 --- a/src/en/readlightnovel/ReadLightNovelScraper.js +++ b/src/en/readlightnovel/ReadLightNovelScraper.js @@ -1,7 +1,7 @@ const cheerio = require("cheerio"); const fetch = require("node-fetch"); const request = require("request"); -const { htmlToText } = require("html-to-text"); +const { parseHtml } = require("../../helper"); const baseUrl = "https://www.readlightnovel.org"; const searchUrl = "https://www.readlightnovel.org/detailed-search"; @@ -156,7 +156,8 @@ const chapterScraper = async (req, res) => { $(".hidden").remove(); let chapterText = $(".desc").html(); - chapterText = htmlToText(chapterText).replace( + + chapterText = parseHtml(chapterText).replace( /\n\nSponsored Content\n\n/g, "" ); diff --git a/src/en/readnovelfull/ReadNovelFullScraper.js b/src/en/readnovelfull/ReadNovelFullScraper.js index 634246c..ef5ff7a 100644 --- a/src/en/readnovelfull/ReadNovelFullScraper.js +++ b/src/en/readnovelfull/ReadNovelFullScraper.js @@ -1,6 +1,6 @@ const cheerio = require("cheerio"); const fetch = require("node-fetch"); -const { htmlToText } = require("html-to-text"); +const { parseHtml } = require("../../helper"); const baseUrl = "https://readnovelfull.com"; @@ -134,7 +134,7 @@ const chapterScraper = async (req, res) => { const chapterName = $(".chr-title").attr("title"); let chapterText = $("#chr-content").html(); - chapterText = htmlToText(chapterText); + chapterText = parseHtml(chapterText); let nextChapter = null; if ($("a#next_chap").attr("href")) { diff --git a/src/en/vipnovel/VipNovelScraper.js b/src/en/vipnovel/VipNovelScraper.js index 050b54d..09511be 100644 --- a/src/en/vipnovel/VipNovelScraper.js +++ b/src/en/vipnovel/VipNovelScraper.js @@ -1,6 +1,7 @@ const cheerio = require("cheerio"); const fetch = require("node-fetch"); const { htmlToText } = require("html-to-text"); +const { parseHtml } = require("../../helper"); const baseUrl = "https://vipnovel.com/"; @@ -126,7 +127,7 @@ const chapterScraper = async (req, res) => { } let chapterText = $(".text-left").html(); - chapterText = htmlToText(chapterText); + chapterText = parseHtml(chapterText); let nextChapter = null; diff --git a/src/en/wuxiaworld/WuxiaWorldScraper.js b/src/en/wuxiaworld/WuxiaWorldScraper.js index c04dee6..d1a5b7b 100644 --- a/src/en/wuxiaworld/WuxiaWorldScraper.js +++ b/src/en/wuxiaworld/WuxiaWorldScraper.js @@ -1,6 +1,6 @@ const cheerio = require("cheerio"); const fetch = require("node-fetch"); -const { htmlToText } = require("html-to-text"); +const { parseHtml } = require("../../helper"); const baseUrl = "https://www.wuxiaworld.com/"; @@ -117,7 +117,7 @@ const chapterScraper = async (req, res) => { $("#chapter-content > script").remove(); let chapterText = $("#chapter-content").html(); - chapterText = htmlToText(chapterText); + chapterText = parseHtml(chapterText); let nextChapter = null; diff --git a/src/en/wuxiaworldco/WuxiaWorldCoScraper.js b/src/en/wuxiaworldco/WuxiaWorldCoScraper.js index 80d1f5f..30ec45d 100644 --- a/src/en/wuxiaworldco/WuxiaWorldCoScraper.js +++ b/src/en/wuxiaworldco/WuxiaWorldCoScraper.js @@ -1,6 +1,7 @@ const cheerio = require("cheerio"); const fetch = require("node-fetch"); const { htmlToText } = require("html-to-text"); +const { parseHtml } = require("../../helper"); const baseUrl = "https://www.wuxiaworld.co/"; @@ -95,7 +96,7 @@ const chapterScraper = async (req, res) => { const chapterName = $("h1.chapter-title").text(); let chapterText = $("div.chapter-entity").html(); - chapterText = htmlToText(chapterText); + chapterText = parseHtml(chapterText); let nextChapter = null; diff --git a/src/helper.js b/src/helper.js index d569e3f..5bed3ba 100644 --- a/src/helper.js +++ b/src/helper.js @@ -1,4 +1,5 @@ const chromium = require("chrome-aws-lambda"); +const { htmlToText } = require("html-to-text"); const puppeteer = require("puppeteer"); const UserAgent = require("user-agents"); @@ -6,7 +7,12 @@ const userAgent = new UserAgent(); const scraper = async (url) => { const browser = await puppeteer.launch({ - args: [...chromium.args, "--hide-scrollbars", "--disable-web-security", "--user-agent=" + userAgent + ""], + args: [ + ...chromium.args, + "--hide-scrollbars", + "--disable-web-security", + "--user-agent=" + userAgent + "", + ], defaultViewport: chromium.defaultViewport, executablePath: await chromium.executablePath, headless: true, @@ -20,4 +26,14 @@ const scraper = async (url) => { return body; }; -module.exports = { scraper }; +const parseHtml = (html) => { + let text = htmlToText(html); + text = text.replace( + /(?<=[ A-Za-z0-9_@./#&+-])\n(?=[ A-Za-z0-9_@./#&+-])/g, + " " + ); + + return text; +}; + +module.exports = { scraper, parseHtml };