diff --git a/index.js b/index.js index fbce1bf..29c89d5 100644 --- a/index.js +++ b/index.js @@ -34,6 +34,7 @@ app.use("/api/13/", require("./src/en/freewebnovel/FreeWebNovel")); app.use("/api/14/", require("./src/en/jpmtl/JPMTL")); app.use("/api/15/", require("./src/en/lightnovelpub/LightNovelPub")); app.use("/api/16/", require("./src/en/wuxiaworldco/WuxiaWorldCo")); +app.use("/api/17/", require("./src/en/tapread/TapRead")); const PORT = process.env.PORT || 5000; diff --git a/package-lock.json b/package-lock.json index c10ab65..d8748ab 100644 --- a/package-lock.json +++ b/package-lock.json @@ -839,12 +839,12 @@ "integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE=" }, "form-data": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz", - "integrity": "sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==", + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", + "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", "requires": { "asynckit": "^0.4.0", - "combined-stream": "^1.0.6", + "combined-stream": "^1.0.8", "mime-types": "^2.1.12" } }, @@ -1915,6 +1915,16 @@ "uuid": "^3.3.2" }, "dependencies": { + "form-data": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz", + "integrity": "sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==", + "requires": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.6", + "mime-types": "^2.1.12" + } + }, "qs": { "version": "6.5.2", "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz", diff --git a/package.json b/package.json index d220ecc..a0ee5d6 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,7 @@ "cheerio": "^1.0.0-rc.5", "chrome-aws-lambda": "^8.0.2", "express": "^4.17.1", + "form-data": "^4.0.0", "html-to-text": "^7.1.1", "node-fetch": "^2.6.1", "puppeteer": "^9.0.0", diff --git a/sources.js b/sources.js index 1576233..c77c386 100644 --- a/sources.js +++ b/sources.js @@ -134,6 +134,15 @@ extensions = [ sourceLanguage: "English", status: 1, }, + { + sourceId: 17, + sourceUrl: "http://www.tapread.com/", + sourceName: "TapRead", + sourceCover: + "https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/tapread/icon.png?raw=true", + sourceLanguage: "English", + status: 1, + }, { sourceId: 15, sourceUrl: "https://www.lightnovelpub.com/", diff --git a/src/en/tapread/TapRead.js b/src/en/tapread/TapRead.js new file mode 100644 index 0000000..c341292 --- /dev/null +++ b/src/en/tapread/TapRead.js @@ -0,0 +1,14 @@ +const express = require("express"); +const tapReadScraper = require("./TapReadScraper"); + +const router = express.Router(); + +router.get("/novels/", tapReadScraper.novelsScraper); + +router.get("/novel/:novelUrl", tapReadScraper.novelScraper); + +router.get("/novel/:novelUrl/:chapterUrl", tapReadScraper.chapterScraper); + +router.get("/search/", tapReadScraper.searchScraper); + +module.exports = router; diff --git a/src/en/tapread/TapReadScraper.js b/src/en/tapread/TapReadScraper.js new file mode 100644 index 0000000..8b1c957 --- /dev/null +++ b/src/en/tapread/TapReadScraper.js @@ -0,0 +1,177 @@ +const cheerio = require("cheerio"); +const fetch = require("node-fetch"); +var FormData = require("form-data"); +const { htmlToText } = require("html-to-text"); + +const baseUrl = "http://www.tapread.com"; + +const novelsScraper = async (req, res) => { + const result = await fetch(baseUrl); + const body = await result.text(); + + $ = cheerio.load(body); + + let novels = []; + + $(".section-item").each(function (result) { + const novelName = $(this).find(".book-name").text(); + const novelCover = $(this).find("img").attr("src"); + let novelUrl = $(this).attr("data-id"); + novelUrl += "/"; + + const novel = { + extensionId: 17, + novelName, + novelCover, + novelUrl, + }; + + novels.push(novel); + }); + + res.json(novels); +}; + +const novelScraper = async (req, res) => { + const novelUrl = req.params.novelUrl; + const url = `${baseUrl}/book/detail/${novelUrl}`; + + const result = await fetch(url); + const body = await result.text(); + + $ = cheerio.load(body); + + let novel = {}; + + novel.extensionId = 17; + + novel.sourceName = "TapRead"; + + novel.sourceUrl = url; + + novel.novelUrl = `${novelUrl}/`; + + novel.novelName = $(".book-name").text(); + + novel.novelCover = $("div.book-img > img").attr("src"); + + novel["Genre(s)"] = $("div.book-catalog > span.txt").text(); + + novel.Status = $("div.book-state > span.txt").text(); + + novel["Author(s)"] = $("div.author > span.name").text(); + + novelSummary = $("div.content > p.desc").html(); + novel.novelSummary = htmlToText(novelSummary); + + const getChapters = async (novelId) => { + const chapterListUrl = "http://www.tapread.com/ajax/book/contents"; + + let formData = new FormData(); + formData.append("bookId", novelId); + + const data = await fetch(chapterListUrl, { + method: "POST", + body: formData, + }); + + const chapters = await data.json(); + + let novelChapters = []; + + chapters.result.chapterList.map((chapter) => { + let chapterName = chapter.chapterName; + const releaseDate = chapter.pubTime; + const chapterUrl = chapter.chapterId; + + if (chapter.lock) { + chapterName = "🔒 " + chapterName; + } + + novelChapters.push({ + chapterUrl, + chapterName, + releaseDate, + }); + }); + + return novelChapters; + }; + + novel.novelChapters = await getChapters(novelUrl); + + res.json(novel); +}; + +const chapterScraper = async (req, res) => { + let novelUrl = req.params.novelUrl; + let chapterUrl = req.params.chapterUrl; + + let formData = new FormData(); + formData.append("bookId", novelUrl); + formData.append("chapterId", chapterUrl); + + const url = "http://www.tapread.com/ajax/book/chapter"; + + const result = await fetch(url, { + method: "POST", + body: formData, + }); + const body = await result.json(); + + const chapterName = body.result.chapterName; + const chapterText = htmlToText(body.result.content); + const nextChapter = body.result.nextChapterId; + const prevChapter = body.result.preChapterId; + + const chapter = { + extensionId: 17, + novelUrl, + chapterUrl, + chapterName, + chapterText, + nextChapter, + prevChapter, + }; + + res.json(chapter); +}; + +const searchScraper = async (req, res) => { + const searchTerm = req.query.s; + + let formData = new FormData(); + formData.append("storyType", 1); + formData.append("pageNo", 1); + formData.append("searchText", searchTerm); + + const url = "http://www.tapread.com/ajax/search/story"; + + const data = await fetch(url, { + method: "POST", + body: formData, + }); + + const body = await data.json(); + + console.log(body.result.storyList); + + const novels = []; + + body.result.storyList.map((novel) => { + const novelName = novel.storyName; + const novelUrl = novel.storyId; + const novelCover = novel.coverUrl; + + novels.push({ extensionId: 17, novelName, novelCover, novelUrl }); + }); + + res.json(novels); +}; + +module.exports = tapReadScraper = { + novelsScraper, + novelScraper, + chapterScraper, + searchScraper, +}; diff --git a/src/en/tapread/icon.png b/src/en/tapread/icon.png new file mode 100644 index 0000000..1eb534e Binary files /dev/null and b/src/en/tapread/icon.png differ