Create v2
This commit is contained in:
Rajarshee Chatterjee
@@ -1,3 +0,0 @@
|
||||
/node_modules
|
||||
|
||||
Procfile
|
||||
@@ -1,21 +0,0 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2021 Rajarshee Chatterjee
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
@@ -1,97 +0,0 @@
|
||||
# LNReader Sources
|
||||
|
||||
Sources for [LNReader](https://github.com/rajarsheechatterjee/lnreader)
|
||||
|
||||
## Sources
|
||||
|
||||
| Name | Source URL |
|
||||
| ------------------ | ------------------------------------------ |
|
||||
| BoxNovel | https://boxnovel.com/ |
|
||||
| ReadLightNovel | https://www.readlightnovel.org |
|
||||
| FastNovel | https://fastnovel.net/ |
|
||||
| ReadNovelFull | https://readnovelfull.com/ |
|
||||
| MTLNovel | https://www.mtlnovel.com/ |
|
||||
| NovelHall | https://www.novelhall.com/ |
|
||||
| WuxiaWorld | https://www.wuxiaworld.com/ |
|
||||
| NovelFull | https://novelfull.com/ |
|
||||
| NovelTrench | https://noveltrench.com/ |
|
||||
| VipNovel | https://vipnovel.com/ |
|
||||
| KissLightNovels | https://kisslightnovels.info/ |
|
||||
| WuxiaWorldSite | https://wuxiaworldsite.co/ |
|
||||
| FreeWebNovel | https://freewebnovel.com/ |
|
||||
| JPMTL | https://jpmtl.com/ |
|
||||
| LightNovelPub | https://www.lightnovelpub.com/ |
|
||||
| WuxiaWorld.co | https://www.wuxiaworld.co/ |
|
||||
| TapRead | http://www.tapread.com/ |
|
||||
| NovelUpdates.cc | https://www.novelupdates.cc/ |
|
||||
| ReadLightNovel.cc | https://m.readlightnovel.cc/ |
|
||||
| WuxiaWorld.cloud | http://wuxiaworld.cloud/ |
|
||||
| WoopRead | https://woopread.com/ |
|
||||
| Foxaholic | https://foxaholic.com/ |
|
||||
| Tunovelaligera | https://tunovelaligera.com/ |
|
||||
| SkyNovels | https://www.skynovels.net/ |
|
||||
| EinherjarProject | https://einherjarproject.net/ |
|
||||
| NovelasLigera | https://novelasligera.com/ |
|
||||
| ComradeMao | https://comrademao.com/ |
|
||||
| YuukiTls | https://yuukitls.com/ |
|
||||
| Hasu Translations | https://hasutl.wordpress.com/ |
|
||||
| Oasis Translations | https://oasistranslations.wordpress.com/ |
|
||||
| Novela Wuxia | http://www.novelawuxia.com/ |
|
||||
| NovelPassion | https://www.novelpassion.com/ |
|
||||
| Royal Road | https://www.royalroad.com/ |
|
||||
| Scribble Hub | https://www.scribblehub.com/ |
|
||||
| Syosetu | https://syosetu.com |
|
||||
| LNMTL | https://lnmtl.com/ |
|
||||
| SkyNovel | https://skynovel.org/ |
|
||||
| NovelCake | https://novelcake.com/ |
|
||||
| NovelsRock | https://novelsrock.com/ |
|
||||
| ZinnNovel | https://zinnovel.com/ |
|
||||
| NovelTranslate | https://noveltranslate.com/ |
|
||||
| LunarLetters | https://www.lunarletters.com/ |
|
||||
| SleepyTranslations | https://sleepytranslations.com/ |
|
||||
| FreeNovelMe | https://freenovel.me/ |
|
||||
| 1stKissNovel | https://1stkissnovel.love/ |
|
||||
| DaoNovel | https://daonovel.com/ |
|
||||
| Wuxia.Blog | https://www.wuxia.blog/ |
|
||||
| Wuxia.City | https://wuxia.city/ |
|
||||
| Novel Updates | https://www.novelupdates.com/ |
|
||||
| Ranobes | https://ranobes.net/ |
|
||||
| Yushubo | https://m.yushubo.com |
|
||||
| KolNovel | https://kolnovel.com/ |
|
||||
| RewayatAr | https://rewayat-ar.site/ |
|
||||
| MostNovel | https://mostnovel.com/ |
|
||||
| NovelMultiverse | https://www.novelmultiverse.com/ |
|
||||
| LightNovelHeaven | https://lightnovelheaven.com/ |
|
||||
| LightNovelsHub | https://lightnovelshub.com/ |
|
||||
| ArNovel | https://arnovel.me/ |
|
||||
| MeioNovel | https://meionovel.id/ |
|
||||
| WebNovelLover | https://www.webnovelover.com/ |
|
||||
| WLNUpdates | https://www.wlnupdates.com/ |
|
||||
| BoxNovel.online | https://boxnovel.online/ |
|
||||
| ClickNovel | https://clicknovel.net/ |
|
||||
| ReadWebNovels | https://readwebnovels.net/ |
|
||||
| WBNovel | https://wbnovel.com/ |
|
||||
| ReaperScans | https://reaperscans.com/all-series/novels/ |
|
||||
| Readwn.com | https://www.readwn.com/ |
|
||||
| Ranobehub | https://ranobehub.org/ |
|
||||
| Divine Dao Library | https://divinedaolibrary.com/ |
|
||||
|
||||
## Setup locally
|
||||
|
||||
Clone this repository
|
||||
|
||||
```
|
||||
git clone https://github.com/rajarsheechatterjee/lnreader-extensions.git
|
||||
```
|
||||
|
||||
Install dependencies
|
||||
|
||||
```
|
||||
npm install
|
||||
```
|
||||
|
||||
Start the server
|
||||
|
||||
```
|
||||
npm start
|
||||
```
|
||||
@@ -1,78 +0,0 @@
|
||||
const express = require("express");
|
||||
const sources = require("./sources");
|
||||
|
||||
const app = express();
|
||||
|
||||
app.use(
|
||||
express.json({
|
||||
extended: false,
|
||||
})
|
||||
);
|
||||
|
||||
app.get("/", (req, res) =>
|
||||
res.json({
|
||||
Author: "@rajarsheechatterjee",
|
||||
Version: "1.0.0",
|
||||
Github: "https://github.com/rajarsheechatterjee/lnreader-extensions",
|
||||
})
|
||||
);
|
||||
|
||||
// English
|
||||
app.get("/api/", (req, res) => res.json(sources));
|
||||
app.use("/api/1/", require("./src/en/boxnovel/BoxNovel"));
|
||||
app.use("/api/2/", require("./src/en/readlightnovel/ReadLightNovel"));
|
||||
app.use("/api/3/", require("./src/en/fastnovel/FastNovel"));
|
||||
app.use("/api/4/", require("./src/en/readnovelfull/ReadNovelFull"));
|
||||
app.use("/api/5/", require("./src/en/mtlnovel/MTLNovel"));
|
||||
app.use("/api/6/", require("./src/en/novelhall/Novelhall"));
|
||||
app.use("/api/7/", require("./src/en/wuxiaworld/WuxiaWorld"));
|
||||
app.use("/api/8/", require("./src/en/novelfull/NovelFull"));
|
||||
app.use("/api/9/", require("./src/en/noveltrench/NovelTrench"));
|
||||
app.use("/api/10/", require("./src/en/vipnovel/VipNovel"));
|
||||
app.use("/api/11/", require("./src/en/kisslightnovels/KissLightNovels"));
|
||||
app.use("/api/12/", require("./src/en/wuxiaworldsite/WuxiaWorldSite"));
|
||||
app.use("/api/13/", require("./src/en/freewebnovel/FreeWebNovel"));
|
||||
app.use("/api/14/", require("./src/en/jpmtl/JPMTL"));
|
||||
app.use("/api/15/", require("./src/en/lightnovelpub/LightNovelPub"));
|
||||
app.use("/api/16/", require("./src/en/wuxiaworldco/WuxiaWorldCo"));
|
||||
app.use("/api/17/", require("./src/en/tapread/TapRead"));
|
||||
app.use("/api/18/", require("./src/en/novelupdatescc/NovelUpdatesCc"));
|
||||
app.use("/api/19/", require("./src/en/readlightnovelcc/ReadLightNovelCc"));
|
||||
app.use("/api/20/", require("./src/en/wuxiaworldcloud/WuxiaWorldCloud"));
|
||||
app.use("/api/21/", require("./src/en/woopread/WoopRead"));
|
||||
app.use("/api/22/", require("./src/en/foxaholic/Foxaholic"));
|
||||
app.use("/api/27/", require("./src/en/comrademao/ComradeMao"));
|
||||
app.use("/api/33/", require("./src/en/novelpassion/NovelPassion"));
|
||||
app.use("/api/34/", require("./src/en/royalroad/RoyalRoad"));
|
||||
app.use("/api/35/", require("./src/en/scribblehub/ScribbleHub"));
|
||||
app.use("/api/37/", require("./src/en/lnmtl/LNMTL"));
|
||||
app.use("/api/48/", require("./src/en/wuxiablog/WuxiaBlog"));
|
||||
app.use("/api/49/", require("./src/en/wuxiacity/WuxiaCity"));
|
||||
app.use("/api/77/", require("./src/en/wnmtl/WNMTL"));
|
||||
|
||||
// Spanish
|
||||
app.use("/api/23/", require("./src/es/tunovelaligera/Tunovelaligera"));
|
||||
app.use("/api/24/", require("./src/es/skynovels/Skynovels"));
|
||||
app.use("/api/25/", require("./src/es/einherjarproject/EinherjarProject"));
|
||||
app.use("/api/26/", require("./src/es/novelasligera/NovelasLigera"));
|
||||
app.use("/api/28/", require("./src/es/yuukitls/YuukiTls"));
|
||||
app.use("/api/29/", require("./src/es/hasutl/Hasutl"));
|
||||
app.use("/api/30/", require("./src/es/oasistranslations/OasisTranslations"));
|
||||
app.use("/api/31/", require("./src/es/novelawuxia/NovelaWuxia"));
|
||||
|
||||
// Japanese
|
||||
app.use("/api/36", require("./src/jp/syosetu/Syosetu"));
|
||||
|
||||
// Chinese
|
||||
app.use("/api/50/", require("./src/ch/yushubo/Yushubo"));
|
||||
|
||||
// French
|
||||
app.use("/api/76/", require("./src/fr/noveldeglace/NovelDeGlace"));
|
||||
|
||||
// MultiSrc
|
||||
app.use("/api/", require("./multisrc/madara/MadaraGenerator"));
|
||||
app.use("/api/", require("./multisrc/wpmangastream/WPMangaStreamGenerator"));
|
||||
|
||||
const PORT = process.env.PORT || 5000;
|
||||
|
||||
app.listen(PORT, () => console.log(`Server started on port ${PORT}`));
|
||||
@@ -1,132 +0,0 @@
|
||||
const express = require("express");
|
||||
const MadaraScraper = require("./MadaraScraper");
|
||||
|
||||
const router = express.Router();
|
||||
|
||||
const getPath = (extensionId) => {
|
||||
const path = {
|
||||
38: { novels: "manga", novel: "manga", chapter: "manga" },
|
||||
39: { novels: "series", novel: "series", chapter: "series" },
|
||||
40: { novels: "novels", novel: "novel", chapter: "novel" },
|
||||
41: { novels: "manga", novel: "manga", chapter: "manga" },
|
||||
42: { novels: "all-novels", novel: "manga", chapter: "manga" },
|
||||
43: { novels: "series", novel: "series", chapter: "series" },
|
||||
44: { novels: "series", novel: "series", chapter: "series" },
|
||||
45: { novels: "novel", novel: "novel", chapter: "novel" },
|
||||
46: { novels: "novel", novel: "novel", chapter: "novel" },
|
||||
47: { novels: "novel-list", novel: "novel", chapter: "novel" },
|
||||
};
|
||||
|
||||
return path[extensionId];
|
||||
};
|
||||
|
||||
const madaraSources = [
|
||||
{
|
||||
extensionId: 38,
|
||||
scraper: new MadaraScraper(
|
||||
38,
|
||||
"https://skynovel.org/",
|
||||
"SkyNovel",
|
||||
getPath(38)
|
||||
),
|
||||
},
|
||||
{
|
||||
extensionId: 39,
|
||||
scraper: new MadaraScraper(
|
||||
39,
|
||||
"https://novelcake.com/",
|
||||
"NovelCake",
|
||||
getPath(39)
|
||||
),
|
||||
},
|
||||
{
|
||||
extensionId: 40,
|
||||
scraper: new MadaraScraper(
|
||||
40,
|
||||
"https://novelsrock.com/",
|
||||
"NovelsRock",
|
||||
getPath(40)
|
||||
),
|
||||
},
|
||||
{
|
||||
extensionId: 41,
|
||||
scraper: new MadaraScraper(
|
||||
41,
|
||||
"https://zinnovel.com/",
|
||||
"ZinNovel",
|
||||
getPath(41)
|
||||
),
|
||||
},
|
||||
{
|
||||
extensionId: 42,
|
||||
scraper: new MadaraScraper(
|
||||
42,
|
||||
"https://noveltranslate.com/",
|
||||
"NovelTranslate",
|
||||
getPath(42)
|
||||
),
|
||||
},
|
||||
{
|
||||
extensionId: 43,
|
||||
scraper: new MadaraScraper(
|
||||
43,
|
||||
"https://www.lunarletters.com/",
|
||||
"LunarLetters",
|
||||
getPath(43)
|
||||
),
|
||||
},
|
||||
{
|
||||
extensionId: 44,
|
||||
scraper: new MadaraScraper(
|
||||
44,
|
||||
"https://sleepytranslations.com/",
|
||||
"SleepyTranslations",
|
||||
getPath(44)
|
||||
),
|
||||
},
|
||||
{
|
||||
extensionId: 45,
|
||||
scraper: new MadaraScraper(
|
||||
45,
|
||||
"https://freenovel.me/",
|
||||
"FreeNovelMe",
|
||||
getPath(45)
|
||||
),
|
||||
},
|
||||
{
|
||||
extensionId: 46,
|
||||
scraper: new MadaraScraper(
|
||||
46,
|
||||
"https://1stkissnovel.love/",
|
||||
"1stKissNovel",
|
||||
getPath(46)
|
||||
),
|
||||
},
|
||||
{
|
||||
extensionId: 47,
|
||||
scraper: new MadaraScraper(
|
||||
47,
|
||||
"https://daonovel.com/",
|
||||
"DaoNovel",
|
||||
getPath(47)
|
||||
),
|
||||
},
|
||||
];
|
||||
|
||||
madaraSources.map((source) => {
|
||||
router.get(`/${source.extensionId}/novels/`, source.scraper.novelsScraper);
|
||||
|
||||
router.get(
|
||||
`/${source.extensionId}/novel/:novelUrl`,
|
||||
source.scraper.novelScraper
|
||||
);
|
||||
|
||||
router.get(
|
||||
`/${source.extensionId}/novel/:novelUrl/:chapterUrl`,
|
||||
source.scraper.chapterScraper
|
||||
);
|
||||
|
||||
router.get(`/${source.extensionId}/search/`, source.scraper.searchScraper);
|
||||
});
|
||||
|
||||
module.exports = router;
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 50 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 81 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 115 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 66 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 66 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 83 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 74 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 113 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 109 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 99 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 122 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 81 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 64 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 86 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 100 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 73 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 126 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 75 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 159 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 47 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 23 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 114 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 100 KiB |
@@ -1,41 +0,0 @@
|
||||
const express = require("express");
|
||||
const WPMangaStreamScraper = require("./WPMangaStreamScraper");
|
||||
|
||||
const router = express.Router();
|
||||
|
||||
const madaraSources = [
|
||||
{
|
||||
extensionId: 53,
|
||||
scraper: new WPMangaStreamScraper(
|
||||
53,
|
||||
"https://kolnovel.com/",
|
||||
"KolNovel"
|
||||
),
|
||||
},
|
||||
{
|
||||
extensionId: 54,
|
||||
scraper: new WPMangaStreamScraper(
|
||||
54,
|
||||
"https://rewayat-ar.site/",
|
||||
"Rewayat-Ar"
|
||||
),
|
||||
},
|
||||
];
|
||||
|
||||
madaraSources.map((source) => {
|
||||
router.get(`/${source.extensionId}/novels/`, source.scraper.novelsScraper);
|
||||
|
||||
router.get(
|
||||
`/${source.extensionId}/novel/:novelUrl`,
|
||||
source.scraper.novelScraper
|
||||
);
|
||||
|
||||
router.get(
|
||||
`/${source.extensionId}/novel/:novelUrl/:chapterUrl`,
|
||||
source.scraper.chapterScraper
|
||||
);
|
||||
|
||||
router.get(`/${source.extensionId}/search/`, source.scraper.searchScraper);
|
||||
});
|
||||
|
||||
module.exports = router;
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 82 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 141 KiB |
Generated
-5560
File diff suppressed because it is too large
Load Diff
@@ -1,26 +0,0 @@
|
||||
{
|
||||
"name": "lnreader-extensions",
|
||||
"version": "1.0.0",
|
||||
"description": "Extensions for LNReader",
|
||||
"main": "index.js",
|
||||
"scripts": {
|
||||
"start": "node index.js",
|
||||
"server": "nodemon index.js"
|
||||
},
|
||||
"author": "Rajarshee Chatterjee",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"cheerio": "^1.0.0-rc.5",
|
||||
"chrome-aws-lambda": "^8.0.2",
|
||||
"express": "^4.17.1",
|
||||
"form-data": "^4.0.0",
|
||||
"html-to-text": "^7.1.1",
|
||||
"node-fetch": "^2.6.1",
|
||||
"puppeteer": "^9.0.0",
|
||||
"request": "^2.88.2",
|
||||
"user-agents": "^1.0.654"
|
||||
},
|
||||
"devDependencies": {
|
||||
"nodemon": "^2.0.7"
|
||||
}
|
||||
}
|
||||
-471
@@ -1,471 +0,0 @@
|
||||
extensions = [
|
||||
/**
|
||||
* English
|
||||
*/
|
||||
|
||||
{
|
||||
sourceId: 1,
|
||||
sourceUrl: "https://boxnovel.com/",
|
||||
sourceName: "BoxNovel",
|
||||
sourceCover:
|
||||
"https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/boxnovel/icon.png?raw=true",
|
||||
sourceLanguage: "English",
|
||||
status: 1,
|
||||
},
|
||||
{
|
||||
sourceId: 2,
|
||||
sourceUrl: "https://www.readlightnovel.org/",
|
||||
sourceName: "ReadLightNovel",
|
||||
sourceCover:
|
||||
"https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/readlightnovel/icon.png?raw=true",
|
||||
sourceLanguage: "English",
|
||||
status: 1,
|
||||
},
|
||||
{
|
||||
sourceId: 3,
|
||||
sourceUrl: "https://fastnovel.net/",
|
||||
sourceName: "FastNovel",
|
||||
sourceCover:
|
||||
"https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/fastnovel/icon.png?raw=true",
|
||||
sourceLanguage: "English",
|
||||
status: 0,
|
||||
},
|
||||
{
|
||||
sourceId: 4,
|
||||
sourceUrl: "https://readnovelfull.com/",
|
||||
sourceName: "ReadNovelFull",
|
||||
sourceCover:
|
||||
"https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/readnovelfull/icon.png?raw=true",
|
||||
sourceLanguage: "English",
|
||||
status: 1,
|
||||
},
|
||||
{
|
||||
sourceId: 5,
|
||||
sourceUrl: "https://www.mtlnovel.com/",
|
||||
sourceName: "MTLNovel",
|
||||
sourceCover:
|
||||
"https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/mtlnovel/icon.png?raw=true",
|
||||
sourceLanguage: "English",
|
||||
status: 1,
|
||||
},
|
||||
{
|
||||
sourceId: 6,
|
||||
sourceUrl: "https://www.novelhall.com/",
|
||||
sourceName: "NovelHall",
|
||||
sourceCover:
|
||||
"https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/novelhall/icon.png?raw=true",
|
||||
sourceLanguage: "English",
|
||||
status: 1,
|
||||
},
|
||||
{
|
||||
sourceId: 7,
|
||||
sourceUrl: "https://www.wuxiaworld.com/",
|
||||
sourceName: "WuxiaWorld",
|
||||
sourceCover:
|
||||
"https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/wuxiaworld/icon.png?raw=true",
|
||||
sourceLanguage: "English",
|
||||
status: 1,
|
||||
},
|
||||
{
|
||||
sourceId: 8,
|
||||
sourceUrl: "https://novelfull.com/",
|
||||
sourceName: "NovelFull",
|
||||
sourceCover:
|
||||
"https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/novelfull/icon.png?raw=true",
|
||||
sourceLanguage: "English",
|
||||
status: 1,
|
||||
},
|
||||
{
|
||||
sourceId: 9,
|
||||
sourceUrl: "https://noveltrench.com/",
|
||||
sourceName: "NovelTrench",
|
||||
sourceCover:
|
||||
"https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/noveltrench/icon.png?raw=true",
|
||||
sourceLanguage: "English",
|
||||
status: 1,
|
||||
},
|
||||
{
|
||||
sourceId: 10,
|
||||
sourceUrl: "https://vipnovel.com/",
|
||||
sourceName: "VipNovel",
|
||||
sourceCover:
|
||||
"https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/vipnovel/icon.png?raw=true",
|
||||
sourceLanguage: "English",
|
||||
status: 1,
|
||||
},
|
||||
{
|
||||
sourceId: 11,
|
||||
sourceUrl: "https://kisslightnovels.info/",
|
||||
sourceName: "KissLightNovels",
|
||||
sourceCover:
|
||||
"https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/kisslightnovels/icon.png?raw=true",
|
||||
sourceLanguage: "English",
|
||||
status: 1,
|
||||
},
|
||||
{
|
||||
sourceId: 12,
|
||||
sourceUrl: "https://wuxiaworldsite.co/",
|
||||
sourceName: "WuxiaWorldSite",
|
||||
sourceCover:
|
||||
"https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/wuxiaworldsite/icon.png?raw=true",
|
||||
sourceLanguage: "English",
|
||||
status: 1,
|
||||
},
|
||||
{
|
||||
sourceId: 13,
|
||||
sourceUrl: "https://freewebnovel.com/",
|
||||
sourceName: "FreeWebNovel",
|
||||
sourceCover:
|
||||
"https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/freewebnovel/icon.png?raw=true",
|
||||
sourceLanguage: "English",
|
||||
status: 1,
|
||||
},
|
||||
{
|
||||
sourceId: 14,
|
||||
sourceUrl: "https://jpmtl.com/",
|
||||
sourceName: "JPMTL",
|
||||
sourceCover:
|
||||
"https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/jpmtl/icon.png?raw=true",
|
||||
sourceLanguage: "English",
|
||||
status: 1,
|
||||
},
|
||||
{
|
||||
sourceId: 15,
|
||||
sourceUrl: "https://www.lightnovelpub.com/",
|
||||
sourceName: "LightNovelPub",
|
||||
sourceCover:
|
||||
"https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/lightnovelpub/icon.png?raw=true",
|
||||
sourceLanguage: "English",
|
||||
status: 1,
|
||||
},
|
||||
{
|
||||
sourceId: 16,
|
||||
sourceUrl: "https://www.wuxiaworld.co/",
|
||||
sourceName: "WuxiaWorld.co",
|
||||
sourceCover:
|
||||
"https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/wuxiaworldco/icon.png?raw=true",
|
||||
sourceLanguage: "English",
|
||||
status: 1,
|
||||
},
|
||||
{
|
||||
sourceId: 17,
|
||||
sourceUrl: "http://www.tapread.com/",
|
||||
sourceName: "TapRead",
|
||||
sourceCover:
|
||||
"https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/tapread/icon.png?raw=true",
|
||||
sourceLanguage: "English",
|
||||
status: 1,
|
||||
},
|
||||
{
|
||||
sourceId: 18,
|
||||
sourceUrl: "https://www.novelupdates.cc/",
|
||||
sourceName: "NovelUpdates.cc",
|
||||
sourceCover:
|
||||
"https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/novelupdatescc/icon.png?raw=true",
|
||||
sourceLanguage: "English",
|
||||
status: 1,
|
||||
},
|
||||
{
|
||||
sourceId: 19,
|
||||
sourceUrl: "https://m.readlightnovel.cc/",
|
||||
sourceName: "ReadLightNovel.cc",
|
||||
sourceCover:
|
||||
"https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/readlightnovelcc/icon.png?raw=true",
|
||||
sourceLanguage: "English",
|
||||
status: 1,
|
||||
},
|
||||
{
|
||||
sourceId: 20,
|
||||
sourceUrl: "http://wuxiaworld.cloud/",
|
||||
sourceName: "WuxiaWorld.cloud",
|
||||
sourceCover:
|
||||
"https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/wuxiaworldcloud/icon.png?raw=true",
|
||||
sourceLanguage: "English",
|
||||
status: 1,
|
||||
},
|
||||
{
|
||||
sourceId: 21,
|
||||
sourceUrl: "https://woopread.com/",
|
||||
sourceName: "WoopRead",
|
||||
sourceCover:
|
||||
"https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/woopread/icon.png?raw=true",
|
||||
sourceLanguage: "English",
|
||||
status: 1,
|
||||
},
|
||||
{
|
||||
sourceId: 22,
|
||||
sourceUrl: "https://foxaholic.com/",
|
||||
sourceName: "Foxaholic",
|
||||
sourceCover:
|
||||
"https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/foxaholic/icon.png?raw=true",
|
||||
sourceLanguage: "English",
|
||||
status: 1,
|
||||
},
|
||||
{
|
||||
sourceId: 27,
|
||||
sourceUrl: "https://comrademao.com/",
|
||||
sourceName: "ComradeMao",
|
||||
sourceCover:
|
||||
"https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/comrademao/icon.png?raw=true",
|
||||
sourceLanguage: "English",
|
||||
status: 1,
|
||||
},
|
||||
{
|
||||
sourceId: 33,
|
||||
sourceUrl: "https://www.novelpassion.com/",
|
||||
sourceName: "NovelPassion",
|
||||
sourceCover:
|
||||
"https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/novelpassion/icon.png?raw=true",
|
||||
sourceLanguage: "English",
|
||||
status: 1,
|
||||
},
|
||||
{
|
||||
sourceId: 34,
|
||||
sourceUrl: "https://www.royalroad.com/",
|
||||
sourceName: "Royal Road",
|
||||
sourceCover:
|
||||
"https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/royalroad/icon.png?raw=true",
|
||||
sourceLanguage: "English",
|
||||
status: 1,
|
||||
},
|
||||
{
|
||||
sourceId: 35,
|
||||
sourceUrl: "https://www.scribblehub.com/",
|
||||
sourceName: "Scribble Hub",
|
||||
sourceCover:
|
||||
"https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/scribblehub/icon.png?raw=true",
|
||||
sourceLanguage: "English",
|
||||
status: 1,
|
||||
},
|
||||
{
|
||||
sourceId: 37,
|
||||
sourceUrl: "https://lnmtl.com/",
|
||||
sourceName: "LNMTL",
|
||||
sourceCover:
|
||||
"https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/lnmtl/icon.png?raw=true",
|
||||
sourceLanguage: "English",
|
||||
status: 1,
|
||||
},
|
||||
{
|
||||
sourceId: 38,
|
||||
sourceUrl: "https://skynovel.org/",
|
||||
sourceName: "SkyNovel",
|
||||
sourceCover:
|
||||
"https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/multisrc/madara/icons/skynovel.png?raw=true",
|
||||
sourceLanguage: "English",
|
||||
status: 1,
|
||||
},
|
||||
{
|
||||
sourceId: 39,
|
||||
sourceUrl: "https://novelcake.com/",
|
||||
sourceName: "NovelCake",
|
||||
sourceCover:
|
||||
"https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/multisrc/madara/icons/novelcake.png?raw=true",
|
||||
sourceLanguage: "English",
|
||||
status: 1,
|
||||
},
|
||||
{
|
||||
sourceId: 40,
|
||||
sourceUrl: "https://novelsrock.com/",
|
||||
sourceName: "NovelsRock",
|
||||
sourceCover:
|
||||
"https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/multisrc/madara/icons/novelsrock.png?raw=true",
|
||||
sourceLanguage: "English",
|
||||
status: 1,
|
||||
},
|
||||
{
|
||||
sourceId: 41,
|
||||
sourceUrl: "https://zinnovel.com/",
|
||||
sourceName: "ZinNovel",
|
||||
sourceCover:
|
||||
"https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/multisrc/madara/icons/zinnovel.png?raw=true",
|
||||
sourceLanguage: "English",
|
||||
status: 1,
|
||||
},
|
||||
{
|
||||
sourceId: 42,
|
||||
sourceUrl: "https://noveltranslate.com/",
|
||||
sourceName: "NovelTranslate",
|
||||
sourceCover:
|
||||
"https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/multisrc/madara/icons/noveltranslate.png?raw=true",
|
||||
sourceLanguage: "English",
|
||||
status: 1,
|
||||
},
|
||||
{
|
||||
sourceId: 43,
|
||||
sourceUrl: "https://www.lunarletters.com/",
|
||||
sourceName: "LunarLetters",
|
||||
sourceCover:
|
||||
"https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/multisrc/madara/icons/lunarletters.png?raw=true",
|
||||
sourceLanguage: "English",
|
||||
status: 1,
|
||||
},
|
||||
{
|
||||
sourceId: 44,
|
||||
sourceUrl: "https://sleepytranslations.com/",
|
||||
sourceName: "SleepyTranslations",
|
||||
sourceCover:
|
||||
"https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/multisrc/madara/icons/sleepytranslations.png?raw=true",
|
||||
sourceLanguage: "English",
|
||||
status: 1,
|
||||
},
|
||||
{
|
||||
sourceId: 45,
|
||||
sourceUrl: "https://freenovel.me/",
|
||||
sourceName: "FreeNovelMe",
|
||||
sourceCover:
|
||||
"https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/multisrc/madara/icons/freenovel.png?raw=true",
|
||||
sourceLanguage: "English",
|
||||
status: 1,
|
||||
},
|
||||
{
|
||||
sourceId: 46,
|
||||
sourceUrl: "https://1stkissnovel.love/",
|
||||
sourceName: "1stKissNovel",
|
||||
sourceCover:
|
||||
"https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/multisrc/madara/icons/1stkissnovel.png?raw=true",
|
||||
sourceLanguage: "English",
|
||||
status: 1,
|
||||
},
|
||||
{
|
||||
sourceId: 47,
|
||||
sourceUrl: "https://daonovel.com/",
|
||||
sourceName: "DaoNovel",
|
||||
sourceCover:
|
||||
"https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/multisrc/madara/icons/daonovel.png?raw=true",
|
||||
sourceLanguage: "English",
|
||||
status: 1,
|
||||
},
|
||||
{
|
||||
sourceId: 48,
|
||||
sourceUrl: "https://www.wuxia.blog/",
|
||||
sourceName: "Wuxia.Blog",
|
||||
sourceCover:
|
||||
"https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/wuxiablog/icon.png?raw=true",
|
||||
sourceLanguage: "English",
|
||||
status: 1,
|
||||
},
|
||||
{
|
||||
sourceId: 49,
|
||||
sourceUrl: "https://wuxia.city/",
|
||||
sourceName: "Wuxia.City",
|
||||
sourceCover:
|
||||
"https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/en/wuxiacity/icon.png?raw=true",
|
||||
sourceLanguage: "English",
|
||||
status: 1,
|
||||
},
|
||||
|
||||
/**
|
||||
* Spanish
|
||||
*/
|
||||
|
||||
{
|
||||
sourceId: 23,
|
||||
sourceUrl: "https://tunovelaligera.com/",
|
||||
sourceName: "Tunovelaligera",
|
||||
sourceCover:
|
||||
"https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/es/tunovelaligera/icon.png?raw=true",
|
||||
sourceLanguage: "Spanish",
|
||||
status: 1,
|
||||
},
|
||||
{
|
||||
sourceId: 24,
|
||||
sourceUrl: "https://www.skynovels.net/",
|
||||
sourceName: "SkyNovels",
|
||||
sourceCover:
|
||||
"https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/es/skynovels/icon.png?raw=true",
|
||||
sourceLanguage: "Spanish",
|
||||
status: 1,
|
||||
},
|
||||
{
|
||||
sourceId: 25,
|
||||
sourceUrl: "https://einherjarproject.net/",
|
||||
sourceName: "EinherjarProject",
|
||||
sourceCover:
|
||||
"https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/es/einherjarproject/icon.png?raw=true",
|
||||
sourceLanguage: "Spanish",
|
||||
status: 1,
|
||||
},
|
||||
{
|
||||
sourceId: 26,
|
||||
sourceUrl: "https://novelasligera.com/",
|
||||
sourceName: "NovelasLigera",
|
||||
sourceCover:
|
||||
"https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/es/novelasligera/icon.png?raw=true",
|
||||
sourceLanguage: "Spanish",
|
||||
status: 1,
|
||||
},
|
||||
{
|
||||
sourceId: 28,
|
||||
sourceUrl: "https://yuukitls.com/",
|
||||
sourceName: "YuukiTls",
|
||||
sourceCover:
|
||||
"https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/es/yuukitls/icon.png?raw=true",
|
||||
sourceLanguage: "Spanish",
|
||||
status: 1,
|
||||
},
|
||||
{
|
||||
sourceId: 29,
|
||||
sourceUrl: "https://hasutl.wordpress.com/",
|
||||
sourceName: "Hasu Translations",
|
||||
sourceCover:
|
||||
"https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/es/hasutl/icon.png?raw=true",
|
||||
sourceLanguage: "Spanish",
|
||||
status: 1,
|
||||
},
|
||||
{
|
||||
sourceId: 30,
|
||||
sourceUrl: "https://oasistranslations.wordpress.com/",
|
||||
sourceName: "Oasis Translations",
|
||||
sourceCover:
|
||||
"https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/es/oasistranslations/icon.png?raw=true",
|
||||
sourceLanguage: "Spanish",
|
||||
status: 1,
|
||||
},
|
||||
{
|
||||
sourceId: 31,
|
||||
sourceUrl: "http://www.novelawuxia.com/",
|
||||
sourceName: "Novela Wuxia",
|
||||
sourceCover:
|
||||
"https://github.com/rajarsheechatterjee/lnreader-extensions/blob/main/src/es/novelawuxia/icon.png?raw=true",
|
||||
sourceLanguage: "Spanish",
|
||||
status: 1,
|
||||
},
|
||||
|
||||
/**
|
||||
* Japanese
|
||||
*/
|
||||
{
|
||||
sourceId: 36,
|
||||
sourceUrl: "https://syosetsu.com",
|
||||
sourceName: "Syosetsu",
|
||||
sourceCover:
|
||||
"https://github.com/skillgg/lnreader-sources/blob/main/src/jp/syosetu/icon.png?raw=true",
|
||||
sourceLanguage: "Japanese",
|
||||
status: 1,
|
||||
},
|
||||
|
||||
/**
|
||||
* Arabic
|
||||
*/
|
||||
{
|
||||
sourceId: 53,
|
||||
sourceUrl: "https://kolnovel.com/",
|
||||
sourceName: "KolNovel",
|
||||
sourceCover:
|
||||
"https://github.com/LNReader/lnreader-sources/blob/main/multisrc/wpmangastream/icons/kolnovel.png?raw=true",
|
||||
sourceLanguage: "Arabic",
|
||||
status: 1,
|
||||
},
|
||||
{
|
||||
sourceId: 54,
|
||||
sourceUrl: "https://rewayat-ar.site/",
|
||||
sourceName: "Rewayat-Ar",
|
||||
sourceCover:
|
||||
"https://github.com/LNReader/lnreader-sources/blob/main/multisrc/wpmangastream/icons/rewayatar.png?raw=true",
|
||||
sourceLanguage: "Arabic",
|
||||
status: 1,
|
||||
},
|
||||
];
|
||||
|
||||
module.exports = extensions;
|
||||
@@ -0,0 +1,186 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
|
||||
const sourceId = 59;
|
||||
const sourceName = "ArNovel";
|
||||
|
||||
const baseUrl = "https://arnovel.me/";
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
let totalPages = 21;
|
||||
let url = baseUrl + "novels/page/" + page;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".page-item-detail").each(function () {
|
||||
const novelName = $(this).find(".h5 > a").text();
|
||||
const novelCover =
|
||||
"https://github.com/LNReader/lnreader-sources/blob/main/src/coverNotAvailable.jpg?raw=true";
|
||||
|
||||
let novelUrl = $(this).find(".h5 > a").attr("href").split("/")[4];
|
||||
|
||||
const novel = {
|
||||
sourceId,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return { totalPages, novels };
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = `${baseUrl}novel/${novelUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = { sourceId, sourceName, url, novelUrl };
|
||||
|
||||
novel.novelName = $(".post-title > h1")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
novel.novelCover =
|
||||
"https://github.com/LNReader/lnreader-sources/blob/main/src/coverNotAvailable.jpg?raw=true";
|
||||
|
||||
$(".post-content_item").each(function () {
|
||||
detailName = $(this)
|
||||
.find(".summary-heading > h5")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
detail = $(this)
|
||||
.find(".summary-content")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
switch (detailName) {
|
||||
case "التصنيفات":
|
||||
novel.genre = detail.trim().replace(/[\t\n]/g, ",");
|
||||
break;
|
||||
case "المؤلف (ين)":
|
||||
novel.author = detail.trim();
|
||||
break;
|
||||
case "الحالة":
|
||||
novel.status = detail.trim();
|
||||
break;
|
||||
}
|
||||
});
|
||||
|
||||
$(".description-summary > div.summary__content").find("em").remove();
|
||||
|
||||
novel.summary = $(".description-summary > div.summary__content")
|
||||
.text()
|
||||
.trim();
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
const novelId = $("#manga-chapters-holder").attr("data-id");
|
||||
|
||||
let formData = new FormData();
|
||||
formData.append("action", "manga_get_chapters");
|
||||
formData.append("manga", novelId);
|
||||
|
||||
const data = await fetch("https://www.arnovel.me/wp-admin/admin-ajax.php", {
|
||||
method: "POST",
|
||||
body: formData,
|
||||
});
|
||||
const text = await data.text();
|
||||
|
||||
$ = cheerio.load(text);
|
||||
|
||||
$(".wp-manga-chapter").each(function () {
|
||||
chapterName = $(this)
|
||||
.find("a")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
releaseDate = $(this).find("span").text().trim();
|
||||
|
||||
chapterUrl = $(this).find("a").attr("href").split("/");
|
||||
|
||||
chapterUrl[6]
|
||||
? (chapterUrl = chapterUrl[5] + "/" + chapterUrl[6])
|
||||
: (chapterUrl = chapterUrl[5]);
|
||||
|
||||
novelChapters.push({ chapterName, releaseDate, chapterUrl });
|
||||
});
|
||||
|
||||
novel.chapters = novelChapters.reverse();
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = `${baseUrl}novel/${novelUrl}/${chapterUrl}/`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
const chapterName = $("h1#chapter-heading").text();
|
||||
const chapterText = $(".reading-content").html();
|
||||
|
||||
const chapter = {
|
||||
sourceId,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const url = `${baseUrl}?s=${searchTerm}&post_type=wp-manga`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".c-tabs-item__content").each(function () {
|
||||
const novelName = $(this).find(".h4 > a").text();
|
||||
const novelCover =
|
||||
"https://github.com/LNReader/lnreader-sources/blob/main/src/coverNotAvailable.jpg?raw=true";
|
||||
|
||||
let novelUrl = $(this).find(".h4 > a").attr("href").split("/")[4];
|
||||
|
||||
const novel = {
|
||||
sourceId,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const ArNovelScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default ArNovelScraper;
|
||||
@@ -0,0 +1,183 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
|
||||
const baseUrl = "https://m.yushubo.com";
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
const url = `${baseUrl}/all/order/hits_week+desc.html?page=${page}`;
|
||||
const totalPages = 5846;
|
||||
|
||||
const result = await fetch(url, {
|
||||
headers: {
|
||||
"User-Agent":
|
||||
"'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36",
|
||||
},
|
||||
});
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".clearfix > li").each(function (result) {
|
||||
const novelUrl =
|
||||
$(this).find("a").attr("href").replace(".html", "").substring(1) +
|
||||
"/";
|
||||
|
||||
const novelName = $(this).find("a").text();
|
||||
let novelCover = $(this).find("img").attr("src");
|
||||
novelCover = novelCover.includes("https://tva1.sinaimg.cn/")
|
||||
? `${novelCover}`
|
||||
: `${baseUrl}${novelCover}`;
|
||||
|
||||
const novel = {
|
||||
sourceId: 52,
|
||||
novelUrl,
|
||||
novelName,
|
||||
novelCover,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return { totalPages, novels };
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = `${baseUrl}/${novelUrl.slice(0, -1)}.html`;
|
||||
|
||||
const result = await fetch(url, {
|
||||
headers: {
|
||||
"User-Agent":
|
||||
"'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36",
|
||||
},
|
||||
});
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.sourceId = 52;
|
||||
|
||||
novel.sourceName = "Yushubo";
|
||||
|
||||
novel.url = url;
|
||||
|
||||
novel.novelUrl = novelUrl;
|
||||
|
||||
novel.novelName = $("dd > h2").text();
|
||||
|
||||
novel.novelCover = $("dt > img").attr("src");
|
||||
|
||||
novel.summary = $("div.content > p").text();
|
||||
|
||||
let info = [];
|
||||
$(".info a").each(function (result) {
|
||||
info.push($(this).text());
|
||||
});
|
||||
|
||||
novel.genre = info[0];
|
||||
|
||||
novel.author = info[1];
|
||||
|
||||
novel.artist = null;
|
||||
|
||||
novel.status = "Unknown";
|
||||
|
||||
let chapters = [];
|
||||
|
||||
$(".bookshelf-list a").each(function (result) {
|
||||
let chapterUrl = $(this).attr("href").substring(1);
|
||||
let chapterName = $(this).attr("title");
|
||||
let releaseDate = null;
|
||||
|
||||
chapters.push({
|
||||
chapterName,
|
||||
releaseDate,
|
||||
chapterUrl,
|
||||
});
|
||||
});
|
||||
|
||||
novel.chapters = chapters;
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = `${baseUrl}/${chapterUrl}`;
|
||||
|
||||
const result = await fetch(url, {
|
||||
headers: {
|
||||
"User-Agent":
|
||||
"'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36",
|
||||
},
|
||||
});
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
const chapterName = $(".read-section h3").text();
|
||||
let chapterText = $(".read-section").html();
|
||||
|
||||
const chapter = {
|
||||
sourceId: 52,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const searchUrl = `${baseUrl}/search.html`;
|
||||
|
||||
const result = await fetch(searchUrl, {
|
||||
method: "post",
|
||||
body: JSON.stringify({ keyword: searchTerm }),
|
||||
headers: {
|
||||
"User-Agent":
|
||||
"'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36",
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
});
|
||||
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".clearfix > li").each(function (result) {
|
||||
let novelUrl =
|
||||
$(this).find("a").attr("href").replace(".html", "").substring(1) +
|
||||
"/";
|
||||
|
||||
const novelName = $(this).find("a").text();
|
||||
let novelCover = $(this).find("img").attr("src");
|
||||
novelCover = novelCover.includes("https://tva1.sinaimg.cn/")
|
||||
? `${novelCover}`
|
||||
: `${baseUrl}${novelCover}`;
|
||||
|
||||
const novel = {
|
||||
sourceId: 52,
|
||||
novelUrl,
|
||||
novelName,
|
||||
novelCover,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const YushuboScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default YushuboScraper;
|
||||
@@ -1,14 +0,0 @@
|
||||
const express = require("express");
|
||||
const YushuboScraper = require("./YushuboScraper");
|
||||
|
||||
const router = express.Router();
|
||||
|
||||
router.get("/novels/", YushuboScraper.novelsScraper);
|
||||
|
||||
router.get("/novel/:novelUrl", YushuboScraper.novelScraper);
|
||||
|
||||
router.get("/novel/:novelUrl/:chapterUrl", YushuboScraper.chapterScraper);
|
||||
|
||||
router.get("/search/", YushuboScraper.searchScraper);
|
||||
|
||||
module.exports = router;
|
||||
@@ -1,177 +0,0 @@
|
||||
const cheerio = require("cheerio");
|
||||
const fetch = require("node-fetch");
|
||||
const UserAgent = require("user-agents");
|
||||
|
||||
const userAgent = new UserAgent();
|
||||
|
||||
const baseUrl = "https://m.yushubo.com";
|
||||
|
||||
const novelsScraper = async (req, res) => {
|
||||
const url = `${baseUrl}/all/order/hits_week+desc.html`;
|
||||
|
||||
const result = await fetch(url, { headers: { "User-Agent": userAgent } });
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".clearfix > li").each(function (result) {
|
||||
const novelUrl =
|
||||
$(this).find("a").attr("href").replace(".html", "").substring(1) + "/";
|
||||
|
||||
const novelName = $(this).find("a").text();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
|
||||
const novel = {
|
||||
extensionId: 50,
|
||||
novelUrl,
|
||||
novelName,
|
||||
novelCover,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
res.json(novels);
|
||||
};
|
||||
|
||||
const novelScraper = async (req, res) => {
|
||||
const novelUrl = req.params.novelUrl;
|
||||
const url = `${baseUrl}/${novelUrl}.html`;
|
||||
|
||||
const result = await fetch(url, { headers: { "User-Agent": userAgent } });
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.extensionId = 50;
|
||||
|
||||
novel.sourceName = "Yushubo";
|
||||
|
||||
novel.sourceUrl = url;
|
||||
|
||||
novel.novelUrl = novelUrl + "/";
|
||||
|
||||
novel.novelName = $("dd > h2").text();
|
||||
|
||||
novel.novelCover = $("dt > img").attr("src");
|
||||
|
||||
novel.novelSummary = $("div.content > p").text();
|
||||
|
||||
let info = [];
|
||||
$(".info a").each(function (result) {
|
||||
info.push($(this).text());
|
||||
});
|
||||
|
||||
novel["Author(s)"] = info[0];
|
||||
|
||||
novel["Genre(s)"] = info[1];
|
||||
|
||||
novel["Artist(s)"] = null;
|
||||
|
||||
novel.Status = "Unknown";
|
||||
|
||||
let chapters = [];
|
||||
|
||||
$(".bookshelf-list a").each(function (result) {
|
||||
let chapterUrl = $(this).attr("href").substring(1);
|
||||
let chapterName = $(this).attr("title");
|
||||
let releaseDate = null;
|
||||
|
||||
chapters.push({
|
||||
chapterName,
|
||||
releaseDate,
|
||||
chapterUrl,
|
||||
});
|
||||
});
|
||||
|
||||
novel.novelChapters = chapters;
|
||||
|
||||
res.json(novel);
|
||||
};
|
||||
|
||||
const chapterScraper = async (req, res) => {
|
||||
let novelUrl = req.params.novelUrl.replace(".html/", "");
|
||||
let chapterUrl = req.params.chapterUrl;
|
||||
|
||||
const url = `${baseUrl}/${chapterUrl}`;
|
||||
|
||||
const result = await fetch(url, { headers: { "User-Agent": userAgent } });
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
const chapterName = $(".read-section h3").text();
|
||||
const chapterText = $(".read-section p").text();
|
||||
|
||||
let nextChapter = null;
|
||||
if ($(".btn-next").attr("href")) {
|
||||
nextChapter = $(".btn-prev").attr("href");
|
||||
}
|
||||
|
||||
let prevChapter = null;
|
||||
if ($(".btn-prev").attr("href")) {
|
||||
prevChapter = $(".btn-prev").attr("href");
|
||||
}
|
||||
|
||||
const chapter = {
|
||||
extensionId: 50,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
nextChapter,
|
||||
prevChapter,
|
||||
};
|
||||
|
||||
res.json(chapter);
|
||||
};
|
||||
|
||||
const searchScraper = async (req, res) => {
|
||||
const searchTerm = req.query.s;
|
||||
const searchUrl = `${baseUrl}/search.html`;
|
||||
|
||||
const result = await fetch(searchUrl, {
|
||||
method: "post",
|
||||
body: JSON.stringify({ keyword: searchTerm }),
|
||||
headers: { "User-Agent": userAgent, "Content-Type": "application/json" },
|
||||
});
|
||||
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".clearfix > li").each(function (result) {
|
||||
let novelUrl =
|
||||
$(this).find("a").attr("href").replace(".html", "").substring(1) + "/";
|
||||
|
||||
const novelName = $(this).find("a").text();
|
||||
let novelCover = $(this).find("img").attr("src");
|
||||
novelCover = novelCover.startsWith("https://tva1.sinaimg.cn/")
|
||||
? `${novelCover}`
|
||||
: `${baseUrl}${novelCover}`;
|
||||
|
||||
const novel = {
|
||||
extensionId: 50,
|
||||
novelUrl,
|
||||
novelName,
|
||||
novelCover,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
res.json(novels);
|
||||
};
|
||||
|
||||
module.exports = YushuboScraper = {
|
||||
novelsScraper,
|
||||
novelScraper,
|
||||
chapterScraper,
|
||||
searchScraper,
|
||||
};
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 57 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 12 KiB |
@@ -1,15 +1,13 @@
|
||||
const cheerio = require("cheerio");
|
||||
const fetch = require("node-fetch");
|
||||
const { parseHtml } = require("../../helper");
|
||||
import cheerio from "react-native-cheerio";
|
||||
|
||||
const baseUrl = "https://boxnovel.com/novel";
|
||||
const searchUrl = "https://boxnovel.com/";
|
||||
|
||||
const novelsScraper = async (req, res) => {
|
||||
const orderBy = req.params.o;
|
||||
const pageNo = req.params.pageNo;
|
||||
const sourceId = 1;
|
||||
|
||||
let url = `${baseUrl}/page/1/?m_orderby=${orderBy}`;
|
||||
const popularNovels = async (page) => {
|
||||
let url = `${baseUrl}/page/${page}/?m_orderby=rating`;
|
||||
let totalPages = 73;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
@@ -26,7 +24,7 @@ const novelsScraper = async (req, res) => {
|
||||
novelUrl = novelUrl.replace(`${baseUrl}/`, "");
|
||||
|
||||
const novel = {
|
||||
extensionId: 1,
|
||||
sourceId,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
@@ -35,12 +33,11 @@ const novelsScraper = async (req, res) => {
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
res.json(novels);
|
||||
return { totalPages, novels };
|
||||
};
|
||||
|
||||
const novelScraper = async (req, res) => {
|
||||
const novelUrl = req.params.novelUrl;
|
||||
const url = `${baseUrl}/${novelUrl}/`;
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = `${baseUrl}/${novelUrl}`;
|
||||
|
||||
console.log(url);
|
||||
|
||||
@@ -53,13 +50,13 @@ const novelScraper = async (req, res) => {
|
||||
|
||||
$(".post-title > h3 > span").remove();
|
||||
|
||||
novel.extensionId = 1;
|
||||
novel.sourceId = 1;
|
||||
|
||||
novel.sourceName = "BoxNovel";
|
||||
|
||||
novel.sourceUrl = url;
|
||||
novel.url = url;
|
||||
|
||||
novel.novelUrl = `${novelUrl}/`;
|
||||
novel.novelUrl = novelUrl;
|
||||
|
||||
novel.novelName = $(".post-title > h3")
|
||||
.text()
|
||||
@@ -80,44 +77,53 @@ const novelScraper = async (req, res) => {
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
novel[detailName] = detail;
|
||||
switch (detailName) {
|
||||
case "Genre(s)":
|
||||
novel.genre = detail.trim().replace(/[\t\n]/g, ",");
|
||||
break;
|
||||
case "Author(s)":
|
||||
novel.author = detail.trim();
|
||||
break;
|
||||
case "Artist(s)":
|
||||
novel.status = detail.trim();
|
||||
break;
|
||||
}
|
||||
});
|
||||
|
||||
$(".description-summary > div.summary__content").find("em").remove();
|
||||
|
||||
novel.novelSummary = $(".description-summary > div.summary__content > div")
|
||||
novel.summary = $(".description-summary > div.summary__content")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "");
|
||||
.replace(/[\t\n]|Description|Summary/g, "");
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
$(".wp-manga-chapter").each(function (result) {
|
||||
chapterName = $(this)
|
||||
const chapterName = $(this)
|
||||
.find("a")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
releaseDate = $(this)
|
||||
const releaseDate = $(this)
|
||||
.find("span")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
chapterUrl = $(this).find("a").attr("href").replace(url, "");
|
||||
const chapterUrl = $(this).find("a").attr("href").replace(url, "");
|
||||
|
||||
novelChapters.push({ chapterName, releaseDate, chapterUrl });
|
||||
const chapter = { chapterName, releaseDate, chapterUrl };
|
||||
|
||||
novelChapters.push(chapter);
|
||||
});
|
||||
|
||||
novel.novelChapters = novelChapters.reverse();
|
||||
novel.chapters = novelChapters.reverse();
|
||||
|
||||
res.json(novel);
|
||||
return novel;
|
||||
};
|
||||
|
||||
const chapterScraper = async (req, res) => {
|
||||
let novelUrl = req.params.novelUrl;
|
||||
let chapterUrl = req.params.chapterUrl;
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = `${baseUrl}/${novelUrl}/${chapterUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
@@ -127,47 +133,19 @@ const chapterScraper = async (req, res) => {
|
||||
|
||||
const chapterName = $("div.text-left > h3").text();
|
||||
let chapterText = $(".reading-content").html();
|
||||
chapterText = parseHtml(chapterText);
|
||||
|
||||
let nextChapter = null;
|
||||
|
||||
if ($(".nav-next").length) {
|
||||
nextChapter = $(".nav-next")
|
||||
.find("a")
|
||||
.attr("href")
|
||||
.replace(baseUrl + "/" + novelUrl + "/", "");
|
||||
}
|
||||
|
||||
let prevChapter = null;
|
||||
|
||||
if ($(".nav-previous").length) {
|
||||
prevChapter = $(".nav-previous")
|
||||
.find("a")
|
||||
.attr("href")
|
||||
.replace(baseUrl + "/" + novelUrl + "/", "");
|
||||
}
|
||||
|
||||
novelUrl = novelUrl + "/";
|
||||
chapterUrl = chapterUrl + "/";
|
||||
|
||||
const chapter = {
|
||||
extensionId: 1,
|
||||
sourceId,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
nextChapter,
|
||||
prevChapter,
|
||||
};
|
||||
|
||||
res.json(chapter);
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchScraper = async (req, res) => {
|
||||
const searchTerm = req.query.s;
|
||||
const orderBy = req.query.o;
|
||||
|
||||
const url = `${searchUrl}?s=${searchTerm}&post_type=wp-manga&m_orderby=${orderBy}`;
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const url = `${searchUrl}?s=${searchTerm}&post_type=wp-manga&m_orderby=rating`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
@@ -184,7 +162,7 @@ const searchScraper = async (req, res) => {
|
||||
novelUrl = novelUrl.replace(`${baseUrl}/`, "");
|
||||
|
||||
const novel = {
|
||||
extensionId: 1,
|
||||
sourceId,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
@@ -193,12 +171,14 @@ const searchScraper = async (req, res) => {
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
res.json(novels);
|
||||
return novels;
|
||||
};
|
||||
|
||||
module.exports = boxNovelScraper = {
|
||||
novelsScraper,
|
||||
novelScraper,
|
||||
chapterScraper,
|
||||
searchScraper,
|
||||
const BoxNovelScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default BoxNovelScraper;
|
||||
@@ -1,14 +0,0 @@
|
||||
const express = require("express");
|
||||
const boxNovelScraper = require("./BoxNovelScraper");
|
||||
|
||||
const router = express.Router();
|
||||
|
||||
router.get("/novels/:pageNo", boxNovelScraper.novelsScraper);
|
||||
|
||||
router.get("/novel/:novelUrl", boxNovelScraper.novelScraper);
|
||||
|
||||
router.get("/novel/:novelUrl/:chapterUrl", boxNovelScraper.chapterScraper);
|
||||
|
||||
router.get("/search/", boxNovelScraper.searchScraper);
|
||||
|
||||
module.exports = router;
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 106 KiB |
@@ -0,0 +1,204 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
|
||||
const sourceId = 27;
|
||||
|
||||
const sourceName = "Comrade Mao";
|
||||
|
||||
const baseUrl = "https://comrademao.com/";
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
let url = baseUrl + "novel/page/" + page;
|
||||
const totalPages = 111;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$("div.container")
|
||||
.find("section")
|
||||
.each(function () {
|
||||
const novelName = $(this)
|
||||
.find("div.columns > div:nth-child(2) > a")
|
||||
.text();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
|
||||
let novelUrl = $(this)
|
||||
.find("div.columns > div:nth-child(2) > a")
|
||||
.attr("href");
|
||||
novelUrl = novelUrl.replace(baseUrl + "novel/", "");
|
||||
|
||||
const novel = {
|
||||
sourceId,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return { novels, totalPages };
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = baseUrl + "novel/" + novelUrl;
|
||||
console.log(url);
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = { sourceId, sourceName, url, novelUrl };
|
||||
|
||||
novel.novelName = $(
|
||||
"#NovelInfo > div > div.column.is-one-third.has-text-centered > p"
|
||||
)
|
||||
.text()
|
||||
.trim();
|
||||
|
||||
novel.novelCover = $("img.attachment-post-thumbnail").attr("src");
|
||||
|
||||
novel.summary = $("#NovelInfo > div > div:nth-child(2) > p").text().trim();
|
||||
|
||||
novel.genre = $("#NovelInfo > p:nth-child(2)")
|
||||
.text()
|
||||
.replace(/Genre(s):|\s/g, "");
|
||||
|
||||
novel.status = $("#NovelInfo > p:nth-child(5)")
|
||||
.text()
|
||||
.replace(/Status:|\s/g, "")
|
||||
.trim();
|
||||
|
||||
novel.author = $("#NovelInfo > p:nth-child(4)")
|
||||
.text()
|
||||
.replace(/Publisher:|\s/g, "")
|
||||
.trim();
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
// $("table > tbody")
|
||||
// .find("tr")
|
||||
// .each(function () {
|
||||
// const releaseDate = $(this).find("span").text();
|
||||
|
||||
// const chapterName = $(this).find("a").text();
|
||||
// const chapterUrl = $(this).find("a").attr("href").split("/")[5];
|
||||
|
||||
// novelChapters.push({
|
||||
// chapterName,
|
||||
// chapterUrl,
|
||||
// releaseDate,
|
||||
// });
|
||||
// });
|
||||
|
||||
// $("small").remove();
|
||||
|
||||
let chapterUrlPrefix = $("table > tbody")
|
||||
.find("tr")
|
||||
.first()
|
||||
.find("a")
|
||||
.attr("href")
|
||||
.slice(0, -1)
|
||||
.split("/")
|
||||
.pop()
|
||||
.split("-");
|
||||
|
||||
chapterUrlPrefix.pop();
|
||||
|
||||
chapterUrlPrefix = chapterUrlPrefix.join("-").toLowerCase();
|
||||
|
||||
let latestChapter = $("table > tbody")
|
||||
.find("tr")
|
||||
.first()
|
||||
.find("a")
|
||||
.text()
|
||||
.replace(/^\D+/g, "");
|
||||
|
||||
for (let i = 1; i <= latestChapter; i++) {
|
||||
const chapterName = "Chapter " + i;
|
||||
const releaseDate = null;
|
||||
const chapterUrl = chapterUrlPrefix + "-" + i + "/";
|
||||
|
||||
const chapter = {
|
||||
chapterName,
|
||||
releaseDate,
|
||||
chapterUrl,
|
||||
};
|
||||
|
||||
novelChapters.push(chapter);
|
||||
}
|
||||
|
||||
novel.chapters = novelChapters;
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = `${baseUrl}mtl/${novelUrl}/${chapterUrl}`;
|
||||
|
||||
console.log(url);
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let chapterName = "";
|
||||
|
||||
let chapterText = $("#content").html();
|
||||
|
||||
const chapter = {
|
||||
sourceId,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const url = baseUrl + "?s=" + searchTerm + "&post_type=novel";
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".newbox")
|
||||
.find("li")
|
||||
.each(function () {
|
||||
const novelName = $(this).find("h3").text();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
|
||||
let novelUrl = $(this).find("a").attr("href");
|
||||
novelUrl = novelUrl.replace(baseUrl + "novel/", "");
|
||||
|
||||
const novel = {
|
||||
sourceId,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const ComradeMaoScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default ComradeMaoScraper;
|
||||
@@ -1,14 +0,0 @@
|
||||
const express = require("express");
|
||||
const ComradeMaoScraper = require("./ComradeMaoScraper");
|
||||
|
||||
const router = express.Router();
|
||||
|
||||
router.get("/novels/", ComradeMaoScraper.novelsScraper);
|
||||
|
||||
router.get("/novel/:novelUrl", ComradeMaoScraper.novelScraper);
|
||||
|
||||
router.get("/novel/:novelUrl/:chapterUrl", ComradeMaoScraper.chapterScraper);
|
||||
|
||||
router.get("/search/", ComradeMaoScraper.searchScraper);
|
||||
|
||||
module.exports = router;
|
||||
@@ -1,211 +0,0 @@
|
||||
const cheerio = require("cheerio");
|
||||
const fetch = require("node-fetch");
|
||||
const { htmlToText } = require("html-to-text");
|
||||
const { parseHtml } = require("../../helper");
|
||||
|
||||
const baseUrl = "https://comrademao.com/";
|
||||
|
||||
const novelsScraper = async (req, res) => {
|
||||
let url = baseUrl;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$("article.blog-article").each(function (result) {
|
||||
const novelName = $(this).find("h5 > a").text();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
|
||||
let novelUrl = $(this).find("h5 > a").attr("href");
|
||||
novelUrl = novelUrl.replace(baseUrl + "novel/", "");
|
||||
|
||||
const novel = {
|
||||
extensionId: 27,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
res.json(novels);
|
||||
};
|
||||
|
||||
const novelScraper = async (req, res) => {
|
||||
const novelUrl = req.params.novelUrl;
|
||||
const url = baseUrl + "novel/" + novelUrl;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.extensionId = 27;
|
||||
|
||||
novel.sourceName = "Comrade Mao";
|
||||
|
||||
novel.sourceUrl = url;
|
||||
|
||||
novel.novelUrl = novelUrl + "/";
|
||||
|
||||
function getNovelName(y) {
|
||||
return y
|
||||
.replace("-", " ")
|
||||
.replace(/(?:^|\s)\S/g, (a) => a.toUpperCase());
|
||||
}
|
||||
|
||||
novelName = getNovelName(novelUrl);
|
||||
novel.novelName = novelName;
|
||||
|
||||
novel.novelCover = $("#thumbnail > img").attr("src");
|
||||
|
||||
let novelSummary = $("div#Description").html();
|
||||
novel.novelSummary = htmlToText(novelSummary);
|
||||
|
||||
novel["Genre(s)"] = $("div#Genre")
|
||||
.text()
|
||||
.replace(/Genre:|\s/g, "");
|
||||
|
||||
novel.Status = $("div#Status")
|
||||
.text()
|
||||
.replace(/Status:|\s/g, "");
|
||||
|
||||
novel["Author(s)"] = $("div#Publisher")
|
||||
.text()
|
||||
.replace(/Publisher:|\s/g, "");
|
||||
|
||||
// novel['Genre(s)'] = $("div#Genre").
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
let chapterUrlPrefix = $("tbody > tr")
|
||||
.first()
|
||||
.next()
|
||||
.find("td > a")
|
||||
.text()
|
||||
.split(" ");
|
||||
|
||||
chapterUrlPrefix.pop();
|
||||
|
||||
chapterUrlPrefix = chapterUrlPrefix.join("-").toLowerCase();
|
||||
|
||||
const latestChapter = $("tbody > tr")
|
||||
.first()
|
||||
.next()
|
||||
.find("td > a")
|
||||
.text()
|
||||
.replace(/^\D+/g, "");
|
||||
|
||||
for (let i = 1; i <= latestChapter; i++) {
|
||||
const chapterName = "Chapter " + i;
|
||||
const releaseDate = null;
|
||||
const chapterUrl = chapterUrlPrefix + "-" + i + "/";
|
||||
|
||||
const chapter = {
|
||||
chapterName,
|
||||
releaseDate,
|
||||
chapterUrl,
|
||||
};
|
||||
|
||||
novelChapters.push(chapter);
|
||||
}
|
||||
|
||||
novel.novelChapters = novelChapters;
|
||||
|
||||
res.json(novel);
|
||||
};
|
||||
|
||||
const chapterScraper = async (req, res) => {
|
||||
let novelUrl = req.params.novelUrl;
|
||||
let chapterUrl = req.params.chapterUrl;
|
||||
|
||||
const url = `${baseUrl}mtl/${novelUrl}/${chapterUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let chapterName = $(".entry-title").text();
|
||||
|
||||
let chapterText = $(".entry-content").html();
|
||||
chapterText = parseHtml(chapterText);
|
||||
|
||||
let nextChapter = null;
|
||||
|
||||
if ($(".nav-next").find("a").length) {
|
||||
nextChapter = $(".nav-next")
|
||||
.find("a")
|
||||
.attr("href")
|
||||
.replace(baseUrl + "mtl/" + novelUrl + "/", "");
|
||||
}
|
||||
|
||||
let prevChapter = null;
|
||||
|
||||
if ($(".nav-previous").find("a").length) {
|
||||
prevChapter = $(".nav-previous")
|
||||
.find("a")
|
||||
.attr("href")
|
||||
.replace(baseUrl + "mtl/" + novelUrl + "/", "");
|
||||
}
|
||||
|
||||
novelUrl = novelUrl + "/";
|
||||
chapterUrl = chapterUrl + "/";
|
||||
|
||||
const chapter = {
|
||||
extensionId: 27,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
nextChapter,
|
||||
prevChapter,
|
||||
};
|
||||
|
||||
res.json(chapter);
|
||||
};
|
||||
|
||||
const searchScraper = async (req, res) => {
|
||||
const searchTerm = req.query.s;
|
||||
|
||||
const url = baseUrl + "?s=" + searchTerm + "&post_type=novel";
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$("article.blog-article").each(function (result) {
|
||||
const novelName = $(this).find("h5 > a").text();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
|
||||
let novelUrl = $(this).find("h5 > a").attr("href");
|
||||
novelUrl = novelUrl.replace(baseUrl + "novel/", "");
|
||||
|
||||
const novel = {
|
||||
extensionId: 27,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
res.json(novels);
|
||||
};
|
||||
|
||||
module.exports = vipNovelScraper = {
|
||||
novelsScraper,
|
||||
novelScraper,
|
||||
chapterScraper,
|
||||
searchScraper,
|
||||
};
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 42 KiB |
@@ -0,0 +1,152 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
|
||||
const sourceId = 70;
|
||||
|
||||
const sourceName = "Divine Dao Library";
|
||||
|
||||
const baseUrl = "https://www.divinedaolibrary.com/";
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
let url = baseUrl + "novels";
|
||||
const totalPages = 1;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$("#main")
|
||||
.find("li")
|
||||
.each(function () {
|
||||
const novelName = $(this).find("a").text();
|
||||
const novelCover = null;
|
||||
const novelUrl = $(this).find(" a").attr("href");
|
||||
|
||||
const novel = {
|
||||
sourceId,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return { novels, totalPages };
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = novelUrl;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = { sourceId, sourceName, url, novelUrl };
|
||||
|
||||
novel.novelName = $("h1.entry-title").text().trim();
|
||||
|
||||
novel.novelCover =
|
||||
$(".entry-content").find("img").attr("src") ||
|
||||
"https://github.com/LNReader/lnreader-sources/blob/main/src/coverNotAvailable.jpg?raw=true";
|
||||
|
||||
novel.summary = $("#main > article > div > p:nth-child(6)").text().trim();
|
||||
|
||||
novel.genre = null;
|
||||
|
||||
novel.status = null;
|
||||
|
||||
novel.author = $("#main > article > div > h3:nth-child(2)")
|
||||
.text()
|
||||
.replace(/Author:/g, "")
|
||||
.trim();
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
$("#main")
|
||||
.find("li > span > a")
|
||||
.each(function () {
|
||||
const chapterName = $(this).text().trim();
|
||||
const releaseDate = null;
|
||||
const chapterUrl = $(this).attr("href");
|
||||
|
||||
novelChapters.push({ chapterName, releaseDate, chapterUrl });
|
||||
});
|
||||
|
||||
novel.chapters = novelChapters;
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = chapterUrl;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let chapterName = $(".entry-title").text().trim();
|
||||
|
||||
let chapterText = $(".entry-content").html();
|
||||
|
||||
if (!chapterText) {
|
||||
chapterText = $(".page-header").html();
|
||||
}
|
||||
|
||||
const chapter = {
|
||||
sourceId,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
let url = baseUrl + "novels";
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$("#main")
|
||||
.find("li")
|
||||
.each(function () {
|
||||
const novelName = $(this).find("a").text();
|
||||
const novelCover = null;
|
||||
const novelUrl = $(this).find(" a").attr("href");
|
||||
|
||||
const novel = {
|
||||
sourceId,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
novels = novels.filter((novel) =>
|
||||
novel.novelName.toLowerCase().includes(searchTerm.toLowerCase())
|
||||
);
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const DivineDaoLibraryScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default DivineDaoLibraryScraper;
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 40 KiB |
@@ -1,13 +1,14 @@
|
||||
const cheerio = require("cheerio");
|
||||
const { scraper } = require("../../helper");
|
||||
import cheerio from "react-native-cheerio";
|
||||
|
||||
const baseUrl = "https://fastnovel.net";
|
||||
const searchUrl = `https://fastnovel.net/search/`;
|
||||
|
||||
const novelsScraper = async (req, res) => {
|
||||
const url = `${baseUrl}/list/most-popular.html`;
|
||||
const popularNovels = async (page) => {
|
||||
let totalPages = 39;
|
||||
const url = `${baseUrl}/list/most-popular.html?page=${page}`;
|
||||
|
||||
const body = await scraper(url);
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
@@ -19,7 +20,7 @@ const novelsScraper = async (req, res) => {
|
||||
const novelUrl = $(this).find("a").attr("href").substring(1);
|
||||
|
||||
const novel = {
|
||||
extensionId: 3,
|
||||
sourceId: 3,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
@@ -28,50 +29,59 @@ const novelsScraper = async (req, res) => {
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
res.json(novels);
|
||||
return { totalPages, novels };
|
||||
};
|
||||
|
||||
const novelScraper = async (req, res) => {
|
||||
const novelUrl = req.params.novelUrl;
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = `${baseUrl}/${novelUrl}/`;
|
||||
|
||||
const body = await scraper(url);
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.extensionId = 3;
|
||||
novel.sourceId = 3;
|
||||
|
||||
novel.sourceName = "FastNovel";
|
||||
|
||||
novel.sourceUrl = url;
|
||||
novel.url = url;
|
||||
|
||||
novel.novelUrl = novelUrl + "/";
|
||||
novel.novelUrl = novelUrl;
|
||||
|
||||
novel.novelName = $("h1").text();
|
||||
|
||||
novel.novelCover = $(".book-cover").attr("data-original");
|
||||
|
||||
novel.novelSummary = $("div.film-content > p").text();
|
||||
novel.summary = $("div.film-content > p").text();
|
||||
|
||||
novel["Author(s)"] = $("li > label")
|
||||
novel.author = $("li > label")
|
||||
.filter(function () {
|
||||
return $(this).text().trim() === "Author:";
|
||||
})
|
||||
.next()
|
||||
.text();
|
||||
|
||||
novel["Genre(s)"] = $("li")
|
||||
novel.genre = $("li")
|
||||
.filter(function () {
|
||||
return $(this).find("label").text().trim() === "Genre:";
|
||||
})
|
||||
.text()
|
||||
.replace("Genre:", "");
|
||||
|
||||
novel["Artist(s)"] = null;
|
||||
novel.artist = null;
|
||||
|
||||
novel.Status = null;
|
||||
novel.status = null;
|
||||
|
||||
novel.status = $("li")
|
||||
.filter(function () {
|
||||
return $(this).find("label").text().trim() === "Status:";
|
||||
})
|
||||
.text()
|
||||
.includes("Completed")
|
||||
? "Completed"
|
||||
: "Ongoing";
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
@@ -90,64 +100,38 @@ const novelScraper = async (req, res) => {
|
||||
novelChapters.push(novel);
|
||||
});
|
||||
|
||||
novel.novelChapters = novelChapters;
|
||||
novel.chapters = novelChapters;
|
||||
|
||||
res.json(novel);
|
||||
return novel;
|
||||
};
|
||||
|
||||
const chapterScraper = async (req, res) => {
|
||||
let novelUrl = req.params.novelUrl;
|
||||
let chapterUrl = req.params.chapterUrl;
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = `${baseUrl}/${novelUrl}/${chapterUrl}`;
|
||||
|
||||
const body = await scraper(url);
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
const chapterName = $(".episode-name").text();
|
||||
const chapterText = $("#chapter-body").text();
|
||||
|
||||
let nextChapter = null;
|
||||
|
||||
nextChapter = $("div.btn-group > a")
|
||||
.filter(function () {
|
||||
return $(this).text().trim() === "Next Chapter";
|
||||
})
|
||||
.attr("href")
|
||||
.replace("/", "");
|
||||
|
||||
let prevChapter = null;
|
||||
|
||||
prevChapter = $("div.btn-group > a")
|
||||
.filter(function () {
|
||||
return $(this).text().trim() === "Prev Chapter";
|
||||
})
|
||||
.attr("href")
|
||||
.replace("/", "");
|
||||
|
||||
novelUrl = novelUrl + "/";
|
||||
chapterUrl = chapterUrl + "/";
|
||||
|
||||
let chapterText = $("#chapter-body").html();
|
||||
const chapter = {
|
||||
extensionId: 3,
|
||||
sourceId: 3,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
nextChapter,
|
||||
prevChapter,
|
||||
};
|
||||
|
||||
res.json(chapter);
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchScraper = async (req, res) => {
|
||||
const searchTerm = req.query.s;
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const url = `${searchUrl}${searchTerm}`;
|
||||
|
||||
const body = await scraper(url);
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
@@ -159,7 +143,7 @@ const searchScraper = async (req, res) => {
|
||||
const novelUrl = $(this).find("a").attr("href").substring(1);
|
||||
|
||||
const novel = {
|
||||
extensionId: 3,
|
||||
sourceId: 3,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
@@ -168,12 +152,14 @@ const searchScraper = async (req, res) => {
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
res.json(novels);
|
||||
return novels;
|
||||
};
|
||||
|
||||
module.exports = fastNovelScraper = {
|
||||
novelsScraper,
|
||||
novelScraper,
|
||||
chapterScraper,
|
||||
searchScraper,
|
||||
const fastNovelScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default fastNovelScraper;
|
||||
@@ -1,14 +0,0 @@
|
||||
const express = require("express");
|
||||
const fastNovelScraper = require("./FastNovelScraper");
|
||||
|
||||
const router = express.Router();
|
||||
|
||||
router.get("/novels/", fastNovelScraper.novelsScraper);
|
||||
|
||||
router.get("/novel/:novelUrl", fastNovelScraper.novelScraper);
|
||||
|
||||
router.get("/novel/:novelUrl/:chapterUrl", fastNovelScraper.chapterScraper);
|
||||
|
||||
router.get("/search/", fastNovelScraper.searchScraper);
|
||||
|
||||
module.exports = router;
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 75 KiB |
@@ -1,12 +1,10 @@
|
||||
const cheerio = require("cheerio");
|
||||
const FormData = require("form-data");
|
||||
const fetch = require("node-fetch");
|
||||
const { parseHtml } = require("../../helper");
|
||||
import cheerio from "react-native-cheerio";
|
||||
|
||||
const baseUrl = "https://foxaholic.com/";
|
||||
|
||||
const novelsScraper = async (req, res) => {
|
||||
let url = baseUrl + "novel/?m_orderby=rating";
|
||||
const popularNovels = async (page) => {
|
||||
let totalPages = 5;
|
||||
let url = baseUrl + "novel/page/" + page + "/?m_orderby=rating";
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
@@ -23,7 +21,7 @@ const novelsScraper = async (req, res) => {
|
||||
novelUrl += "/";
|
||||
|
||||
const novel = {
|
||||
extensionId: 22,
|
||||
sourceId: 22,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
@@ -32,12 +30,11 @@ const novelsScraper = async (req, res) => {
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
res.json(novels);
|
||||
return { totalPages, novels };
|
||||
};
|
||||
|
||||
const novelScraper = async (req, res) => {
|
||||
const novelUrl = req.params.novelUrl;
|
||||
const url = `${baseUrl}novel/${novelUrl}/`;
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = `${baseUrl}novel/${novelUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
@@ -46,13 +43,13 @@ const novelScraper = async (req, res) => {
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.extensionId = 22;
|
||||
novel.sourceId = 22;
|
||||
|
||||
novel.sourceName = "Foxaholic";
|
||||
|
||||
novel.sourceUrl = url;
|
||||
novel.url = url;
|
||||
|
||||
novel.novelUrl = `${novelUrl}/`;
|
||||
novel.novelUrl = novelUrl;
|
||||
|
||||
novel.novelName = $(".post-title > h1")
|
||||
.text()
|
||||
@@ -73,22 +70,24 @@ const novelScraper = async (req, res) => {
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
novel[detailName] = detail;
|
||||
switch (detailName) {
|
||||
case "Genre":
|
||||
novel.genre = detail.trim().replace(/[\t\n]/g, ",");
|
||||
break;
|
||||
case "Author":
|
||||
novel.author = detail.trim();
|
||||
break;
|
||||
case "Novel":
|
||||
novel.status = detail.trim();
|
||||
break;
|
||||
}
|
||||
});
|
||||
|
||||
novel["Genre(s)"] = novel.Genre.replace(/, /g, ",");
|
||||
novel["Author(s)"] = novel.Author;
|
||||
novel.Status = null;
|
||||
novel["Artist(s)"] = null;
|
||||
|
||||
delete novel.Genre;
|
||||
delete novel.Author;
|
||||
|
||||
$(".description-summary > div.summary__content").find("em").remove();
|
||||
|
||||
novel.novelSummary = $(".description-summary > div.summary__content")
|
||||
novel.summary = $(".description-summary > div.summary__content")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "");
|
||||
.trim();
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
@@ -127,25 +126,13 @@ const novelScraper = async (req, res) => {
|
||||
novelChapters.push({ chapterName, releaseDate, chapterUrl });
|
||||
});
|
||||
|
||||
novel.novelChapters = novelChapters.reverse();
|
||||
novel.chapters = novelChapters.reverse();
|
||||
|
||||
res.json(novel);
|
||||
return novel;
|
||||
};
|
||||
|
||||
const chapterScraper = async (req, res) => {
|
||||
let novelUrl = req.params.novelUrl;
|
||||
let chapterUrl = req.params.chapterUrl;
|
||||
let volumeUrl = req.params.volumeUrl;
|
||||
|
||||
let optionalUrl;
|
||||
|
||||
if (volumeUrl) {
|
||||
optionalUrl = volumeUrl + "/";
|
||||
} else {
|
||||
optionalUrl = "";
|
||||
}
|
||||
|
||||
const url = `${baseUrl}novel/${novelUrl}/${optionalUrl}${chapterUrl}/`;
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = `${baseUrl}novel/${novelUrl}${chapterUrl}/`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
@@ -154,45 +141,18 @@ const chapterScraper = async (req, res) => {
|
||||
|
||||
const chapterName = $("h1#chapter-heading").text();
|
||||
let chapterText = $(".reading-content").html();
|
||||
chapterText = parseHtml(chapterText);
|
||||
|
||||
let nextChapter = null;
|
||||
|
||||
if ($(".nav-next").length) {
|
||||
nextChapter = $(".nav-next").find("a").attr("href").split("/");
|
||||
nextChapter[6]
|
||||
? (nextChapter = nextChapter[5] + "/" + nextChapter[6])
|
||||
: (nextChapter = nextChapter[5]);
|
||||
}
|
||||
|
||||
let prevChapter = null;
|
||||
|
||||
if ($(".nav-previous").length) {
|
||||
prevChapter = $(".nav-previous").find("a").attr("href").split("/");
|
||||
prevChapter[6]
|
||||
? (prevChapter = prevChapter[5] + "/" + prevChapter[6])
|
||||
: (prevChapter = prevChapter[5]);
|
||||
}
|
||||
|
||||
novelUrl += "/";
|
||||
chapterUrl = optionalUrl + chapterUrl;
|
||||
|
||||
const chapter = {
|
||||
extensionId: 22,
|
||||
sourceId: 22,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
nextChapter,
|
||||
prevChapter,
|
||||
};
|
||||
|
||||
res.json(chapter);
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchScraper = async (req, res) => {
|
||||
const searchTerm = req.query.s;
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const url = `${baseUrl}?s=${searchTerm}&post_type=wp-manga`;
|
||||
|
||||
const result = await fetch(url);
|
||||
@@ -209,7 +169,7 @@ const searchScraper = async (req, res) => {
|
||||
let novelUrl = $(this).find(".h4 > a").attr("href").split("/")[4];
|
||||
novelUrl += "/";
|
||||
const novel = {
|
||||
extensionId: 22,
|
||||
sourceId: 22,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
@@ -218,12 +178,14 @@ const searchScraper = async (req, res) => {
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
res.json(novels);
|
||||
return novels;
|
||||
};
|
||||
|
||||
module.exports = novelTrenchScraper = {
|
||||
novelsScraper,
|
||||
novelScraper,
|
||||
chapterScraper,
|
||||
searchScraper,
|
||||
const FoxaHolicScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default FoxaHolicScraper;
|
||||
@@ -1,16 +0,0 @@
|
||||
const express = require("express");
|
||||
const FoxaholicScraper = require("./FoxaholicScraper");
|
||||
|
||||
const router = express.Router();
|
||||
|
||||
router.get("/novels/", FoxaholicScraper.novelsScraper);
|
||||
|
||||
router.get("/novel/:novelUrl", FoxaholicScraper.novelScraper);
|
||||
|
||||
router.get(
|
||||
"/novel/:novelUrl/:volumeUrl?/:chapterUrl",
|
||||
FoxaholicScraper.chapterScraper
|
||||
);
|
||||
router.get("/search/", FoxaholicScraper.searchScraper);
|
||||
|
||||
module.exports = router;
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 73 KiB |
@@ -0,0 +1,183 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
|
||||
const baseUrl = "https://freewebnovel.com/";
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
let totalPages = 30;
|
||||
let url = baseUrl + "completed-novel/" + page;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".li-row").each(function (result) {
|
||||
const novelName = $(this).find(".tit").text();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
|
||||
let novelUrl = $(this)
|
||||
.find("h3 > a")
|
||||
.attr("href")
|
||||
.replace(".html", "")
|
||||
.slice(1);
|
||||
|
||||
const novel = {
|
||||
sourceId: 13,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return { totalPages, novels };
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = `${baseUrl}${novelUrl.replace("/", "")}.html`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.sourceId = 13;
|
||||
|
||||
novel.sourceName = "FreeWebNovel";
|
||||
|
||||
novel.url = url;
|
||||
|
||||
novel.novelUrl = novelUrl;
|
||||
|
||||
novel.novelName = $("h1.tit").text();
|
||||
|
||||
novel.novelCover = $(".pic > img").attr("src");
|
||||
|
||||
novel.genre = $("[title=Genre]")
|
||||
.next()
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "");
|
||||
|
||||
novel.author = $("[title=Author]")
|
||||
.next()
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "");
|
||||
|
||||
novel.artist = null;
|
||||
|
||||
novel.status = $("[title=Status]")
|
||||
.next()
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "");
|
||||
|
||||
let novelSummary = $(".inner").text().trim();
|
||||
novel.summary = novelSummary;
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
let latestChapter;
|
||||
|
||||
$("h3.tit").each(function (res) {
|
||||
if ($(this).find("a").text() === novel.novelName) {
|
||||
latestChapter = $(this).next().find("span.s3").text().match(/\d+/);
|
||||
}
|
||||
});
|
||||
|
||||
latestChapter = latestChapter[0];
|
||||
|
||||
for (let i = 1; i <= parseInt(latestChapter); i++) {
|
||||
const chapterName = "Chapter " + i;
|
||||
|
||||
const releaseDate = null;
|
||||
|
||||
const chapterUrl = "chapter-" + i;
|
||||
|
||||
const chapter = { chapterName, releaseDate, chapterUrl };
|
||||
|
||||
novelChapters.push(chapter);
|
||||
}
|
||||
|
||||
novel.chapters = novelChapters;
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
let novelId = novelUrl.replace("/", "");
|
||||
|
||||
const url = `${baseUrl}${novelId}/${chapterUrl}.html`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let chapterName = $("h1.tit").text();
|
||||
|
||||
let chapterText = $("div.txt").html();
|
||||
|
||||
const chapter = {
|
||||
sourceId: 13,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const url = baseUrl + "search/";
|
||||
|
||||
const formData = new FormData();
|
||||
formData.append("searchkey", searchTerm);
|
||||
|
||||
const result = await fetch(url, {
|
||||
method: "POST",
|
||||
body: formData,
|
||||
});
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".li-row > .li > .con").each(function (result) {
|
||||
const novelName = $(this).find(".tit").text();
|
||||
const novelCover = $(this).find(".pic > a > img").attr("data-cfsrc");
|
||||
|
||||
let novelUrl = $(this)
|
||||
.find("h3 > a")
|
||||
.attr("href")
|
||||
.replace(".html", "")
|
||||
.slice(1);
|
||||
|
||||
novelUrl += "/";
|
||||
|
||||
const novel = {
|
||||
sourceId: 13,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const FreeWebNovelScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default FreeWebNovelScraper;
|
||||
@@ -1,14 +0,0 @@
|
||||
const express = require("express");
|
||||
const freeWebNovelScraper = require("./FreeWebNovelScraper");
|
||||
|
||||
const router = express.Router();
|
||||
|
||||
router.get("/novels/", freeWebNovelScraper.novelsScraper);
|
||||
|
||||
router.get("/novel/:novelUrl", freeWebNovelScraper.novelScraper);
|
||||
|
||||
router.get("/novel/:novelUrl/:chapterUrl", freeWebNovelScraper.chapterScraper);
|
||||
|
||||
router.get("/search/", freeWebNovelScraper.searchScraper);
|
||||
|
||||
module.exports = router;
|
||||
@@ -1,218 +0,0 @@
|
||||
const cheerio = require("cheerio");
|
||||
const fetch = require("node-fetch");
|
||||
const request = require("request");
|
||||
const { htmlToText } = require("html-to-text");
|
||||
const { parseHtml } = require("../../helper");
|
||||
|
||||
const baseUrl = "https://freewebnovel.com/";
|
||||
|
||||
const novelsScraper = async (req, res) => {
|
||||
let url = baseUrl + "most-popular-novel/";
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".li-row").each(function (result) {
|
||||
const novelName = $(this).find(".tit").text();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
|
||||
let novelUrl = $(this)
|
||||
.find("h3 > a")
|
||||
.attr("href")
|
||||
.replace(".html", "")
|
||||
.slice(1);
|
||||
|
||||
novelUrl += "/";
|
||||
|
||||
const novel = {
|
||||
extensionId: 13,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
res.json(novels);
|
||||
};
|
||||
|
||||
const novelScraper = async (req, res) => {
|
||||
const novelUrl = req.params.novelUrl;
|
||||
const url = `${baseUrl}${novelUrl}.html`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.extensionId = 13;
|
||||
|
||||
novel.sourceName = "FreeWebNovel";
|
||||
|
||||
novel.sourceUrl = url;
|
||||
|
||||
novel.novelUrl = `${novelUrl.replace(".html", "")}/`;
|
||||
|
||||
novel.novelName = $("h1.tit").text();
|
||||
|
||||
novel.novelCover = $(".pic > img").attr("src");
|
||||
|
||||
novel["Genre(s)"] = $("[title=Genre]")
|
||||
.next()
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "");
|
||||
|
||||
novel["Author(s)"] = $("[title=Author]")
|
||||
.next()
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "");
|
||||
|
||||
novel["Artist(s)"] = null;
|
||||
|
||||
novel.Status = $("[title=Status]")
|
||||
.next()
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "");
|
||||
|
||||
novel.Alternative = $("[title='Alternative names']")
|
||||
.next()
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "");
|
||||
|
||||
let novelSummary = $(".inner").html();
|
||||
novel.novelSummary = htmlToText(novelSummary);
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
let latestChapter;
|
||||
|
||||
$("h3.tit").each(function (res) {
|
||||
if ($(this).find("a").text() === novel.novelName) {
|
||||
// console.log($(this).find("a").text());
|
||||
// console.log(novel.novelName);
|
||||
latestChapter = $(this).next().find("span.s3").text().match(/\d+/);
|
||||
}
|
||||
});
|
||||
|
||||
latestChapter = latestChapter[0];
|
||||
|
||||
for (let i = 1; i <= parseInt(latestChapter); i++) {
|
||||
const chapterName = "Chapter " + i;
|
||||
|
||||
const releaseDate = null;
|
||||
|
||||
const chapterUrl = "chapter-" + i;
|
||||
|
||||
const chapter = { chapterName, releaseDate, chapterUrl };
|
||||
|
||||
novelChapters.push(chapter);
|
||||
}
|
||||
|
||||
novel.novelChapters = novelChapters;
|
||||
|
||||
res.json(novel);
|
||||
};
|
||||
|
||||
const chapterScraper = async (req, res) => {
|
||||
let novelUrl = req.params.novelUrl;
|
||||
let chapterUrl = req.params.chapterUrl;
|
||||
|
||||
const url = `${baseUrl}${novelUrl}/${chapterUrl}.html`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let chapterName = $("h1.tit").text();
|
||||
|
||||
let chapterText = $("div.txt").html();
|
||||
chapterText = parseHtml(chapterText);
|
||||
|
||||
let nextChapter = null;
|
||||
|
||||
if ($("[title='Read Next chapter']").length) {
|
||||
nextChapter = $("[title='Read Next chapter']")
|
||||
.attr("href")
|
||||
.replace("/" + novelUrl + "/", "")
|
||||
.replace(".html", "");
|
||||
}
|
||||
|
||||
let prevChapter = null;
|
||||
|
||||
if ($("[title='Read Privious Chapter']").length) {
|
||||
prevChapter = $("[title='Read Privious Chapter']")
|
||||
.attr("href")
|
||||
.replace("/" + novelUrl + "/", "")
|
||||
.replace(".html", "");
|
||||
}
|
||||
novelUrl = novelUrl + "/";
|
||||
chapterUrl = chapterUrl;
|
||||
|
||||
const chapter = {
|
||||
extensionId: 13,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
nextChapter,
|
||||
prevChapter,
|
||||
};
|
||||
|
||||
res.json(chapter);
|
||||
};
|
||||
|
||||
const searchScraper = async (req, res) => {
|
||||
const searchTerm = req.query.s;
|
||||
|
||||
const url = baseUrl + "search/";
|
||||
|
||||
request.post(
|
||||
{ url, form: { searchkey: searchTerm } },
|
||||
(err, response, body) => {
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".li-row > .li > .con").each(function (result) {
|
||||
const novelName = $(this).find(".tit").text();
|
||||
const novelCover = $(this)
|
||||
.find(".pic > a > img")
|
||||
.attr("data-cfsrc");
|
||||
|
||||
let novelUrl = $(this)
|
||||
.find("h3 > a")
|
||||
.attr("href")
|
||||
.replace(".html", "")
|
||||
.slice(1);
|
||||
|
||||
novelUrl += "/";
|
||||
|
||||
const novel = {
|
||||
extensionId: 13,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
res.json(novels);
|
||||
}
|
||||
);
|
||||
};
|
||||
|
||||
module.exports = vipNovelScraper = {
|
||||
novelsScraper,
|
||||
novelScraper,
|
||||
chapterScraper,
|
||||
searchScraper,
|
||||
};
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 80 KiB |
@@ -1,12 +1,13 @@
|
||||
const cheerio = require("cheerio");
|
||||
const fetch = require("node-fetch");
|
||||
const { parseHtml } = require("../../helper");
|
||||
import cheerio from "react-native-cheerio";
|
||||
|
||||
const baseUrl = "https://jpmtl.com/";
|
||||
|
||||
const novelsScraper = async (req, res) => {
|
||||
const popularNovels = async (page) => {
|
||||
let totalPages = 157;
|
||||
let url =
|
||||
"https://jpmtl.com/v2/book/show/browse?query=&categories=&content_type=0&direction=0&page=2&limit=20&type=5&status=all&language=3&exclude_categories=";
|
||||
"https://jpmtl.com/v2/book/show/browse?query=&categories=&content_type=0&direction=0&page=" +
|
||||
page +
|
||||
"&limit=20&type=5&status=all&language=3&exclude_categories=";
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.json();
|
||||
@@ -18,16 +19,15 @@ const novelsScraper = async (req, res) => {
|
||||
const novelCover = item.cover;
|
||||
const novelUrl = item.id + "/";
|
||||
|
||||
const novel = { extensionId: 14, novelName, novelCover, novelUrl };
|
||||
const novel = { sourceId: 14, novelName, novelCover, novelUrl };
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
res.json(novels);
|
||||
return { totalPages, novels };
|
||||
};
|
||||
|
||||
const novelScraper = async (req, res) => {
|
||||
const novelUrl = req.params.novelUrl;
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = `${baseUrl}books/${novelUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
@@ -37,13 +37,13 @@ const novelScraper = async (req, res) => {
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.extensionId = 14;
|
||||
novel.sourceId = 14;
|
||||
|
||||
novel.sourceName = "JPMTL";
|
||||
|
||||
novel.sourceUrl = url;
|
||||
novel.url = url;
|
||||
|
||||
novel.novelUrl = `${novelUrl}/`;
|
||||
novel.novelUrl = novelUrl;
|
||||
|
||||
novel.novelName = $("h1.book-sidebar__title").text();
|
||||
|
||||
@@ -64,18 +64,28 @@ const novelScraper = async (req, res) => {
|
||||
novel[detailName] = detail;
|
||||
});
|
||||
|
||||
novel.novelSummary = $(".main-book__synopsis").text();
|
||||
novel.summary = $(".main-book__synopsis").text();
|
||||
|
||||
novel["Genre(s)"] = "";
|
||||
novel.genre = "";
|
||||
|
||||
$("a.main-book__category").each(function (result) {
|
||||
novel["Genre(s)"] += $(this).text();
|
||||
novel.genre += $(this).text();
|
||||
});
|
||||
|
||||
novel["Author(s)"] = $(".book-sidebar__author > .book-sidebar__info")
|
||||
function capitalizeFirstLetter(string) {
|
||||
return string.charAt(0).toUpperCase() + string.slice(1);
|
||||
}
|
||||
|
||||
const info = $(".book-sidebar__author > .book-sidebar__info")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
.trim()
|
||||
.split("・");
|
||||
|
||||
if (info) {
|
||||
novel.author = info[0];
|
||||
|
||||
novel.status = capitalizeFirstLetter(info[1]);
|
||||
}
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
@@ -91,7 +101,7 @@ const novelScraper = async (req, res) => {
|
||||
const chapterUrl = chapter.id;
|
||||
|
||||
const obj = {
|
||||
extensionId: 14,
|
||||
sourceId: 14,
|
||||
chapterName,
|
||||
releaseDate,
|
||||
chapterUrl,
|
||||
@@ -101,60 +111,35 @@ const novelScraper = async (req, res) => {
|
||||
});
|
||||
});
|
||||
|
||||
novel.novelChapters = novelChapters;
|
||||
novel.chapters = novelChapters;
|
||||
|
||||
res.json(novel);
|
||||
return novel;
|
||||
};
|
||||
|
||||
const chapterScraper = async (req, res) => {
|
||||
let novelUrl = req.params.novelUrl;
|
||||
let chapterUrl = req.params.chapterUrl;
|
||||
|
||||
const url = `${baseUrl}books/${novelUrl}/${chapterUrl}`;
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = `${baseUrl}books/${novelUrl}${chapterUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
console.log(url);
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
const chapterName = $(".chapter-content__title").text();
|
||||
let chapterText = $(".chapter-content__content").html();
|
||||
chapterText = parseHtml(chapterText);
|
||||
|
||||
let nextChapter = null;
|
||||
let prevChapter = null;
|
||||
|
||||
$(".chapter-wrapper__nav").each(function (result) {
|
||||
const chId = $(this)
|
||||
.attr("href")
|
||||
.replace("/books/" + novelUrl + "/", "");
|
||||
if (chId < chapterUrl) {
|
||||
prevChapter = chId;
|
||||
} else {
|
||||
nextChapter = chId;
|
||||
}
|
||||
});
|
||||
|
||||
novelUrl += "/";
|
||||
|
||||
const chapter = {
|
||||
extensionId: 14,
|
||||
sourceId: 14,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
nextChapter,
|
||||
prevChapter,
|
||||
};
|
||||
|
||||
res.json(chapter);
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchScraper = async (req, res) => {
|
||||
const searchTerm = req.query.s;
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const url = `https://jpmtl.com/v2/book/show/browse?query=${searchTerm}&categories=&content_type=2&direction=0&page=1&limit=20&type=5&status=all&language=3&exclude_categories=`;
|
||||
|
||||
const result = await fetch(url);
|
||||
@@ -167,17 +152,19 @@ const searchScraper = async (req, res) => {
|
||||
const novelCover = item.cover;
|
||||
const novelUrl = item.id + "/";
|
||||
|
||||
const novel = { extensionId: 14, novelName, novelCover, novelUrl };
|
||||
const novel = { sourceId: 14, novelName, novelCover, novelUrl };
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
res.json(novels);
|
||||
return novels;
|
||||
};
|
||||
|
||||
module.exports = novelTrenchScraper = {
|
||||
novelsScraper,
|
||||
novelScraper,
|
||||
chapterScraper,
|
||||
searchScraper,
|
||||
const JPMTLScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default JPMTLScraper;
|
||||
@@ -1,14 +0,0 @@
|
||||
const express = require("express");
|
||||
const JPMTLScraper = require("./JPMTLScraper");
|
||||
|
||||
const router = express.Router();
|
||||
|
||||
router.get("/novels/", JPMTLScraper.novelsScraper);
|
||||
|
||||
router.get("/novel/:novelUrl", JPMTLScraper.novelScraper);
|
||||
|
||||
router.get("/novel/:novelUrl/:chapterUrl", JPMTLScraper.chapterScraper);
|
||||
|
||||
router.get("/search/", JPMTLScraper.searchScraper);
|
||||
|
||||
module.exports = router;
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 48 KiB |
@@ -1,11 +1,12 @@
|
||||
const cheerio = require("cheerio");
|
||||
const fetch = require("node-fetch");
|
||||
const { parseHtml } = require("../../helper");
|
||||
import cheerio from "react-native-cheerio";
|
||||
|
||||
const baseUrl = "https://kisslightnovels.info/";
|
||||
|
||||
const novelsScraper = async (req, res) => {
|
||||
const result = await fetch(baseUrl);
|
||||
const popularNovels = async (page) => {
|
||||
let totalPages = 26;
|
||||
const url = `${baseUrl}light-novel/page/${page}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
@@ -20,7 +21,7 @@ const novelsScraper = async (req, res) => {
|
||||
novelUrl = novelUrl.replace(`${baseUrl}novel/`, "");
|
||||
|
||||
const novel = {
|
||||
extensionId: 11,
|
||||
sourceId: 11,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
@@ -29,12 +30,11 @@ const novelsScraper = async (req, res) => {
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
res.json(novels);
|
||||
return { totalPages, novels };
|
||||
};
|
||||
|
||||
const novelScraper = async (req, res) => {
|
||||
const novelUrl = req.params.novelUrl;
|
||||
const url = `${baseUrl}novel/${novelUrl}/`;
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = `${baseUrl}novel/${novelUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
@@ -43,13 +43,13 @@ const novelScraper = async (req, res) => {
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.extensionId = 11;
|
||||
novel.sourceId = 11;
|
||||
|
||||
novel.sourceName = "KissLightNovels";
|
||||
|
||||
novel.sourceUrl = url;
|
||||
novel.url = url;
|
||||
|
||||
novel.novelUrl = `${novelUrl}/`;
|
||||
novel.novelUrl = novelUrl;
|
||||
|
||||
novel.novelName = $(".post-title > h1")
|
||||
.text()
|
||||
@@ -70,12 +70,22 @@ const novelScraper = async (req, res) => {
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
novel[detailName] = detail;
|
||||
switch (detailName) {
|
||||
case "Genre(s)":
|
||||
novel.genre = detail.trim().replace(/[\t\n]/g, ",");
|
||||
break;
|
||||
case "Author(s)":
|
||||
novel.author = detail.trim();
|
||||
break;
|
||||
case "Artist(s)":
|
||||
novel.status = detail.trim();
|
||||
break;
|
||||
}
|
||||
});
|
||||
|
||||
$(".description-summary > div.summary__content").find("em").remove();
|
||||
|
||||
novel.novelSummary = $(".description-summary > div.summary__content")
|
||||
novel.summary = $(".description-summary > div.summary__content")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "");
|
||||
|
||||
@@ -99,15 +109,12 @@ const novelScraper = async (req, res) => {
|
||||
novelChapters.push({ chapterName, releaseDate, chapterUrl });
|
||||
});
|
||||
|
||||
novel.novelChapters = novelChapters.reverse();
|
||||
novel.chapters = novelChapters.reverse();
|
||||
|
||||
res.json(novel);
|
||||
return novel;
|
||||
};
|
||||
|
||||
const chapterScraper = async (req, res) => {
|
||||
let novelUrl = req.params.novelUrl;
|
||||
let chapterUrl = req.params.chapterUrl;
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = `${baseUrl}novel/${novelUrl}/${chapterUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
@@ -118,46 +125,20 @@ const chapterScraper = async (req, res) => {
|
||||
$("#snippet-box").remove();
|
||||
|
||||
const chapterName = $("div.text-left > h4").text();
|
||||
|
||||
let chapterText = $(".reading-content").html();
|
||||
chapterText = parseHtml(chapterText);
|
||||
|
||||
let nextChapter = null;
|
||||
|
||||
if ($(".nav-next").length) {
|
||||
nextChapter = $(".nav-next")
|
||||
.find("a")
|
||||
.attr("href")
|
||||
.replace(baseUrl + "novel/" + novelUrl + "/", "");
|
||||
}
|
||||
|
||||
let prevChapter = null;
|
||||
|
||||
if ($(".nav-previous").length) {
|
||||
prevChapter = $(".nav-previous")
|
||||
.find("a")
|
||||
.attr("href")
|
||||
.replace(baseUrl + "novel/" + novelUrl + "/", "");
|
||||
}
|
||||
|
||||
novelUrl = novelUrl + "/";
|
||||
chapterUrl = chapterUrl + "/";
|
||||
|
||||
const chapter = {
|
||||
extensionId: 11,
|
||||
sourceId: 11,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
nextChapter,
|
||||
prevChapter,
|
||||
};
|
||||
|
||||
res.json(chapter);
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchScraper = async (req, res) => {
|
||||
const searchTerm = req.query.s;
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const url = `${baseUrl}?s=${searchTerm}&post_type=wp-manga&m_orderby=rating`;
|
||||
|
||||
const result = await fetch(url);
|
||||
@@ -175,7 +156,7 @@ const searchScraper = async (req, res) => {
|
||||
novelUrl = novelUrl.replace(`${baseUrl}novel/`, "");
|
||||
|
||||
const novel = {
|
||||
extensionId: 11,
|
||||
sourceId: 11,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
@@ -184,12 +165,14 @@ const searchScraper = async (req, res) => {
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
res.json(novels);
|
||||
return novels;
|
||||
};
|
||||
|
||||
module.exports = kissLightNovelScraper = {
|
||||
novelsScraper,
|
||||
novelScraper,
|
||||
chapterScraper,
|
||||
searchScraper,
|
||||
const kissLightNovelScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default kissLightNovelScraper;
|
||||
@@ -1,17 +0,0 @@
|
||||
const express = require("express");
|
||||
const kissLightNovelsScraper = require("./KissLightNovelsScraper");
|
||||
|
||||
const router = express.Router();
|
||||
|
||||
router.get("/novels/", kissLightNovelsScraper.novelsScraper);
|
||||
|
||||
router.get("/novel/:novelUrl", kissLightNovelsScraper.novelScraper);
|
||||
|
||||
router.get(
|
||||
"/novel/:novelUrl/:chapterUrl",
|
||||
kissLightNovelsScraper.chapterScraper
|
||||
);
|
||||
|
||||
router.get("/search/", kissLightNovelsScraper.searchScraper);
|
||||
|
||||
module.exports = router;
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 126 KiB |
@@ -0,0 +1,215 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
|
||||
const baseUrl = "https://www.lightnovelpub.com/";
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
let totalPages = 33;
|
||||
let url = baseUrl + "browse/all/popular/all/" + page;
|
||||
|
||||
let headers = new Headers({
|
||||
Accept: "application/json",
|
||||
"Content-Type": "application/json",
|
||||
"User-Agent":
|
||||
"'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36",
|
||||
});
|
||||
|
||||
const result = await fetch(url, {
|
||||
method: "GET",
|
||||
headers: headers,
|
||||
});
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".novel-item.ads").remove();
|
||||
|
||||
$(".novel-item").each(function () {
|
||||
const novelName = $(this)
|
||||
.find(".novel-title")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "");
|
||||
|
||||
const novelCover = $(this).find("img").attr("data-src");
|
||||
|
||||
let novelUrl = $(this)
|
||||
.find(".novel-title > a")
|
||||
.attr("href")
|
||||
.replace(`/novel/`, "");
|
||||
novelUrl += "/";
|
||||
|
||||
const novel = {
|
||||
sourceId: 15,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return { totalPages, novels };
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = `${baseUrl}novel/${novelUrl}/`;
|
||||
|
||||
let headers = new Headers({
|
||||
Accept: "application/json",
|
||||
"Content-Type": "application/json",
|
||||
"User-Agent":
|
||||
"'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36",
|
||||
});
|
||||
|
||||
const result = await fetch(url, {
|
||||
method: "GET",
|
||||
headers: headers,
|
||||
});
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.sourceId = 15;
|
||||
|
||||
novel.sourceName = "LightNovelPub";
|
||||
|
||||
novel.url = url;
|
||||
|
||||
novel.novelUrl = novelUrl;
|
||||
|
||||
novel.novelName = $("h1.novel-title")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
novel.novelCover = $("figure.cover > img").attr("data-src");
|
||||
|
||||
novel.genre = "";
|
||||
|
||||
$("div.categories > ul > li").each(function () {
|
||||
novel.genre +=
|
||||
$(this)
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "") + ",";
|
||||
});
|
||||
|
||||
$("div.header-stats > span").each(function () {
|
||||
if ($(this).find("small").text() === "Status") {
|
||||
novel.status = $(this).find("strong").text();
|
||||
}
|
||||
});
|
||||
|
||||
novel.genre = novel.genre.slice(0, -1);
|
||||
|
||||
novel.author = $(".author > a > span").text();
|
||||
|
||||
novel.summary = $(".summary > .content").text().trim();
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
let totalChapters;
|
||||
|
||||
totalChapters = $(".header-stats > span").first().text().match(/\d+/)[0];
|
||||
|
||||
for (let i = 1; i <= totalChapters; i++) {
|
||||
const chapterName = "Chapter " + i;
|
||||
|
||||
const releaseDate = null;
|
||||
|
||||
const chapterUrl = "chapter-" + i;
|
||||
|
||||
const chapter = { chapterName, releaseDate, chapterUrl };
|
||||
|
||||
novelChapters.push(chapter);
|
||||
}
|
||||
|
||||
novel.chapters = novelChapters;
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = `${baseUrl}novel/${novelUrl}/${chapterUrl}`;
|
||||
|
||||
let headers = new Headers({
|
||||
Accept: "application/json",
|
||||
"Content-Type": "application/json",
|
||||
"User-Agent":
|
||||
"'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36",
|
||||
});
|
||||
|
||||
const result = await fetch(url, {
|
||||
method: "GET",
|
||||
headers: headers,
|
||||
});
|
||||
const body = await result.text();
|
||||
$ = cheerio.load(body);
|
||||
|
||||
const chapterName = $("h2").text();
|
||||
let chapterText = $("#chapter-container").html();
|
||||
const chapter = {
|
||||
sourceId: 15,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const url = `${baseUrl}lnwsearchlive?inputContent=${searchTerm}`;
|
||||
|
||||
let headers = new Headers({
|
||||
Accept: "application/json",
|
||||
"Content-Type": "application/json",
|
||||
"User-Agent":
|
||||
"'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36",
|
||||
});
|
||||
|
||||
const result = await fetch(url, {
|
||||
method: "GET",
|
||||
headers: headers,
|
||||
});
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
let results = JSON.parse($("body").text());
|
||||
|
||||
$ = cheerio.load(results.resultview);
|
||||
|
||||
$(".novel-item").each(function () {
|
||||
const novelName = $(this).find("h4.novel-title").text();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
|
||||
let novelUrl = $(this).find("a").attr("href").replace(`/novel/`, "");
|
||||
novelUrl += "/";
|
||||
|
||||
const novel = {
|
||||
sourceId: 15,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const lightNovelPubScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default lightNovelPubScraper;
|
||||
@@ -1,14 +0,0 @@
|
||||
const express = require("express");
|
||||
const lightNovelPubScraper = require("./LightNovelPubScraper");
|
||||
|
||||
const router = express.Router();
|
||||
|
||||
router.get("/novels/", lightNovelPubScraper.novelsScraper);
|
||||
|
||||
router.get("/novel/:novelUrl", lightNovelPubScraper.novelScraper);
|
||||
|
||||
router.get("/novel/:novelUrl/:chapterUrl", lightNovelPubScraper.chapterScraper);
|
||||
|
||||
router.get("/search/", lightNovelPubScraper.searchScraper);
|
||||
|
||||
module.exports = router;
|
||||
@@ -1,204 +0,0 @@
|
||||
const cheerio = require("cheerio");
|
||||
const fetch = require("node-fetch");
|
||||
const { htmlToText } = require("html-to-text");
|
||||
const { parseHtml } = require("../../helper");
|
||||
const { scraper } = require("../../helper");
|
||||
|
||||
const baseUrl = "https://www.lightnovelpub.com/";
|
||||
|
||||
const novelsScraper = async (req, res) => {
|
||||
let url = baseUrl + "browse/all/popular/all/1";
|
||||
|
||||
const body = await scraper(url);
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".novel-item.ads").remove();
|
||||
|
||||
$(".novel-item").each(function (result) {
|
||||
const novelName = $(this)
|
||||
.find(".novel-title")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "");
|
||||
|
||||
const novelCover = $(this).find("img").attr("data-src");
|
||||
|
||||
let novelUrl = $(this)
|
||||
.find(".novel-title > a")
|
||||
.attr("href")
|
||||
.replace(`/novel/`, "");
|
||||
novelUrl += "/";
|
||||
|
||||
const novel = {
|
||||
extensionId: 15,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
res.json(novels);
|
||||
};
|
||||
|
||||
const novelScraper = async (req, res) => {
|
||||
const novelUrl = req.params.novelUrl;
|
||||
const url = `${baseUrl}novel/${novelUrl}/`;
|
||||
|
||||
const body = await scraper(url);
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.extensionId = 15;
|
||||
|
||||
novel.sourceName = "LightNovelPub";
|
||||
|
||||
novel.sourceUrl = url;
|
||||
|
||||
novel.novelUrl = `${novelUrl}/`;
|
||||
|
||||
novel.novelName = $("h1.novel-title")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
novel.novelCover = $("figure.cover > img").attr("src");
|
||||
|
||||
novel["Genre(s)"] = "";
|
||||
|
||||
$("div.categories > ul > li").each(function (result) {
|
||||
novel["Genre(s)"] +=
|
||||
$(this)
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "") + ",";
|
||||
});
|
||||
|
||||
$("div.header-stats > span").each(function (result) {
|
||||
if ($(this).find("small").text() === "Status") {
|
||||
novel.Status = $(this).find("strong").text();
|
||||
}
|
||||
});
|
||||
|
||||
novel["Genre(s)"] = novel["Genre(s)"].slice(0, -1);
|
||||
|
||||
novel["Author(s)"] = $(".author > a > span").text();
|
||||
|
||||
novelSummary = $(".summary > .content").html();
|
||||
novel.novelSummary = htmlToText(novelSummary);
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
let firstChapterNo, totalChapters;
|
||||
|
||||
totalChapters = $(".header-stats > span").first().text().match(/\d+/)[0];
|
||||
firstChapterNo = $(".chapter-no").first().text().match(/\d+/)[0];
|
||||
|
||||
for (let i = firstChapterNo; i <= totalChapters; i++) {
|
||||
const chapterName = "Chapter " + i;
|
||||
|
||||
const releaseDate = null;
|
||||
|
||||
const chapterUrl = "chapter-" + i;
|
||||
|
||||
const chapter = { chapterName, releaseDate, chapterUrl };
|
||||
|
||||
novelChapters.push(chapter);
|
||||
}
|
||||
|
||||
novel.novelChapters = novelChapters;
|
||||
|
||||
res.json(novel);
|
||||
};
|
||||
|
||||
const chapterScraper = async (req, res) => {
|
||||
let novelUrl = req.params.novelUrl;
|
||||
let chapterUrl = req.params.chapterUrl;
|
||||
|
||||
const url = `${baseUrl}novel/${novelUrl}/${chapterUrl}`;
|
||||
|
||||
const body = await scraper(url);
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
const chapterName = $("h2").text();
|
||||
let chapterText = $("#chapter-container").html();
|
||||
chapterText = parseHtml(chapterText);
|
||||
|
||||
let nextChapter = null;
|
||||
|
||||
if (!$("a.nextchap.isDisabled").length) {
|
||||
nextChapter = $("a.nextchap")
|
||||
.attr("href")
|
||||
.replace("/novel/" + novelUrl + "/", "");
|
||||
}
|
||||
|
||||
let prevChapter = null;
|
||||
|
||||
if (!$("a.prevchap.isDisabled").length) {
|
||||
prevChapter = $("a.prevchap")
|
||||
.attr("href")
|
||||
.replace("/novel/" + novelUrl + "/", "");
|
||||
}
|
||||
|
||||
novelUrl = novelUrl + "/";
|
||||
chapterUrl = chapterUrl + "/";
|
||||
|
||||
const chapter = {
|
||||
extensionId: 15,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
nextChapter,
|
||||
prevChapter,
|
||||
};
|
||||
|
||||
res.json(chapter);
|
||||
};
|
||||
|
||||
const searchScraper = async (req, res) => {
|
||||
const searchTerm = req.query.s;
|
||||
|
||||
const url = `${baseUrl}lnwsearchlive?inputContent=${searchTerm}`;
|
||||
|
||||
const result = await scraper(url);
|
||||
|
||||
$ = cheerio.load(result);
|
||||
|
||||
let novels = [];
|
||||
|
||||
let results = JSON.parse($("body").text());
|
||||
|
||||
$ = cheerio.load(results.resultview);
|
||||
|
||||
$(".novel-item").each(function (result) {
|
||||
const novelName = $(this).find("h4.novel-title").text();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
|
||||
let novelUrl = $(this).find("a").attr("href").replace(`/novel/`, "");
|
||||
novelUrl += "/";
|
||||
|
||||
const novel = {
|
||||
extensionId: 15,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
res.json(novels);
|
||||
};
|
||||
|
||||
module.exports = lightNovelPubScraper = {
|
||||
novelsScraper,
|
||||
novelScraper,
|
||||
chapterScraper,
|
||||
searchScraper,
|
||||
};
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 91 KiB |
@@ -0,0 +1,159 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
import NovelUpdatesScraper from "./novelupdates";
|
||||
|
||||
const sourceId = 72;
|
||||
const sourceName = "LightNovelUpdates";
|
||||
const baseUrl = "https://www.lightnovelupdates.com/";
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
let url = `${baseUrl}novel/page/${page}/?m_orderby=rating`;
|
||||
let totalPages = 500;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".page-item-detail").each(function () {
|
||||
const novelName = $(this).find(".h5 > a").text();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
const novelUrl = $(this).find(".h5 > a").attr("href");
|
||||
|
||||
const novel = {
|
||||
sourceId,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return { totalPages, novels };
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = novelUrl;
|
||||
|
||||
console.log(url);
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = { sourceId, sourceName, novelUrl, url };
|
||||
|
||||
novel.novelName = $(".post-title > h1").text().trim();
|
||||
novel.novelCover =
|
||||
$(".summary_image > a > img").attr("src") ||
|
||||
"https://github.com/LNReader/lnreader-sources/blob/main/src/coverNotAvailable.jpg?raw=true";
|
||||
|
||||
$(".post-content_item").each(function (result) {
|
||||
detailName = $(this).find(".summary-heading > h5").text().trim();
|
||||
detail = $(this).find(".summary-content").text().trim();
|
||||
|
||||
switch (detailName) {
|
||||
case "Genre(s)":
|
||||
novel.genre = detail.trim().replace(/[\t\n]/g, ",");
|
||||
break;
|
||||
case "Author(s)":
|
||||
novel.author = detail.trim();
|
||||
break;
|
||||
case "Artist(s)":
|
||||
novel.status = detail.trim();
|
||||
break;
|
||||
}
|
||||
});
|
||||
|
||||
novel.summary = $("div.summary__content").text();
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
const novelId = $(".rating-post-id").attr("value");
|
||||
|
||||
let formData = new FormData();
|
||||
formData.append("action", "manga_get_chapters");
|
||||
formData.append("manga", novelId);
|
||||
|
||||
const data = await fetch(
|
||||
"https://www.lightnovelupdates.com/wp-admin/admin-ajax.php",
|
||||
{
|
||||
method: "POST",
|
||||
body: formData,
|
||||
}
|
||||
);
|
||||
const text = await data.text();
|
||||
|
||||
$ = cheerio.load(text);
|
||||
|
||||
$(".wp-manga-chapter").each(function () {
|
||||
const chapterName = $(this).find("a").text().trim();
|
||||
const releaseDate = $(this).find("span").text().trim();
|
||||
const chapterUrl = $(this).find("a").attr("href");
|
||||
|
||||
const chapter = { chapterName, releaseDate, chapterUrl };
|
||||
|
||||
novelChapters.push(chapter);
|
||||
});
|
||||
|
||||
novel.chapters = novelChapters.reverse();
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const chapterName = "";
|
||||
const chapterText = (
|
||||
await NovelUpdatesScraper.parseChapter(novelUrl, chapterUrl)
|
||||
).chapterText;
|
||||
|
||||
const chapter = {
|
||||
sourceId,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const url = `${baseUrl}?s=${searchTerm}&post_type=wp-manga&m_orderby=rating`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".c-tabs-item__content").each(function (result) {
|
||||
const novelName = $(this).find(".h4 > a").text();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
const novelUrl = $(this).find(".h4 > a").attr("href");
|
||||
|
||||
const novel = {
|
||||
sourceId,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const LightNovelUpdatesScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default LightNovelUpdatesScraper;
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 81 KiB |
@@ -1,11 +1,12 @@
|
||||
const cheerio = require("cheerio");
|
||||
const fetch = require("node-fetch");
|
||||
const { parseHtml } = require("../../helper");
|
||||
import cheerio from "react-native-cheerio";
|
||||
import { showToast } from "../../hooks/showToast";
|
||||
import { htmlToText } from "../helpers/htmlToText";
|
||||
|
||||
const baseUrl = "https://lnmtl.com/";
|
||||
|
||||
const novelsScraper = async (req, res) => {
|
||||
let url = baseUrl + "novel?page=1"; // + page;
|
||||
const popularNovels = async (page) => {
|
||||
const totalPages = 63;
|
||||
let url = baseUrl + "novel?page=" + page;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
@@ -22,7 +23,7 @@ const novelsScraper = async (req, res) => {
|
||||
novelUrl = novelUrl.replace(baseUrl + "novel/", "");
|
||||
|
||||
const novel = {
|
||||
extensionId: 37,
|
||||
sourceId: 37,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
@@ -31,11 +32,12 @@ const novelsScraper = async (req, res) => {
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
res.json(novels);
|
||||
return { totalPages, novels };
|
||||
};
|
||||
|
||||
const novelScraper = async (req, res) => {
|
||||
const novelUrl = req.params.novelUrl;
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
showToast("LNMTL might take around 20-30 seconds.");
|
||||
|
||||
const url = baseUrl + "novel/" + novelUrl;
|
||||
|
||||
const result = await fetch(url);
|
||||
@@ -45,11 +47,11 @@ const novelScraper = async (req, res) => {
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.extensionId = 37;
|
||||
novel.sourceId = 37;
|
||||
|
||||
novel.sourceName = "LNMTL";
|
||||
|
||||
novel.sourceUrl = url;
|
||||
novel.url = url;
|
||||
|
||||
novel.novelUrl = novelUrl;
|
||||
|
||||
@@ -57,19 +59,19 @@ const novelScraper = async (req, res) => {
|
||||
|
||||
novel.novelCover = $("div.novel").find("img").attr("src");
|
||||
|
||||
novel.novelSummary = $("div.description").text().trim();
|
||||
novel.summary = $("div.description").text().trim();
|
||||
|
||||
novel["Author(s)"] = $(
|
||||
novel.author = $(
|
||||
"main > div:nth-child(3) > div > div.col-lg-3.col-md-4 > div:nth-child(2) > div.panel-body > dl:nth-child(1) > dd > span"
|
||||
).text();
|
||||
|
||||
novel.Status = $(
|
||||
novel.status = $(
|
||||
"main > div:nth-child(3) > div > div.col-lg-3.col-md-4 > div:nth-child(2) > div.panel-body > dl:nth-child(2) > dd"
|
||||
)
|
||||
.text()
|
||||
.trim();
|
||||
|
||||
novel["Genre(s)"] = $(
|
||||
novel.genre = $(
|
||||
"main > div.container > div > div.col-lg-3.col-md-4 > div:nth-child(4) > div.panel-body > ul"
|
||||
)
|
||||
.text()
|
||||
@@ -112,17 +114,13 @@ const novelScraper = async (req, res) => {
|
||||
}
|
||||
}
|
||||
|
||||
novel.novelChapters = chapters;
|
||||
novel.chapters = chapters;
|
||||
|
||||
res.json(novel);
|
||||
return novel;
|
||||
};
|
||||
|
||||
const chapterScraper = async (req, res) => {
|
||||
let novelUrl = req.params.novelUrl;
|
||||
let chapterUrl = req.params.chapterUrl;
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = `${baseUrl}chapter/${chapterUrl}`;
|
||||
console.log(url);
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
@@ -134,27 +132,28 @@ const chapterScraper = async (req, res) => {
|
||||
$(".original").remove();
|
||||
|
||||
let chapterText = $(".chapter-body").html();
|
||||
chapterText = chapterName + "\n\n" + parseHtml(chapterText);
|
||||
|
||||
let nextChapter = null;
|
||||
let prevChapter = null;
|
||||
if (!chapterText) {
|
||||
chapterText = $(".alert.alert-warning").text();
|
||||
}
|
||||
|
||||
chapterText =
|
||||
chapterName +
|
||||
"\n\n" +
|
||||
htmlToText(chapterText, { removeLineBreaks: false });
|
||||
|
||||
const chapter = {
|
||||
extensionId: 37,
|
||||
sourceId: 37,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
nextChapter,
|
||||
prevChapter,
|
||||
};
|
||||
|
||||
res.json(chapter);
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchScraper = async (req, res) => {
|
||||
const searchTerm = req.query.s;
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const url = "https://lnmtl.com/term";
|
||||
|
||||
const result = await fetch(url);
|
||||
@@ -172,22 +171,24 @@ const searchScraper = async (req, res) => {
|
||||
novels = JSON.parse(novels);
|
||||
|
||||
novels = novels.filter((novel) =>
|
||||
novel.name.toLowerCase().includes(searchTerm)
|
||||
novel.name.toLowerCase().includes(searchTerm.toLowerCase())
|
||||
);
|
||||
|
||||
novels = novels.map((novel) => ({
|
||||
extensionId: 37,
|
||||
sourceId: 37,
|
||||
novelName: novel.name,
|
||||
novelUrl: novel.slug,
|
||||
novelCover: novel.image,
|
||||
}));
|
||||
|
||||
res.json(novels);
|
||||
return novels;
|
||||
};
|
||||
|
||||
module.exports = lnmtlScraper = {
|
||||
novelsScraper,
|
||||
novelScraper,
|
||||
chapterScraper,
|
||||
searchScraper,
|
||||
const LNMTLScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default LNMTLScraper;
|
||||
@@ -1,11 +0,0 @@
|
||||
const express = require("express");
|
||||
const lnmtlScraper = require("./LNMTLScraper");
|
||||
|
||||
const router = express.Router();
|
||||
|
||||
router.get("/novels/", lnmtlScraper.novelsScraper);
|
||||
router.get("/novel/:novelUrl", lnmtlScraper.novelScraper);
|
||||
router.get("/novel/:novelUrl/:chapterUrl", lnmtlScraper.chapterScraper);
|
||||
router.get("/search/", lnmtlScraper.searchScraper);
|
||||
|
||||
module.exports = router;
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 52 KiB |
@@ -1,16 +1,24 @@
|
||||
const cheerio = require("cheerio");
|
||||
const FormData = require("form-data");
|
||||
const fetch = require("node-fetch");
|
||||
const UserAgent = require("user-agents");
|
||||
|
||||
const { scraper, parseHtml } = require("../../helper");
|
||||
import cheerio from "react-native-cheerio";
|
||||
|
||||
const baseUrl = "https://www.mtlnovel.com";
|
||||
|
||||
const novelsScraper = async (req, res) => {
|
||||
const url = `${baseUrl}/alltime-rank/`;
|
||||
const popularNovels = async (page) => {
|
||||
let totalPages = 10;
|
||||
const url = `${baseUrl}/alltime-rank/page/${page}`;
|
||||
|
||||
const body = await scraper(url);
|
||||
console.log(url);
|
||||
|
||||
let headers = new Headers({
|
||||
referer: "https://www.mtlnovel.com/",
|
||||
"User-Agent":
|
||||
"'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36",
|
||||
});
|
||||
|
||||
const result = await fetch(url, {
|
||||
method: "GET",
|
||||
headers: headers,
|
||||
});
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
@@ -24,7 +32,7 @@ const novelsScraper = async (req, res) => {
|
||||
novelUrl = novelUrl.replace("https://www.mtlnovel.com/", "");
|
||||
|
||||
const novel = {
|
||||
extensionId: 5,
|
||||
sourceId: 5,
|
||||
novelUrl,
|
||||
novelName,
|
||||
novelCover,
|
||||
@@ -33,35 +41,43 @@ const novelsScraper = async (req, res) => {
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
res.json(novels);
|
||||
return { totalPages, novels };
|
||||
};
|
||||
|
||||
const novelScraper = async (req, res) => {
|
||||
const novelUrl = req.params.novelUrl;
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = `${baseUrl}/${novelUrl}`;
|
||||
|
||||
const body = await scraper(url);
|
||||
let headers = new Headers({
|
||||
referer: "https://www.mtlnovel.com/alltime-rank/",
|
||||
"User-Agent":
|
||||
"'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36",
|
||||
});
|
||||
|
||||
const result = await fetch(url, {
|
||||
method: "GET",
|
||||
headers: headers,
|
||||
});
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.extensionId = 5;
|
||||
novel.sourceId = 5;
|
||||
|
||||
novel.sourceName = "MTLNovel";
|
||||
|
||||
novel.sourceUrl = url;
|
||||
novel.url = url;
|
||||
|
||||
novel.novelUrl = novelUrl + "/";
|
||||
novel.novelUrl = novelUrl;
|
||||
|
||||
novel.novelName = $("h1.entry-title").text();
|
||||
|
||||
novel.novelCover = $(".nov-head > amp-img").attr("src");
|
||||
|
||||
novel.novelSummary = $("div.desc > h2").next().text();
|
||||
novel.summary = $("div.desc > h2").next().text();
|
||||
|
||||
novel["Author(s)"] = $("tr > td")
|
||||
novel.author = $("tr > td")
|
||||
.filter(function () {
|
||||
return $(this).prev().text().trim() === "Author";
|
||||
})
|
||||
@@ -69,7 +85,7 @@ const novelScraper = async (req, res) => {
|
||||
.text()
|
||||
.replace("Auhtor:", "");
|
||||
|
||||
novel.Status = $("tr > td")
|
||||
novel.status = $("tr > td")
|
||||
.filter(function () {
|
||||
return $(this).prev().text().trim() === "Status";
|
||||
})
|
||||
@@ -77,7 +93,7 @@ const novelScraper = async (req, res) => {
|
||||
.text()
|
||||
.replace("Status:", "");
|
||||
|
||||
novel["Genre(s)"] = $("td")
|
||||
novel.genre = $("td")
|
||||
.filter(function () {
|
||||
return $(this).prev().text().trim() === "Genre";
|
||||
})
|
||||
@@ -85,12 +101,13 @@ const novelScraper = async (req, res) => {
|
||||
.text()
|
||||
.replace("Genre:", "");
|
||||
|
||||
novel["Artist(s)"] = null;
|
||||
novel.artist = null;
|
||||
|
||||
const chapterListUrl = url + "/chapter-list/";
|
||||
|
||||
const getChapters = async () => {
|
||||
const body = await scraper(chapterListUrl);
|
||||
const result = await fetch(chapterListUrl);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
@@ -104,7 +121,7 @@ const novelScraper = async (req, res) => {
|
||||
|
||||
let chapterUrl = $(this).attr("href");
|
||||
chapterUrl = chapterUrl.replace(
|
||||
`https://www.mtlnovel.com/${novelUrl}/`,
|
||||
`https://www.mtlnovel.com/${novelUrl}`,
|
||||
""
|
||||
);
|
||||
|
||||
@@ -114,60 +131,44 @@ const novelScraper = async (req, res) => {
|
||||
releaseDate,
|
||||
});
|
||||
});
|
||||
return novelChapters;
|
||||
return novelChapters.reverse();
|
||||
};
|
||||
|
||||
novel.novelChapters = await getChapters();
|
||||
novel.chapters = await getChapters();
|
||||
|
||||
res.json(novel);
|
||||
return novel;
|
||||
};
|
||||
|
||||
const chapterScraper = async (req, res) => {
|
||||
let novelUrl = req.params.novelUrl;
|
||||
let chapterUrl = req.params.chapterUrl;
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = `${baseUrl}/${novelUrl}${chapterUrl}`;
|
||||
console.log(url);
|
||||
|
||||
const url = `${baseUrl}/${novelUrl}/${chapterUrl}/`;
|
||||
|
||||
const body = await scraper(url);
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
const chapterName = $("h1.main-title").text();
|
||||
let chapterText = $("div.par").html();
|
||||
chapterText = parseHtml(chapterText);
|
||||
|
||||
let nextChapter = null;
|
||||
if ($("a.next").attr("href")) {
|
||||
nextChapter = $("a.next").attr("href").replace(`/${novelUrl}/`, "");
|
||||
}
|
||||
|
||||
let prevChapter = null;
|
||||
if ($("a.prev").attr("href")) {
|
||||
prevChapter = $("a.prev").attr("href").replace(`/${novelUrl}/`, "");
|
||||
}
|
||||
|
||||
chapter = {
|
||||
extensionId: 5,
|
||||
const chapter = {
|
||||
sourceId: 5,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
nextChapter,
|
||||
prevChapter,
|
||||
};
|
||||
|
||||
res.json(chapter);
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchScraper = async (req, res) => {
|
||||
const searchTerm = req.query.s;
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const searchUrl =
|
||||
"https://www.mtlnovel.com/wp-admin/admin-ajax.php?action=autosuggest&q=" +
|
||||
searchTerm +
|
||||
"&__amp_source_origin=https%3A%2F%2Fwww.mtlnovel.com";
|
||||
|
||||
const result = await scraper(searchUrl);
|
||||
const res = await fetch(searchUrl);
|
||||
const result = await res.text();
|
||||
|
||||
$ = cheerio.load(result);
|
||||
|
||||
@@ -183,17 +184,19 @@ const searchScraper = async (req, res) => {
|
||||
""
|
||||
);
|
||||
|
||||
const novel = { extensionId: 5, novelName, novelCover, novelUrl };
|
||||
const novel = { sourceId: 5, novelName, novelCover, novelUrl };
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
res.json(novels);
|
||||
return novels;
|
||||
};
|
||||
|
||||
module.exports = mtlNovelScraper = {
|
||||
novelsScraper,
|
||||
novelScraper,
|
||||
chapterScraper,
|
||||
searchScraper,
|
||||
const mtlNovelScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default mtlNovelScraper;
|
||||
@@ -1,14 +0,0 @@
|
||||
const express = require("express");
|
||||
const mtlNovelScraper = require("./MTLNovelScraper");
|
||||
|
||||
const router = express.Router();
|
||||
|
||||
router.get("/novels/", mtlNovelScraper.novelsScraper);
|
||||
|
||||
router.get("/novel/:novelUrl", mtlNovelScraper.novelScraper);
|
||||
|
||||
router.get("/novel/:novelUrl/:chapterUrl", mtlNovelScraper.chapterScraper);
|
||||
|
||||
router.get("/search/", mtlNovelScraper.searchScraper);
|
||||
|
||||
module.exports = router;
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 43 KiB |
@@ -1,11 +1,10 @@
|
||||
const cheerio = require("cheerio");
|
||||
const fetch = require("node-fetch");
|
||||
const { parseHtml } = require("../../helper");
|
||||
import cheerio from "react-native-cheerio";
|
||||
|
||||
const baseUrl = "https://novelfull.com";
|
||||
|
||||
const novelsScraper = async (req, res) => {
|
||||
const url = `${baseUrl}/most-popular`;
|
||||
const popularNovels = async (page) => {
|
||||
let totalPages = 42;
|
||||
const url = `${baseUrl}/most-popular?page=${page}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
@@ -28,7 +27,7 @@ const novelsScraper = async (req, res) => {
|
||||
novelUrl = `${novelUrl}/`;
|
||||
|
||||
const novel = {
|
||||
extensionId: 8,
|
||||
sourceId: 8,
|
||||
novelUrl,
|
||||
novelName,
|
||||
novelCover,
|
||||
@@ -37,12 +36,11 @@ const novelsScraper = async (req, res) => {
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
res.json(novels);
|
||||
return { totalPages, novels };
|
||||
};
|
||||
|
||||
const novelScraper = async (req, res) => {
|
||||
const novelUrl = req.params.novelUrl;
|
||||
const url = `${baseUrl}/${novelUrl}.html`;
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = `${baseUrl}/${novelUrl.slice(0, -1)}.html`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
@@ -51,37 +49,37 @@ const novelScraper = async (req, res) => {
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.extensionId = 8;
|
||||
novel.sourceId = 8;
|
||||
|
||||
novel.sourceName = "NovelFull";
|
||||
|
||||
novel.sourceUrl = url;
|
||||
novel.url = url;
|
||||
|
||||
novel.novelUrl = `${novelUrl}/`;
|
||||
novel.novelUrl = novelUrl;
|
||||
|
||||
novel.novelName = $("div.book > img").attr("alt");
|
||||
|
||||
novel.novelCover = baseUrl + $("div.book > img").attr("src");
|
||||
|
||||
novel.novelSummary = $("div.desc-text").text();
|
||||
novel.summary = $("div.desc-text").text().trim();
|
||||
|
||||
novel["Author(s)"] = $("div.info > div > h3")
|
||||
novel.author = $("div.info > div > h3")
|
||||
.filter(function () {
|
||||
return $(this).text().trim() === "Author:";
|
||||
})
|
||||
.siblings()
|
||||
.text();
|
||||
|
||||
novel["Genre(s)"] = $("div.info > div")
|
||||
novel.genre = $("div.info > div")
|
||||
.filter(function () {
|
||||
return $(this).find("h3").text().trim() === "Genre:";
|
||||
})
|
||||
.text()
|
||||
.replace("Genre:", "");
|
||||
|
||||
novel["Artist(s)"] = null;
|
||||
novel.artist = null;
|
||||
|
||||
novel.Status = $("div.info > div > h3")
|
||||
novel.status = $("div.info > div > h3")
|
||||
.filter(function () {
|
||||
return $(this).text().trim() === "Status:";
|
||||
})
|
||||
@@ -105,7 +103,7 @@ const novelScraper = async (req, res) => {
|
||||
let chapterName = $(this).text();
|
||||
let releaseDate = null;
|
||||
let chapterUrl = $(this).attr("value");
|
||||
chapterUrl = chapterUrl.replace(`/${novelUrl}/`, "");
|
||||
chapterUrl = chapterUrl.replace(`/${novelUrl}`, "");
|
||||
|
||||
novelChapters.push({
|
||||
chapterName,
|
||||
@@ -116,16 +114,13 @@ const novelScraper = async (req, res) => {
|
||||
return novelChapters;
|
||||
};
|
||||
|
||||
novel.novelChapters = await getChapters(novelId);
|
||||
novel.chapters = await getChapters(novelId);
|
||||
|
||||
res.json(novel);
|
||||
return novel;
|
||||
};
|
||||
|
||||
const chapterScraper = async (req, res) => {
|
||||
const novelUrl = req.params.novelUrl.replace(".html/", "");
|
||||
const chapterUrl = req.params.chapterUrl;
|
||||
|
||||
const url = `${baseUrl}/${novelUrl}/${chapterUrl}`;
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = `${baseUrl}/${novelUrl}${chapterUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
@@ -134,37 +129,18 @@ const chapterScraper = async (req, res) => {
|
||||
|
||||
const chapterName = $(".chapter-title").attr("title");
|
||||
let chapterText = $("#chapter-content").html();
|
||||
chapterText = parseHtml(chapterText);
|
||||
|
||||
let nextChapter = null;
|
||||
if ($("a#next_chap").attr("href")) {
|
||||
nextChapter = $("a#next_chap")
|
||||
.attr("href")
|
||||
.replace(`/${novelUrl}/`, "");
|
||||
}
|
||||
|
||||
let prevChapter = null;
|
||||
if ($("a#prev_chap").attr("href")) {
|
||||
prevChapter = $("a#prev_chap")
|
||||
.attr("href")
|
||||
.replace(`/${novelUrl}/`, "");
|
||||
}
|
||||
|
||||
const chapter = {
|
||||
extensionId: 8,
|
||||
sourceId: 8,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
nextChapter,
|
||||
prevChapter,
|
||||
};
|
||||
|
||||
res.json(chapter);
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchScraper = async (req, res) => {
|
||||
const searchTerm = req.query.s;
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const searchUrl = `https://novelfull.com/search?keyword=`;
|
||||
|
||||
const url = `${searchUrl}${searchTerm}`;
|
||||
@@ -188,7 +164,7 @@ const searchScraper = async (req, res) => {
|
||||
const novelCover = baseUrl + $(this).find("img").attr("src");
|
||||
|
||||
const novel = {
|
||||
extensionId: 8,
|
||||
sourceId: 8,
|
||||
novelUrl,
|
||||
novelName,
|
||||
novelCover,
|
||||
@@ -197,12 +173,14 @@ const searchScraper = async (req, res) => {
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
res.json(novels);
|
||||
return novels;
|
||||
};
|
||||
|
||||
module.exports = novelFullScraper = {
|
||||
novelsScraper,
|
||||
novelScraper,
|
||||
chapterScraper,
|
||||
searchScraper,
|
||||
const novelFullScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default novelFullScraper;
|
||||
@@ -1,13 +0,0 @@
|
||||
const express = require("express");
|
||||
const novelFullScraper = require("./NovelFullScraper");
|
||||
const router = express.Router();
|
||||
|
||||
router.get("/novels/", novelFullScraper.novelsScraper);
|
||||
|
||||
router.get("/novel/:novelUrl", novelFullScraper.novelScraper);
|
||||
|
||||
router.get("/novel/:novelUrl/:chapterUrl", novelFullScraper.chapterScraper);
|
||||
|
||||
router.get("/search/", novelFullScraper.searchScraper);
|
||||
|
||||
module.exports = router;
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 63 KiB |
@@ -1,10 +1,9 @@
|
||||
const cheerio = require("cheerio");
|
||||
const fetch = require("node-fetch");
|
||||
const { parseHtml } = require("../../helper");
|
||||
import cheerio from "react-native-cheerio";
|
||||
|
||||
const baseUrl = "https://www.novelhall.com/";
|
||||
|
||||
const novelsScraper = async (req, res) => {
|
||||
const popularNovels = async (page) => {
|
||||
let totalPages = 1;
|
||||
const result = await fetch(baseUrl);
|
||||
const body = await result.text();
|
||||
|
||||
@@ -20,7 +19,7 @@ const novelsScraper = async (req, res) => {
|
||||
const novelUrl = $(this).find("a").attr("href").substring(1);
|
||||
|
||||
const novel = {
|
||||
extensionId: 6,
|
||||
sourceId: 6,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
@@ -29,11 +28,10 @@ const novelsScraper = async (req, res) => {
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
res.json(novels);
|
||||
return { totalPages, novels };
|
||||
};
|
||||
|
||||
const novelScraper = async (req, res) => {
|
||||
const novelUrl = req.params.novelUrl;
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = `${baseUrl}${novelUrl}/`;
|
||||
|
||||
const result = await fetch(url);
|
||||
@@ -43,29 +41,29 @@ const novelScraper = async (req, res) => {
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.extensionId = 6;
|
||||
novel.sourceId = 6;
|
||||
|
||||
novel.sourceName = "NovelHall";
|
||||
|
||||
novel.sourceUrl = url;
|
||||
novel.url = url;
|
||||
|
||||
novel.novelUrl = `${novelUrl}/`;
|
||||
novel.novelUrl = novelUrl;
|
||||
|
||||
novel.novelName = $("h1").text();
|
||||
|
||||
novel.novelCover = $("div.book-img > img").attr("src");
|
||||
|
||||
novel.novelSummary = $("div.intro")
|
||||
novel.summary = $("div.intro")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "");
|
||||
|
||||
novel["Author(s)"] = $("span.blue").first().text().replace("Author：", "");
|
||||
novel.author = $("span.blue").first().text().replace("Author：", "");
|
||||
|
||||
novel["Genre(s)"] = $("a.red").text();
|
||||
novel.genre = $("a.red").text();
|
||||
|
||||
novel["Artist(s)"] = null;
|
||||
novel.artist = null;
|
||||
|
||||
novel.Status = $("span.blue").first().next().text().replace("Status：", "");
|
||||
novel.status = $("span.blue").first().next().text().replace("Status：", "");
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
@@ -91,15 +89,12 @@ const novelScraper = async (req, res) => {
|
||||
novelChapters.push(chapter);
|
||||
});
|
||||
|
||||
novel.novelChapters = novelChapters;
|
||||
novel.chapters = novelChapters;
|
||||
|
||||
res.json(novel);
|
||||
return novel;
|
||||
};
|
||||
|
||||
const chapterScraper = async (req, res) => {
|
||||
let novelUrl = req.params.novelUrl.replace(".html/", "");
|
||||
let chapterUrl = req.params.chapterUrl;
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = `${baseUrl}${novelUrl}/${chapterUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
@@ -109,38 +104,18 @@ const chapterScraper = async (req, res) => {
|
||||
|
||||
const chapterName = $("h1").text();
|
||||
let chapterText = $("div.entry-content").html();
|
||||
chapterText = parseHtml(chapterText);
|
||||
|
||||
let nextChapter = null;
|
||||
nextChapter = $('a[rel="next"]').attr("href");
|
||||
if (nextChapter) {
|
||||
nextChapter = nextChapter.replace(`/${req.params.novelUrl}/`, "");
|
||||
}
|
||||
|
||||
let prevChapter = null;
|
||||
prevChapter = $('a[rel="prev"]').attr("href");
|
||||
if (prevChapter) {
|
||||
prevChapter = prevChapter.replace(`/${req.params.novelUrl}/`, "");
|
||||
}
|
||||
|
||||
novelUrl = novelUrl + "/";
|
||||
chapterUrl = chapterUrl + "/";
|
||||
|
||||
chapter = {
|
||||
extensionId: 6,
|
||||
sourceId: 6,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
nextChapter,
|
||||
prevChapter,
|
||||
};
|
||||
|
||||
res.json(chapter);
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchScraper = async (req, res) => {
|
||||
const searchTerm = req.query.s;
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const searchUrl = `${baseUrl}index.php?s=so&module=book&keyword=`;
|
||||
|
||||
const url = `${searchUrl}${searchTerm}`;
|
||||
@@ -164,7 +139,7 @@ const searchScraper = async (req, res) => {
|
||||
}
|
||||
|
||||
novel = {
|
||||
extensionId: 6,
|
||||
sourceId: 6,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
@@ -173,12 +148,14 @@ const searchScraper = async (req, res) => {
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
res.json(novels);
|
||||
return novels;
|
||||
};
|
||||
|
||||
module.exports = novelhallScraper = {
|
||||
novelsScraper,
|
||||
novelScraper,
|
||||
chapterScraper,
|
||||
searchScraper,
|
||||
const novelhallScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default novelhallScraper;
|
||||
@@ -1,14 +0,0 @@
|
||||
const express = require("express");
|
||||
const novelhallScraper = require("./NovelhallScraper");
|
||||
|
||||
const router = express.Router();
|
||||
|
||||
router.get("/novels/", novelhallScraper.novelsScraper);
|
||||
|
||||
router.get("/novel/:novelUrl", novelhallScraper.novelScraper);
|
||||
|
||||
router.get("/novel/:novelUrl/:chapterUrl", novelhallScraper.chapterScraper);
|
||||
|
||||
router.get("/search/", novelhallScraper.searchScraper);
|
||||
|
||||
module.exports = router;
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 70 KiB |
@@ -0,0 +1,158 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
import { Status } from "../helpers/constants";
|
||||
|
||||
const sourceId = 71;
|
||||
const sourceName = "NovelOnlineFull.com";
|
||||
const baseUrl = "https://novelonlinefull.com/";
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
let url =
|
||||
baseUrl + "novel_list?type=topview&category=all&state=all&page=" + page;
|
||||
const totalPages = 2155;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".update_item.list_category").each(function () {
|
||||
const novelName = $(this).find("h3").text().trim();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
|
||||
let novelUrl = $(this).find("h3 > a").attr("href");
|
||||
novelUrl = novelUrl.replace(baseUrl + "novel/", "");
|
||||
|
||||
const novel = {
|
||||
sourceId,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return { novels, totalPages };
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = baseUrl + "novel/" + novelUrl;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = { sourceId, sourceName, url, novelUrl };
|
||||
|
||||
novel.novelName = $("h1").text().trim();
|
||||
|
||||
novel.novelCover = $(".info_image > img").attr("src");
|
||||
|
||||
novel.summary = $("#noidungm").text().trim();
|
||||
|
||||
novel.genre = $("div.truyen_if_wrap > ul > li:nth-child(3)")
|
||||
.text()
|
||||
.replace("GENRES: ", "")
|
||||
.replace(/ - /g, ",")
|
||||
.trim();
|
||||
|
||||
console.log(novel.genre);
|
||||
|
||||
novel.status =
|
||||
$(" div.truyen_if_wrap > ul > li:nth-child(4) > a")
|
||||
.text()
|
||||
.replace("STATUS", "")
|
||||
.trim() === "ONGOING"
|
||||
? Status.ONGOING
|
||||
: Status.COMPLETED;
|
||||
|
||||
novel.author = $("div.truyen_if_wrap > ul > li:nth-child(2)")
|
||||
.text()
|
||||
.replace("Author(s):", "")
|
||||
.trim();
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
$(".chapter-list")
|
||||
.find(".row")
|
||||
.each(function () {
|
||||
const releaseDate = $(this).find("span:nth-child(2)").text();
|
||||
|
||||
const chapterName = $(this).find("span:nth-child(1) > a").text();
|
||||
const chapterUrl = $(this).find("a").attr("href");
|
||||
|
||||
novelChapters.push({
|
||||
chapterName,
|
||||
chapterUrl,
|
||||
releaseDate,
|
||||
});
|
||||
});
|
||||
|
||||
novel.chapters = novelChapters.reverse();
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = chapterUrl;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let chapterName = $(".name_chapter").text();
|
||||
let chapterText = $("#vung_doc").html();
|
||||
|
||||
const chapter = {
|
||||
sourceId,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const url = baseUrl + "search_novels/" + searchTerm;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".update_item.list_category").each(function () {
|
||||
const novelName = $(this).find("h3").text();
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
|
||||
let novelUrl = $(this).find("h3 > a").attr("href");
|
||||
novelUrl = novelUrl.replace(baseUrl + "novel/", "");
|
||||
|
||||
const novel = {
|
||||
sourceId,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const NovelOnlineFullScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default NovelOnlineFullScraper;
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 40 KiB |
@@ -1,12 +1,12 @@
|
||||
const cheerio = require("cheerio");
|
||||
const fetch = require("node-fetch");
|
||||
const { htmlToText } = require("html-to-text");
|
||||
const { parseHtml } = require("../../helper");
|
||||
import cheerio from "react-native-cheerio";
|
||||
|
||||
import moment from "moment";
|
||||
|
||||
const baseUrl = "https://www.novelpassion.com/";
|
||||
|
||||
const novelsScraper = async (req, res) => {
|
||||
let url = baseUrl + "category/all?p=1&s=1&f=4";
|
||||
const popularNovels = async (page) => {
|
||||
const totalPages = 42;
|
||||
let url = baseUrl + "category/all?p=" + page + "&s=1&f=4";
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
@@ -27,7 +27,7 @@ const novelsScraper = async (req, res) => {
|
||||
novelUrl = novelUrl.replace("/novel/", "");
|
||||
|
||||
const novel = {
|
||||
extensionId: 33,
|
||||
sourceId: 33,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
@@ -36,11 +36,10 @@ const novelsScraper = async (req, res) => {
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
res.json(novels);
|
||||
return { totalPages, novels };
|
||||
};
|
||||
|
||||
const novelScraper = async (req, res) => {
|
||||
const novelUrl = req.params.novelUrl;
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = baseUrl + "novel/" + novelUrl;
|
||||
|
||||
const result = await fetch(url);
|
||||
@@ -50,11 +49,11 @@ const novelScraper = async (req, res) => {
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.extensionId = 33;
|
||||
novel.sourceId = 33;
|
||||
|
||||
novel.sourceName = "NovelPassion";
|
||||
|
||||
novel.sourceUrl = url;
|
||||
novel.url = url;
|
||||
|
||||
novel.novelUrl = novelUrl;
|
||||
|
||||
@@ -63,8 +62,8 @@ const novelScraper = async (req, res) => {
|
||||
|
||||
novel.novelCover = baseUrl + $("i.g_thumb > img").attr("src");
|
||||
|
||||
let novelSummary = $("div.j_synopsis").html();
|
||||
novel.novelSummary = htmlToText(novelSummary);
|
||||
let novelSummary = $("div.j_synopsis").text().trim();
|
||||
novel.summary = novelSummary;
|
||||
|
||||
$("address.lh20.m14.pr").each(function (result) {
|
||||
detailName = $(this).find("p.ell.dib.vam").text().trim();
|
||||
@@ -73,15 +72,15 @@ const novelScraper = async (req, res) => {
|
||||
|
||||
switch (detailName) {
|
||||
case "Categories:":
|
||||
novel["Genre(s)"] = detail.trim().replace(/ - /g, ",");
|
||||
novel.genre = detail.trim().replace(/ - /g, ",");
|
||||
break;
|
||||
case "Author:":
|
||||
novel["Author(s)"] = detail.trim().replace(/ - /g, ", ");
|
||||
novel.author = detail.trim().replace(/ - /g, ", ");
|
||||
break;
|
||||
}
|
||||
|
||||
if (detailName.includes("Status:")) {
|
||||
novel.Status = detailName.replace("Status:", "").trim();
|
||||
novel.status = detailName.replace("Status:", "").trim();
|
||||
}
|
||||
});
|
||||
|
||||
@@ -92,7 +91,8 @@ const novelScraper = async (req, res) => {
|
||||
.each(function (result) {
|
||||
const chapterName = $(this).find("span.sp1").text();
|
||||
|
||||
const releaseDate = $(this).find("i.sp2").text();
|
||||
let releaseDate = $(this).find("i.sp2").text();
|
||||
releaseDate = moment(new Date(releaseDate)).format("MM/DD/YY");
|
||||
|
||||
const chapterUrl = $(this)
|
||||
.find("a")
|
||||
@@ -106,15 +106,12 @@ const novelScraper = async (req, res) => {
|
||||
});
|
||||
});
|
||||
|
||||
novel.novelChapters = novelChapters;
|
||||
novel.chapters = novelChapters.reverse();
|
||||
|
||||
res.json(novel);
|
||||
return novel;
|
||||
};
|
||||
|
||||
const chapterScraper = async (req, res) => {
|
||||
let novelUrl = req.params.novelUrl;
|
||||
let chapterUrl = req.params.chapterUrl;
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = `${baseUrl}novel/${novelUrl}/${chapterUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
@@ -125,27 +122,18 @@ const chapterScraper = async (req, res) => {
|
||||
let chapterName = $("div.cha-content").find("strong").text();
|
||||
|
||||
let chapterText = $("div.cha-content").html();
|
||||
chapterText = parseHtml(chapterText);
|
||||
|
||||
let nextChapter = null;
|
||||
let prevChapter = null;
|
||||
|
||||
const chapter = {
|
||||
extensionId: 33,
|
||||
sourceId: 33,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
nextChapter,
|
||||
prevChapter,
|
||||
};
|
||||
|
||||
res.json(chapter);
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchScraper = async (req, res) => {
|
||||
const searchTerm = req.query.s;
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const url = baseUrl + "search?keyword=" + searchTerm;
|
||||
|
||||
const result = await fetch(url);
|
||||
@@ -167,7 +155,7 @@ const searchScraper = async (req, res) => {
|
||||
novelUrl = novelUrl.replace("/novel/", "");
|
||||
|
||||
const novel = {
|
||||
extensionId: 33,
|
||||
sourceId: 33,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
@@ -176,12 +164,14 @@ const searchScraper = async (req, res) => {
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
res.json(novels);
|
||||
return novels;
|
||||
};
|
||||
|
||||
module.exports = vipNovelScraper = {
|
||||
novelsScraper,
|
||||
novelScraper,
|
||||
chapterScraper,
|
||||
searchScraper,
|
||||
const NovelPassionScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default NovelPassionScraper;
|
||||
@@ -1,14 +0,0 @@
|
||||
const express = require("express");
|
||||
const NovelPassionScraper = require("./NovelPassionScraper");
|
||||
|
||||
const router = express.Router();
|
||||
|
||||
router.get("/novels/", NovelPassionScraper.novelsScraper);
|
||||
|
||||
router.get("/novel/:novelUrl", NovelPassionScraper.novelScraper);
|
||||
|
||||
router.get("/novel/:novelUrl/:chapterUrl", NovelPassionScraper.chapterScraper);
|
||||
|
||||
router.get("/search/", NovelPassionScraper.searchScraper);
|
||||
|
||||
module.exports = router;
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 81 KiB |
@@ -1,11 +1,10 @@
|
||||
const cheerio = require("cheerio");
|
||||
const fetch = require("node-fetch");
|
||||
const { parseHtml } = require("../../helper");
|
||||
import cheerio from "react-native-cheerio";
|
||||
|
||||
const baseUrl = "https://noveltrench.com/";
|
||||
|
||||
const novelsScraper = async (req, res) => {
|
||||
let url = baseUrl;
|
||||
const popularNovels = async (page) => {
|
||||
let totalPages = 270;
|
||||
let url = baseUrl + "manga/page/" + page;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
@@ -22,7 +21,7 @@ const novelsScraper = async (req, res) => {
|
||||
novelUrl = novelUrl.replace(`${baseUrl}manga/`, "");
|
||||
|
||||
const novel = {
|
||||
extensionId: 9,
|
||||
sourceId: 9,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
@@ -31,12 +30,11 @@ const novelsScraper = async (req, res) => {
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
res.json(novels);
|
||||
return { totalPages, novels };
|
||||
};
|
||||
|
||||
const novelScraper = async (req, res) => {
|
||||
const novelUrl = req.params.novelUrl;
|
||||
const url = `${baseUrl}manga/${novelUrl}/`;
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = `${baseUrl}manga/${novelUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
@@ -45,13 +43,13 @@ const novelScraper = async (req, res) => {
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.extensionId = 9;
|
||||
novel.sourceId = 9;
|
||||
|
||||
novel.sourceName = "NovelTrench";
|
||||
|
||||
novel.sourceUrl = url;
|
||||
novel.url = url;
|
||||
|
||||
novel.novelUrl = `${novelUrl}/`;
|
||||
novel.novelUrl = novelUrl;
|
||||
|
||||
novel.novelName = $(".post-title > h1")
|
||||
.text()
|
||||
@@ -72,12 +70,22 @@ const novelScraper = async (req, res) => {
|
||||
.replace(/[\t\n]/g, "")
|
||||
.trim();
|
||||
|
||||
novel[detailName] = detail;
|
||||
switch (detailName) {
|
||||
case "Genre(s)":
|
||||
novel.genre = detail.trim().replace(/[\t\n]/g, ",");
|
||||
break;
|
||||
case "Author(s)":
|
||||
novel.author = detail.trim();
|
||||
break;
|
||||
case "Artist(s)":
|
||||
novel.status = detail.trim();
|
||||
break;
|
||||
}
|
||||
});
|
||||
|
||||
$(".description-summary > div.summary__content").find("em").remove();
|
||||
|
||||
novel.novelSummary = $(".description-summary > div.summary__content")
|
||||
novel.summary = $(".description-summary > div.summary__content")
|
||||
.text()
|
||||
.replace(/[\t\n]/g, "");
|
||||
|
||||
@@ -97,20 +105,16 @@ const novelScraper = async (req, res) => {
|
||||
.trim();
|
||||
|
||||
chapterUrl = $(this).find("a").attr("href").replace(url, "");
|
||||
|
||||
novelChapters.push({ chapterName, releaseDate, chapterUrl });
|
||||
});
|
||||
|
||||
novel.novelChapters = novelChapters.reverse();
|
||||
novel.chapters = novelChapters.reverse();
|
||||
|
||||
res.json(novel);
|
||||
return novel;
|
||||
};
|
||||
|
||||
const chapterScraper = async (req, res) => {
|
||||
let novelUrl = req.params.novelUrl;
|
||||
let chapterUrl = req.params.chapterUrl;
|
||||
|
||||
const url = `${baseUrl}manga/${novelUrl}/${chapterUrl}`;
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = `${baseUrl}manga/${novelUrl}${chapterUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
@@ -119,45 +123,18 @@ const chapterScraper = async (req, res) => {
|
||||
|
||||
const chapterName = $("h1#chapter-heading").text();
|
||||
let chapterText = $(".reading-content").html();
|
||||
chapterText = parseHtml(chapterText);
|
||||
|
||||
let nextChapter = null;
|
||||
|
||||
if ($(".nav-next").length) {
|
||||
nextChapter = $(".nav-next")
|
||||
.find("a")
|
||||
.attr("href")
|
||||
.replace(baseUrl + "manga/" + novelUrl + "/", "");
|
||||
}
|
||||
|
||||
let prevChapter = null;
|
||||
|
||||
if ($(".nav-previous").length) {
|
||||
prevChapter = $(".nav-previous")
|
||||
.find("a")
|
||||
.attr("href")
|
||||
.replace(baseUrl + "manga/" + novelUrl + "/", "");
|
||||
}
|
||||
|
||||
novelUrl = novelUrl + "/";
|
||||
chapterUrl = chapterUrl + "/";
|
||||
|
||||
const chapter = {
|
||||
extensionId: 9,
|
||||
sourceId: 9,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
nextChapter,
|
||||
prevChapter,
|
||||
};
|
||||
|
||||
res.json(chapter);
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchScraper = async (req, res) => {
|
||||
const searchTerm = req.query.s;
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const url = `${baseUrl}?s=${searchTerm}&post_type=wp-manga`;
|
||||
|
||||
const result = await fetch(url);
|
||||
@@ -175,7 +152,7 @@ const searchScraper = async (req, res) => {
|
||||
novelUrl = novelUrl.replace(`${baseUrl}manga/`, "");
|
||||
|
||||
const novel = {
|
||||
extensionId: 9,
|
||||
sourceId: 9,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
@@ -184,12 +161,14 @@ const searchScraper = async (req, res) => {
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
res.json(novels);
|
||||
return novels;
|
||||
};
|
||||
|
||||
module.exports = novelTrenchScraper = {
|
||||
novelsScraper,
|
||||
novelScraper,
|
||||
chapterScraper,
|
||||
searchScraper,
|
||||
const novelTrenchScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default novelTrenchScraper;
|
||||
@@ -1,14 +0,0 @@
|
||||
const express = require("express");
|
||||
const novelTrenchScraper = require("./NovelTrenchScraper");
|
||||
|
||||
const router = express.Router();
|
||||
|
||||
router.get("/novels/", novelTrenchScraper.novelsScraper);
|
||||
|
||||
router.get("/novel/:novelUrl", novelTrenchScraper.novelScraper);
|
||||
|
||||
router.get("/novel/:novelUrl/:chapterUrl", novelTrenchScraper.chapterScraper);
|
||||
|
||||
router.get("/search/", novelTrenchScraper.searchScraper);
|
||||
|
||||
module.exports = router;
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 93 KiB |
@@ -0,0 +1,310 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
|
||||
const sourceId = 50;
|
||||
|
||||
const sourceName = "Novel Updates";
|
||||
|
||||
const baseUrl = "https://www.novelupdates.com/";
|
||||
|
||||
let headers = new Headers({
|
||||
"User-Agent":
|
||||
"'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36",
|
||||
});
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
const totalPages = 100;
|
||||
let url = `${baseUrl}series-ranking/?rank=week&pg=` + page;
|
||||
|
||||
const result = await fetch(url, {
|
||||
method: "GET",
|
||||
headers: headers,
|
||||
});
|
||||
const body = await result.text();
|
||||
|
||||
const $ = cheerio.load(body);
|
||||
|
||||
const novels = [];
|
||||
|
||||
$("div.search_main_box_nu").each(function (res) {
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
const novelName = $(this).find(".search_title > a").text();
|
||||
const novelUrl = $(this)
|
||||
.find(".search_title > a")
|
||||
.attr("href")
|
||||
.split("/")[4];
|
||||
|
||||
const novel = {
|
||||
sourceId,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return { totalPages, novels };
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = `${baseUrl}series/${novelUrl}`;
|
||||
|
||||
const result = await fetch(url, {
|
||||
method: "GET",
|
||||
headers: headers,
|
||||
});
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = {
|
||||
sourceId,
|
||||
sourceName,
|
||||
url,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novel.novelName = $(".seriestitlenu").text();
|
||||
|
||||
novel.novelCover = $(".seriesimg > img").attr("src");
|
||||
|
||||
novel.author = $("#showauthors").text().trim();
|
||||
|
||||
novel.genre = $("#seriesgenre").text().trim().replace(/\s/g, ",");
|
||||
|
||||
novel.status = $("#editstatus").text().includes("Ongoing")
|
||||
? "Ongoing"
|
||||
: "Completed";
|
||||
|
||||
let type = $("#showtype").text().trim();
|
||||
|
||||
let summary = $("#editdescription").text().trim();
|
||||
|
||||
novel.summary = summary + `\n\nType: ${type}`;
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
const novelId = $("input#mypostid").attr("value");
|
||||
|
||||
let formData = new FormData();
|
||||
formData.append("action", "nd_getchapters");
|
||||
formData.append("mygrr", 0);
|
||||
formData.append("mypostid", parseInt(novelId));
|
||||
|
||||
const data = await fetch(
|
||||
"https://www.novelupdates.com/wp-admin/admin-ajax.php",
|
||||
{
|
||||
method: "POST",
|
||||
headers,
|
||||
body: formData,
|
||||
}
|
||||
);
|
||||
const text = await data.text();
|
||||
|
||||
$ = cheerio.load(text);
|
||||
|
||||
$("li.sp_li_chp").each(function () {
|
||||
const chapterName = $(this).text().trim();
|
||||
|
||||
const releaseDate = null;
|
||||
|
||||
const chapterUrl =
|
||||
"https:" + $(this).find("a").first().next().attr("href");
|
||||
|
||||
novelChapters.push({ chapterName, releaseDate, chapterUrl });
|
||||
});
|
||||
|
||||
novel.chapters = novelChapters.reverse();
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
function getLocation(href) {
|
||||
var match = href.match(
|
||||
/^(https?\:)\/\/(([^:\/?#]*)(?:\:([0-9]+))?)([\/]{0,1}[^?#]*)(\?[^#]*|)(#.*|)$/
|
||||
);
|
||||
return match && `${match[1]}//${match[3]}`;
|
||||
}
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = chapterUrl;
|
||||
|
||||
// console.log("Original URL: ", url);
|
||||
|
||||
let result, body;
|
||||
|
||||
let chapterName = "";
|
||||
|
||||
try {
|
||||
result = await fetch(url, {
|
||||
method: "GET",
|
||||
headers: headers,
|
||||
});
|
||||
body = await result.text();
|
||||
|
||||
// console.log(result.url);
|
||||
|
||||
// console.log("Redirected URL: ", result.url);
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let isWuxiaWorld = result.url.toLowerCase().includes("wuxiaworld");
|
||||
|
||||
let isBlogspot = result.url.toLowerCase().includes("blogspot");
|
||||
|
||||
let isTumblr = result.url.toLowerCase().includes("tumblr");
|
||||
|
||||
/**
|
||||
* Checks if its a wwordpress site
|
||||
*/
|
||||
let isWordPress =
|
||||
$('meta[name="generator"]').attr("content") || $("footer").text();
|
||||
|
||||
if (isWordPress) {
|
||||
isWordPress =
|
||||
isWordPress.toLowerCase().includes("wordpress") ||
|
||||
isWordPress.includes("Site Kit by Google") ||
|
||||
$(".powered-by").text().toLowerCase().includes("wordpress");
|
||||
}
|
||||
|
||||
let isRainOfSnow = result.url.toLowerCase().includes("rainofsnow");
|
||||
|
||||
let isWebNovel = result.url.toLowerCase().includes("webnovel");
|
||||
|
||||
let isHostedNovel = result.url.toLowerCase().includes("hostednovel");
|
||||
|
||||
let isScribbleHub = result.url.toLowerCase().includes("scribblehub");
|
||||
|
||||
if (isWuxiaWorld) {
|
||||
chapterText = $("#chapter-content").html();
|
||||
} else if (isRainOfSnow) {
|
||||
chapterText = $("div.content").html();
|
||||
} else if (isTumblr) {
|
||||
chapterText = $(".post").html();
|
||||
} else if (isBlogspot) {
|
||||
$(".post-share-buttons").remove();
|
||||
|
||||
chapterText = $(".entry-content").html();
|
||||
} else if (isHostedNovel) {
|
||||
chapterText = $(".chapter").html();
|
||||
} else if (isScribbleHub) {
|
||||
chapterText = $("div.chp_raw").html();
|
||||
} else if (isWordPress) {
|
||||
/**
|
||||
* Remove wordpress bloat tags
|
||||
*/
|
||||
|
||||
const bloatClasses = [
|
||||
".c-ads",
|
||||
"#madara-comments",
|
||||
"#comments",
|
||||
".content-comments",
|
||||
".sharedaddy",
|
||||
".wp-dark-mode-switcher",
|
||||
".wp-next-post-navi",
|
||||
".wp-block-buttons",
|
||||
".wp-block-columns",
|
||||
".post-cats",
|
||||
".sidebar",
|
||||
".author-avatar",
|
||||
".ezoic-ad",
|
||||
];
|
||||
|
||||
bloatClasses.map((tag) => $(tag).remove());
|
||||
|
||||
chapterText =
|
||||
$(".entry-content").html() ||
|
||||
$(".single_post").html() ||
|
||||
$(".post-entry").html() ||
|
||||
$(".main-content").html() ||
|
||||
$("article.post").html() ||
|
||||
$(".content").html() ||
|
||||
$("#content").html();
|
||||
} else if (isWebNovel) {
|
||||
chapterText = $(".cha-words").html();
|
||||
|
||||
if (!chapterText) {
|
||||
chapterText = $("._content").html();
|
||||
}
|
||||
} else {
|
||||
/**
|
||||
* Remove unnecessary tags
|
||||
*/
|
||||
const tags = ["nav", "header", "footer", ".hidden"];
|
||||
|
||||
tags.map((tag) => $(tag).remove());
|
||||
|
||||
chapterText = $("body").html();
|
||||
}
|
||||
|
||||
if (!chapterText) {
|
||||
chapterText =
|
||||
"Chapter not available.\n\nReport if it's available in webview.";
|
||||
} else {
|
||||
/**
|
||||
* Convert relative urls to absolute
|
||||
*/
|
||||
chapterText = chapterText.replace(
|
||||
/href="\//g,
|
||||
`href="${getLocation(result.url)}/`
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
chapterText = `Chapter not available (Error: ${error.message}).\n\nReport if it's available in webview.`;
|
||||
}
|
||||
|
||||
const chapter = {
|
||||
sourceId,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const url =
|
||||
"https://www.novelupdates.com/?s=" +
|
||||
searchTerm +
|
||||
"&post_type=seriesplans";
|
||||
|
||||
const res = await fetch(url, {
|
||||
method: "GET",
|
||||
headers: headers,
|
||||
});
|
||||
const body = await res.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$("div.search_main_box_nu").each(function (res) {
|
||||
const novelCover = $(this).find("img").attr("src");
|
||||
const novelName = $(this).find(".search_title > a").text();
|
||||
const novelUrl = $(this)
|
||||
.find(".search_title > a")
|
||||
.attr("href")
|
||||
.split("/")[4];
|
||||
|
||||
const novel = {
|
||||
sourceId,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
return novels;
|
||||
};
|
||||
|
||||
const NovelUpdatesScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default NovelUpdatesScraper;
|
||||
@@ -1,14 +0,0 @@
|
||||
// const express = require("express");
|
||||
// const novelUpdatesScraper = require("./NovelUpdatesScraper");
|
||||
|
||||
// const router = express.Router();
|
||||
|
||||
// router.get("/novels/", novelUpdatesScraper.novelsScraper);
|
||||
|
||||
// router.get("/novel/:novelUrl", novelUpdatesScraper.novelScraper);
|
||||
|
||||
// router.get("/novel/:novelUrl/:chapterUrl", novelUpdatesScraper.chapterScraper);
|
||||
|
||||
// router.get("/search/", novelUpdatesScraper.searchScraper);
|
||||
|
||||
// module.exports = router;
|
||||
@@ -1,194 +0,0 @@
|
||||
// import cheerio from "react-native-cheerio";
|
||||
// import { htmlToText } from "../helpers/htmlToText";
|
||||
// import moment from "moment";
|
||||
|
||||
// const sourceId = 50;
|
||||
|
||||
// const sourceName = "Novel Updates";
|
||||
|
||||
// const baseUrl = "https://www.novelupdates.com/";
|
||||
|
||||
// let headers = new Headers({
|
||||
// "User-Agent":
|
||||
// "'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36",
|
||||
// });
|
||||
|
||||
// const popularNovels = async (page) => {
|
||||
// const totalPages = 100;
|
||||
// let url = `${baseUrl}series-ranking/?rank=sixmonths&pg=` + page;
|
||||
|
||||
// const result = await fetch(url, {
|
||||
// method: "GET",
|
||||
// headers: headers,
|
||||
// });
|
||||
// const body = await result.text();
|
||||
|
||||
// const $ = cheerio.load(body);
|
||||
|
||||
// const novels = [];
|
||||
|
||||
// $("div.search_main_box_nu").each(function (res) {
|
||||
// const novelCover = $(this).find("img").attr("src");
|
||||
// const novelName = $(this).find(".search_title > a").text();
|
||||
// const novelUrl = $(this)
|
||||
// .find(".search_title > a")
|
||||
// .attr("href")
|
||||
// .split("/")[4];
|
||||
|
||||
// const novel = {
|
||||
// sourceId,
|
||||
// novelName,
|
||||
// novelCover,
|
||||
// novelUrl,
|
||||
// };
|
||||
|
||||
// novels.push(novel);
|
||||
// });
|
||||
|
||||
// return { totalPages, novels };
|
||||
// };
|
||||
|
||||
// const parseNovelAndChapters = async (novelUrl) => {
|
||||
// const url = `${baseUrl}series/${novelUrl}`;
|
||||
|
||||
// const result = await fetch(url, {
|
||||
// method: "GET",
|
||||
// headers: headers,
|
||||
// });
|
||||
// const body = await result.text();
|
||||
|
||||
// $ = cheerio.load(body);
|
||||
|
||||
// let novel = {
|
||||
// sourceId,
|
||||
// sourceName,
|
||||
// url,
|
||||
// novelUrl,
|
||||
// };
|
||||
|
||||
// novel.novelName = $(".seriestitlenu").text();
|
||||
|
||||
// novel.novelCover = $(".seriesimg > img").attr("src");
|
||||
|
||||
// novel.author = $("#showauthors").text().trim();
|
||||
|
||||
// novel.genre = $(".book-generes").text().trim();
|
||||
|
||||
// novel.status = $("#editstatus").text().includes("Ongoing")
|
||||
// ? "Ongoing"
|
||||
// : "Completed";
|
||||
|
||||
// novel.summary = $("#editdescription").text().trim();
|
||||
|
||||
// let novelChapters = [];
|
||||
|
||||
// const novelId = $("input#mypostid").attr("value");
|
||||
|
||||
// let formData = new FormData();
|
||||
// formData.append("action", "nd_getchapters");
|
||||
// formData.append("mygrr", 0);
|
||||
// formData.append("mypostid", parseInt(novelId));
|
||||
|
||||
// const data = await fetch(
|
||||
// "https://www.novelupdates.com/wp-admin/admin-ajax.php",
|
||||
// {
|
||||
// method: "POST",
|
||||
// headers,
|
||||
// body: formData,
|
||||
// }
|
||||
// );
|
||||
// const text = await data.text();
|
||||
|
||||
// $ = cheerio.load(text);
|
||||
|
||||
// $("li.sp_li_chp").each(function () {
|
||||
// const chapterName = $(this).text().trim();
|
||||
|
||||
// const releaseDate = null;
|
||||
|
||||
// const chapterUrl =
|
||||
// "https:" + $(this).find("a").first().next().attr("href");
|
||||
|
||||
// novelChapters.push({ chapterName, releaseDate, chapterUrl });
|
||||
// });
|
||||
|
||||
// novel.chapters = novelChapters.reverse();
|
||||
|
||||
// return novel;
|
||||
// };
|
||||
|
||||
// const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
// const url = chapterUrl;
|
||||
|
||||
// const result = await fetch(url, {
|
||||
// method: "GET",
|
||||
// headers: headers,
|
||||
// });
|
||||
// const body = await result.text();
|
||||
|
||||
// $ = cheerio.load(body);
|
||||
|
||||
// let chapterName = "";
|
||||
|
||||
// let chapterText = body;
|
||||
|
||||
// let nextChapter = null;
|
||||
// let prevChapter = null;
|
||||
|
||||
// const chapter = {
|
||||
// sourceId,
|
||||
// novelUrl,
|
||||
// chapterUrl,
|
||||
// chapterName,
|
||||
// chapterText,
|
||||
// nextChapter,
|
||||
// prevChapter,
|
||||
// };
|
||||
|
||||
// return chapter;
|
||||
// };
|
||||
|
||||
// const searchNovels = async (searchTerm) => {
|
||||
// const url =
|
||||
// "https://www.novelupdates.com/?s=" +
|
||||
// searchTerm +
|
||||
// "&post_type=seriesplans";
|
||||
|
||||
// const res = await fetch(url, {
|
||||
// method: "GET",
|
||||
// headers: headers,
|
||||
// });
|
||||
// const body = await res.text();
|
||||
|
||||
// $ = cheerio.load(body);
|
||||
|
||||
// let novels = [];
|
||||
|
||||
// $("div.search_main_box_nu").each(function (res) {
|
||||
// const novelCover = $(this).find("img").attr("src");
|
||||
// const novelName = $(this).find(".search_title > a").text();
|
||||
// const novelUrl = $(this)
|
||||
// .find(".search_title > a")
|
||||
// .attr("href")
|
||||
// .split("/")[4];
|
||||
|
||||
// const novel = {
|
||||
// sourceId,
|
||||
// novelName,
|
||||
// novelCover,
|
||||
// novelUrl,
|
||||
// };
|
||||
|
||||
// novels.push(novel);
|
||||
// });
|
||||
// return novels;
|
||||
// };
|
||||
|
||||
// const NovelUpdatesScraper = {
|
||||
// popularNovels,
|
||||
// parseNovelAndChapters,
|
||||
// parseChapter,
|
||||
// searchNovels,
|
||||
// };
|
||||
|
||||
// export default NovelUpdatesScraper;
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 56 KiB |
@@ -1,11 +1,9 @@
|
||||
const cheerio = require("cheerio");
|
||||
const fetch = require("node-fetch");
|
||||
const { htmlToText } = require("html-to-text");
|
||||
const { parseHtml } = require("../../helper");
|
||||
import cheerio from "react-native-cheerio";
|
||||
|
||||
const baseUrl = "https://www.novelupdates.cc/";
|
||||
|
||||
const novelsScraper = async (req, res) => {
|
||||
const popularNovels = async (page) => {
|
||||
let totalPages = 1;
|
||||
const result = await fetch(baseUrl);
|
||||
const body = await result.text();
|
||||
|
||||
@@ -20,7 +18,7 @@ const novelsScraper = async (req, res) => {
|
||||
let novelUrl = $(this).find("a").attr("href").slice(1);
|
||||
|
||||
const novel = {
|
||||
extensionId: 18,
|
||||
sourceId: 18,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
@@ -29,11 +27,10 @@ const novelsScraper = async (req, res) => {
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
res.json(novels);
|
||||
return { totalPages, novels };
|
||||
};
|
||||
|
||||
const novelScraper = async (req, res) => {
|
||||
const novelUrl = req.params.novelUrl;
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = `${baseUrl}${novelUrl}/`;
|
||||
|
||||
const result = await fetch(url);
|
||||
@@ -43,26 +40,26 @@ const novelScraper = async (req, res) => {
|
||||
|
||||
let novel = {};
|
||||
|
||||
novel.extensionId = 18;
|
||||
novel.sourceId = 18;
|
||||
|
||||
novel.sourceName = "NovelUpdates.cc";
|
||||
|
||||
novel.sourceUrl = url;
|
||||
novel.url = url;
|
||||
|
||||
novel.novelUrl = `${novelUrl}/`;
|
||||
novel.novelUrl = novelUrl;
|
||||
|
||||
novel.novelName = $(".book-name").text();
|
||||
|
||||
novel.novelCover = $("div.book-img > img").attr("src");
|
||||
|
||||
novel["Genre(s)"] = $("div.book-catalog > span.txt").text();
|
||||
novel.genre = $("div.book-catalog > span.txt").text();
|
||||
|
||||
novel.Status = $("div.book-state > span.txt").text();
|
||||
novel.status = $("div.book-state > span.txt").text();
|
||||
|
||||
novel["Author(s)"] = $("div.author > span.name").text();
|
||||
novel.author = $("div.author > span.name").text();
|
||||
|
||||
novelSummary = $("div.content > p.desc").html();
|
||||
novel.novelSummary = htmlToText(novelSummary);
|
||||
novelSummary = $("div.content > p.desc").text();
|
||||
novel.summary = novelSummary.trim();
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
@@ -78,15 +75,12 @@ const novelScraper = async (req, res) => {
|
||||
});
|
||||
});
|
||||
|
||||
novel.novelChapters = novelChapters;
|
||||
novel.chapters = novelChapters;
|
||||
|
||||
res.json(novel);
|
||||
return novel;
|
||||
};
|
||||
|
||||
const chapterScraper = async (req, res) => {
|
||||
let novelUrl = req.params.novelUrl;
|
||||
let chapterUrl = req.params.chapterUrl;
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = `${baseUrl}${chapterUrl}`;
|
||||
|
||||
const result = await fetch(url);
|
||||
@@ -96,39 +90,18 @@ const chapterScraper = async (req, res) => {
|
||||
|
||||
const chapterName = $("h1.chapter-title").text();
|
||||
let chapterText = $("div.chapter-entity").html();
|
||||
chapterText = parseHtml(chapterText);
|
||||
|
||||
let nextChapter = null;
|
||||
|
||||
if ($("a.next").length) {
|
||||
nextChapter = $("a.next").attr("href").slice(1);
|
||||
}
|
||||
|
||||
let prevChapter = null;
|
||||
|
||||
if ($("a.prev").length) {
|
||||
prevChapter = $("a.prev").attr("href").slice(1);
|
||||
}
|
||||
|
||||
novelUrl = novelUrl + "/";
|
||||
chapterUrl = chapterUrl + "/";
|
||||
|
||||
const chapter = {
|
||||
extensionId: 18,
|
||||
sourceId: 18,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
nextChapter,
|
||||
prevChapter,
|
||||
};
|
||||
|
||||
res.json(chapter);
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchScraper = async (req, res) => {
|
||||
const searchTerm = req.query.s;
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
const url = `${baseUrl}search/${searchTerm}/1`;
|
||||
|
||||
const result = await fetch(url);
|
||||
@@ -147,7 +120,7 @@ const searchScraper = async (req, res) => {
|
||||
let novelUrl = $(this).find("a").attr("href").slice(1);
|
||||
|
||||
const novel = {
|
||||
extensionId: 18,
|
||||
sourceId: 18,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
@@ -156,12 +129,14 @@ const searchScraper = async (req, res) => {
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
res.json(novels);
|
||||
return novels;
|
||||
};
|
||||
|
||||
module.exports = wuxiaWorldCoScraper = {
|
||||
novelsScraper,
|
||||
novelScraper,
|
||||
chapterScraper,
|
||||
searchScraper,
|
||||
const novelUpdatesCcScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default novelUpdatesCcScraper;
|
||||
@@ -1,17 +0,0 @@
|
||||
const express = require("express");
|
||||
const NovelUpdatesCcScraper = require("./NovelUpdatesCcScraper");
|
||||
|
||||
const router = express.Router();
|
||||
|
||||
router.get("/novels/", NovelUpdatesCcScraper.novelsScraper);
|
||||
|
||||
router.get("/novel/:novelUrl", NovelUpdatesCcScraper.novelScraper);
|
||||
|
||||
router.get(
|
||||
"/novel/:novelUrl/:chapterUrl",
|
||||
NovelUpdatesCcScraper.chapterScraper
|
||||
);
|
||||
|
||||
router.get("/search/", NovelUpdatesCcScraper.searchScraper);
|
||||
|
||||
module.exports = router;
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 57 KiB |
@@ -0,0 +1,140 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
import { Status } from "../helpers/constants";
|
||||
|
||||
const sourceId = 78;
|
||||
|
||||
const sourceName = "RainOfSnow";
|
||||
|
||||
const baseUrl = "https://rainofsnow.com/";
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
let url = baseUrl + "novels/page/" + page;
|
||||
|
||||
const totalPages = 5;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".minbox").each(function () {
|
||||
const novelName = $(this).find("h3").text();
|
||||
const novelCover = $(this).find("img").attr("data-src");
|
||||
const novelUrl = $(this).find("h3 > a").attr("href");
|
||||
|
||||
const novel = {
|
||||
sourceId,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return { novels, totalPages };
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = novelUrl;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novel = { sourceId, sourceName, url, novelUrl };
|
||||
|
||||
novel.novelName = $("h2").text().trim();
|
||||
|
||||
novel.novelCover = $(".imagboca1 > img").attr("data-src");
|
||||
|
||||
novel.summary = $("#synop").text().trim();
|
||||
|
||||
novel.genre = $(
|
||||
"body > div.queen > div > div > div.row > div.col-md-12.col-lg-7 > div > div.backcolor1 > ul > li:nth-child(5) > small"
|
||||
)
|
||||
.text()
|
||||
.trim();
|
||||
|
||||
novel.author = $(
|
||||
"body > div.queen > div > div > div.row > div.col-md-12.col-lg-7 > div > div.backcolor1 > ul > li:nth-child(2) > small"
|
||||
).text();
|
||||
|
||||
novel.status = Status.UNKNOWN;
|
||||
|
||||
let novelChapters = [];
|
||||
|
||||
$("#chapter")
|
||||
.find("li")
|
||||
.each(function () {
|
||||
const chapterName = $(this).find(".chapter").first().text().trim();
|
||||
const releaseDate = $(this).find("small").text();
|
||||
const chapterUrl = $(this).find("a").attr("href");
|
||||
|
||||
novelChapters.push({ chapterName, releaseDate, chapterUrl });
|
||||
});
|
||||
|
||||
novel.chapters = novelChapters;
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
const parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const result = await fetch(chapterUrl);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let chapterName = $(".content > h2").text();
|
||||
let chapterText = $(".content").html();
|
||||
|
||||
const chapter = {
|
||||
sourceId,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName,
|
||||
chapterText,
|
||||
};
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
const searchNovels = async (searchTerm) => {
|
||||
let url = baseUrl + "?s=" + searchTerm;
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
$ = cheerio.load(body);
|
||||
|
||||
let novels = [];
|
||||
|
||||
$(".minbox").each(function () {
|
||||
const novelName = $(this).find("h3").text();
|
||||
const novelCover = $(this).find("img").attr("data-src");
|
||||
const novelUrl = $(this).find("h3 > a").attr("href");
|
||||
|
||||
const novel = {
|
||||
sourceId,
|
||||
novelName,
|
||||
novelCover,
|
||||
novelUrl,
|
||||
};
|
||||
|
||||
novels.push(novel);
|
||||
});
|
||||
|
||||
return novels;
|
||||
};
|
||||
|
||||
const RainOfSnowScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default RainOfSnowScraper;
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 166 KiB |
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user