From 766c7b565ee33ff68199cc01b141527434cf650e Mon Sep 17 00:00:00 2001 From: Rajarshee Chatterjee Date: Fri, 8 Apr 2022 17:40:15 +0530 Subject: [PATCH] feat(source): Add FreeNovelUpdates (Closes LNReader/lnreader-sources#184) --- src/sources/en/freenovelupdates.ts | 150 +++++++++++++++++++++++++++++ src/sources/sourceManager.ts | 2 + src/sources/sources.json | 7 ++ src/utils/isAbsoluteUrl.ts | 21 ++++ 4 files changed, 180 insertions(+) create mode 100644 src/sources/en/freenovelupdates.ts create mode 100644 src/utils/isAbsoluteUrl.ts diff --git a/src/sources/en/freenovelupdates.ts b/src/sources/en/freenovelupdates.ts new file mode 100644 index 00000000..2f628a80 --- /dev/null +++ b/src/sources/en/freenovelupdates.ts @@ -0,0 +1,150 @@ +import * as cheerio from 'cheerio'; +import { isUrlAbsolute } from '../../utils/isAbsoluteUrl'; +import { showToast } from '../../hooks/showToast'; +import { + SourceChapter, + SourceChapterItem, + SourceNovel, + SourceNovelItem, +} from '../types'; + +const sourceId = 112; +const sourceName = 'FreeNovelUpdates'; +const baseUrl = 'https://www.freenovelupdates.com'; + +// eslint-disable-next-line @typescript-eslint/no-unused-vars +const popularNovels = async (page: number) => { + const totalPages = 1; + const url = 'https://www.freenovelupdates.com/genres/light-novel-1002'; + + const result = await fetch(url); + const body = await result.text(); + + const loadedCheerio = cheerio.load(body); + + const novels: SourceNovelItem[] = []; + + loadedCheerio('.books-item').each(function () { + let novelUrl = loadedCheerio(this).find('a').attr('href'); + + if (novelUrl && !isUrlAbsolute(novelUrl)) { + novelUrl = baseUrl + novelUrl; + } + + if (novelUrl) { + const novelName = loadedCheerio(this).find('.title').text().trim(); + let novelCover = loadedCheerio(this).find('img').attr('src'); + + if (novelCover && !isUrlAbsolute(novelCover)) { + novelCover = baseUrl + novelCover; + } + + const novel = { + sourceId, + novelUrl, + novelName, + novelCover, + }; + + novels.push(novel); + } + }); + + return { totalPages, novels }; +}; + +const parseNovelAndChapters = async (novelUrl: string) => { + const url = novelUrl; + + const result = await fetch(url); + const body = await result.text(); + + let loadedCheerio = cheerio.load(body); + + const novel: SourceNovel = { + sourceId, + sourceName, + url: novelUrl, + novelUrl, + chapters: [], + }; + + novel.novelName = loadedCheerio('h1').text(); + + let novelCover = loadedCheerio('.img > img').attr('src'); + + novel.novelCover = novelCover + ? isUrlAbsolute(novelCover) + ? novelCover + : baseUrl + novelCover + : undefined; + + novel.summary = loadedCheerio('.description-content').text().trim(); + + novel.author = loadedCheerio('.author').text().trim(); + + novel.genre = loadedCheerio('.category').text().trim(); + + novel.status = loadedCheerio('.status').text().trim(); + + loadedCheerio('.chapter').each(function () { + let chapterUrl = loadedCheerio(this).find('a').attr('href'); + + if (chapterUrl && !isUrlAbsolute(chapterUrl)) { + chapterUrl = baseUrl + chapterUrl; + } + + if (chapterUrl) { + const chapterName = loadedCheerio(this).find('a').text().trim(); + const releaseDate = null; + + const chapter: SourceChapterItem = { + chapterName, + releaseDate, + chapterUrl, + }; + + novel.chapters?.push(chapter); + } + }); + + return novel; +}; + +const parseChapter = async (novelUrl: string, chapterUrl: string) => { + const url = chapterUrl; + + const result = await fetch(url); + const body = await result.text(); + + const loadedCheerio = cheerio.load(body); + + const chapterName = loadedCheerio('h1').text(); + const chapterText = loadedCheerio('.content').html() || ''; + + const chapter: SourceChapter = { + sourceId, + novelUrl, + chapterUrl, + chapterName, + chapterText, + }; + + return chapter; +}; + +// eslint-disable-next-line @typescript-eslint/no-unused-vars +const searchNovels = async (searchTerm: string) => { + showToast('Search is not available in this source'); + + return []; +}; + +const FreeNovelUpdatesScraper = { + popularNovels, + parseNovelAndChapters, + parseChapter, + searchNovels, +}; + +export default FreeNovelUpdatesScraper; diff --git a/src/sources/sourceManager.ts b/src/sources/sourceManager.ts index 419f1f08..20f09303 100644 --- a/src/sources/sourceManager.ts +++ b/src/sources/sourceManager.ts @@ -111,6 +111,7 @@ import SakuraNovelScraper from './id/sakuranovel'; import { SourceChapter, SourceNovel, SourceNovelItem } from './types'; import AllNovelFullScraper from './en/allnovelfull'; import ReadFreeNovelScraper from './en/readfreenovel'; +import FreeNovelUpdatesScraper from './en/freenovelupdates'; interface PopularNovelsResponse { totalPages: number; @@ -236,6 +237,7 @@ export const sourceManager = (sourceId: number): Scraper => { 109: ReadFreeNovelScraper, 110: TurkceLightNovelsScraper, 111: NovelOwlScraper, + 112: FreeNovelUpdatesScraper, }; return scrapers[sourceId]; diff --git a/src/sources/sources.json b/src/sources/sources.json index a014b546..1d65872b 100644 --- a/src/sources/sources.json +++ b/src/sources/sources.json @@ -747,5 +747,12 @@ "icon": "https://github.com/LNReader/lnreader-sources/blob/main/multisrc/madara/icons/novelowl.png?raw=true", "url": "https://novelowl.com/", "lang": "English" + }, + { + "sourceId": 112, + "sourceName": "FreeNovelUpdates", + "icon": "https://github.com/LNReader/lnreader-sources/blob/main/multisrc/madara/icons/freenovelupdates.png?raw=true", + "url": "https://www.freenovelupdates.com/", + "lang": "English" } ] diff --git a/src/utils/isAbsoluteUrl.ts b/src/utils/isAbsoluteUrl.ts new file mode 100644 index 00000000..224817d8 --- /dev/null +++ b/src/utils/isAbsoluteUrl.ts @@ -0,0 +1,21 @@ +export const isUrlAbsolute = (url: string) => { + if (url.indexOf('//') === 0) { + return true; + } // URL is protocol-relative (= absolute) + if (url.indexOf('://') === -1) { + return false; + } // URL has no protocol (= relative) + if (url.indexOf('.') === -1) { + return false; + } // URL does not contain a dot, i.e. no TLD (= relative, possibly REST) + if (url.indexOf('/') === -1) { + return false; + } // URL does not contain a single slash (= relative) + if (url.indexOf(':') > url.indexOf('/')) { + return false; + } // The first colon comes after the first slash (= relative) + if (url.indexOf('://') < url.indexOf('.')) { + return true; + } // Protocol is defined before first dot (= absolute) + return false; // Anything else must be relative +};