From 0eec38f26fe689a928eb4a4e2dbd9eb88ed426e4 Mon Sep 17 00:00:00 2001 From: Rider21 <58046032+Rider21@users.noreply.github.com> Date: Fri, 5 Aug 2022 18:52:24 +0300 Subject: [PATCH] feat: Source Updates (#350) - Fix ranobes.net (Closes LNReader/lnreader-sources#70) - Add ranobes.com source --- src/screens/history/HistoryScreen.tsx | 3 +- src/screens/updates/UpdatesScreen.tsx | 3 +- src/sources/en/ranobes.js | 208 +++++++++----------------- src/sources/ru/ranobes.js | 160 ++++++++++++++++++++ src/sources/sourceManager.ts | 2 + src/sources/sources.json | 7 + src/utils/constants/dateFormats.ts | 8 - strings/languages/en/strings.json | 10 ++ strings/translations.ts | 9 ++ strings/types/index.ts | 6 + 10 files changed, 269 insertions(+), 147 deletions(-) create mode 100644 src/sources/ru/ranobes.js delete mode 100644 src/utils/constants/dateFormats.ts diff --git a/src/screens/history/HistoryScreen.tsx b/src/screens/history/HistoryScreen.tsx index 54e4c6ab..a416f40c 100644 --- a/src/screens/history/HistoryScreen.tsx +++ b/src/screens/history/HistoryScreen.tsx @@ -15,7 +15,6 @@ import HistoryCard from './components/HistoryCard/HistoryCard'; import { useSearch, useHistory, useBoolean } from '@hooks/index'; import { useTheme } from '../../redux/hooks'; -import { dateFormats } from '../../utils/constants/dateFormats'; import { convertDateToISOString } from '../../database/utils/convertDateToISOString'; import { History } from '../../database/types'; @@ -150,7 +149,7 @@ const HistoryScreen = () => { - {dayjs(date).calendar(null, dateFormats)} + {dayjs(date).calendar()} )} renderItem={({ item }) => ( diff --git a/src/screens/updates/UpdatesScreen.tsx b/src/screens/updates/UpdatesScreen.tsx index 0ba24385..5841d6d7 100644 --- a/src/screens/updates/UpdatesScreen.tsx +++ b/src/screens/updates/UpdatesScreen.tsx @@ -16,7 +16,6 @@ import { ChapterItem, Update } from '../../database/types'; import { useSearch, useUpdates } from '../../hooks'; import { useAppDispatch, useTheme } from '../../redux/hooks'; -import { dateFormats } from '../../utils/constants/dateFormats'; import UpdateCard from './components/UpdateCard/UpdateCard'; import { updateLibraryAction } from '../../redux/updates/updates.actions'; import { @@ -178,7 +177,7 @@ const UpdatesScreen = () => { - {dayjs(date).calendar(null, dateFormats)} + {dayjs(date).calendar()} )} renderItem={({ item }) => ( diff --git a/src/sources/en/ranobes.js b/src/sources/en/ranobes.js index 014f02a5..a39840c5 100644 --- a/src/sources/en/ranobes.js +++ b/src/sources/en/ranobes.js @@ -1,40 +1,33 @@ import * as cheerio from 'cheerio'; import dayjs from 'dayjs'; -import { showToast } from '../../hooks/showToast'; const sourceId = 51; - const sourceName = 'Ranobes'; -const baseUrl = 'https://ranobes.net/'; +const baseUrl = 'https://ranobes.net'; const popularNovels = async page => { - const totalPages = 152; - let url = `${baseUrl}novels/page/${page}`; + let url = `${baseUrl}/novels/page/${page}/`; - let headers = new Headers({ - referer: 'https://ranobes.net/', - 'user-agent': - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36', - }); - - const result = await fetch(url, { method: 'GET', headers: headers }); + const result = await fetch(url); const body = await result.text(); let loadedCheerio = cheerio.load(body); - + const totalPages = parseInt( + loadedCheerio('.pages > a:last-child').text() || '200', + 10, + ); let novels = []; loadedCheerio('article.block.story.shortstory.mod-poster').each(function () { const novelName = loadedCheerio(this).find('h2.title').text(); let novelCover = loadedCheerio(this).find('figure').attr('style'); + let novelUrl = loadedCheerio(this).find('h2.title a').attr('href'); - let novelUrl = loadedCheerio(this) - .find('a') - .attr('href') - .replace(baseUrl + 'novels/', ''); - - novelCover = novelCover.match(/background-image: url(.*);/)[1].slice(1, -1); + novelCover = novelCover + .match(/background-image: url(.*);/)[1] + .slice(1, -1) + .replace('/thumbs/', '/'); const novel = { sourceId, @@ -50,11 +43,7 @@ const popularNovels = async page => { }; const parseNovelAndChapters = async novelUrl => { - showToast('Ranobes: Only has latest 25 chapters.'); - - const url = `${baseUrl}novels/${novelUrl}`; - - const result = await fetch(url); + const result = await fetch(novelUrl); const body = await result.text(); let loadedCheerio = cheerio.load(body); @@ -62,141 +51,67 @@ const parseNovelAndChapters = async novelUrl => { let novel = { sourceId, sourceName, - url, novelUrl, + url: novelUrl, }; novel.novelName = loadedCheerio('meta[property="og:title"]').attr('content'); novel.novelCover = baseUrl + loadedCheerio('.poster > a > img').attr('src'); - novel.author = loadedCheerio('dl.author > dd').text(); - - novel.genre = loadedCheerio('.book-generes').text().trim(); - - novel.summary = loadedCheerio('div[itemprop="description"]').text().trim(); - - novel.author = loadedCheerio('span[itemprop="author"]').text(); - novel.genre = loadedCheerio('span[itemprop="genre"]') + novel.summary = loadedCheerio('div[id="fs-info"] div[class="moreless__full"]') .text() - .replace(', ', ','); + .trim(); - let tempNovelUrl = loadedCheerio('.r-fullstory-chapters-foot') - .find('a') - .next() - .next() - .attr('href') - .split('/')[2]; + novel.author = loadedCheerio('span[class="tag_list"] > a').text(); - const chapterListUrl = `${baseUrl}up/${tempNovelUrl}/page/1`; + let chapterListUrl = loadedCheerio( + 'div.r-fullstory-chapters-foot a[title~=contents]', + ).attr('href'); + + if (!chapterListUrl?.startsWith('http')) { + chapterListUrl = baseUrl + chapterListUrl; + } const chaptersHtml = await fetch(chapterListUrl); - const chapterHtmlToString = await chaptersHtml.text(); + const chaptersHtmlToString = await chaptersHtml.text(); - loadedCheerio = cheerio.load(chapterHtmlToString); - - // let lastPage = $("div.pages > a").last().text(); + loadedCheerio = cheerio.load(chaptersHtmlToString); + let json = loadedCheerio('#dle-content > script:nth-child(3)').html(); + let data = JSON.parse(json.replace('window.__DATA__ =', '')); let novelChapters = []; - loadedCheerio('#dle-content > div.cat_block.cat_line').each(function () { - const chapterName = loadedCheerio(this).find('.title').text(); - - let releaseDate = new Date(); - - let timeAgo = loadedCheerio(this).find('small').text(); - - timeAgo = timeAgo.match(/\d+/)[0]; - - if (timeAgo.includes('hours ago')) { - releaseDate.setTime(releaseDate.getTime() - timeAgo); + for (i = 0; i < data.pages_count; i++) { + if (i > 0) { + let chapterHtml = await fetch(chapterListUrl + 'page/' + (i + 1)); + let chapterHtmlToString = await chapterHtml.text(); + loadedCheerio = cheerio.load(chapterHtmlToString); + json = loadedCheerio('#dle-content > script:nth-child(3)').html(); + data = JSON.parse(json.replace('window.__DATA__ =', '')); } - if (timeAgo.includes('months ago')) { - releaseDate.setMonth(releaseDate.getMonth() - timeAgo); - } - - if (timeAgo.includes('days ago')) { - releaseDate.setDate(releaseDate.getDate() - timeAgo); - } - - releaseDate = dayjs(releaseDate).format('L'); - - const chapterUrl = loadedCheerio(this) - .find('a') - .attr('href') - .replace(baseUrl + 'up/', ''); - - const chapter = { - chapterName, - releaseDate, - chapterUrl, - }; - - novelChapters.push(chapter); - }); - - // for (i = 1; i <= lastPage; i++) { - // const chapterPageUrl = `${baseUrl}up/${tempNovelUrl}/page/${i}`; - - // const chaptersHtml = await fetch(chapterPageUrl); - // const chapterHtmlToString = await chaptersHtml.text(); - // $ = cheerio.load(chapterHtmlToString); - - // $("#dle-content > div.cat_block.cat_line").each(function () { - // const chapterName = $(this).find(".title").text(); - - // let releaseDate = new Date(); - - // let timeAgo = $(this).find("small").text(); - - // timeAgo = timeAgo.match(/\d+/)[0]; - - // if (timeAgo.includes("hours ago")) { - // releaseDate.setTime(releaseDate.getTime() - timeAgo); - // } - - // if (timeAgo.includes("months ago")) { - // releaseDate.setMonth(releaseDate.getMonth() - timeAgo); - // } - - // if (timeAgo.includes("days ago")) { - // releaseDate.setDate(releaseDate.getDate() - timeAgo); - // } - - // releaseDate = dayjs(releaseDate).format("L"); - - // const chapterUrl = $(this) - // .find("a") - // .attr("href") - // .replace(baseUrl + "up/", ""); - - // const chapter = { - // chapterName, - // releaseDate, - // chapterUrl, - // }; - - // novelChapters.push(chapter); - // }); - // } - + data.chapters.forEach(chapter => { + novelChapters.push({ + chapterName: chapter.title, + releaseDate: dayjs(chapter.date).format('LLL'), + chapterUrl: chapter.link, + }); + }); + } novel.chapters = novelChapters.reverse(); return novel; }; const parseChapter = async (novelUrl, chapterUrl) => { - const url = `${baseUrl}up/${chapterUrl}`; - - const result = await fetch(url); + const result = await fetch(chapterUrl); const body = await result.text(); - const loadedCheerio = cheerio.load(body); let chapterName = loadedCheerio('h1.title').text(); - let chapterText = loadedCheerio('#arrticle').html(); + const chapter = { sourceId, novelUrl, @@ -209,12 +124,35 @@ const parseChapter = async (novelUrl, chapterUrl) => { }; const searchNovels = async searchTerm => { - /** - * TODO - */ + let url = `${baseUrl}/index.php?story=${searchTerm}&do=search&subaction=search`; - showToast('Ranobes: Search does not work'); - return []; + const result = await fetch(url); + const body = await result.text(); + + let loadedCheerio = cheerio.load(body); + let novels = []; + + loadedCheerio('article.block.story.shortstory.mod-poster').each(function () { + const novelName = loadedCheerio(this).find('h2.title').text(); + let novelCover = loadedCheerio(this).find('figure').attr('style'); + let novelUrl = loadedCheerio(this).find('h2.title a').attr('href'); + + novelCover = novelCover + .match(/background-image: url(.*);/)[1] + .slice(1, -1) + .replace('/thumbs/', '/'); + + const novel = { + sourceId, + novelName, + novelCover, + novelUrl, + }; + + novels.push(novel); + }); + + return novels; }; const RanobesScraper = { diff --git a/src/sources/ru/ranobes.js b/src/sources/ru/ranobes.js new file mode 100644 index 00000000..4f2aea04 --- /dev/null +++ b/src/sources/ru/ranobes.js @@ -0,0 +1,160 @@ +import * as cheerio from 'cheerio'; + +const sourceId = 132; +const sourceName = 'Ranobes'; + +const baseUrl = 'https://ranobes.com'; + +const popularNovels = async page => { + let url = `${baseUrl}/ranobe/page/${page}/`; + + const result = await fetch(url); + const body = await result.text(); + + let loadedCheerio = cheerio.load(body); + const totalPages = parseInt( + loadedCheerio('.pages > a:last-child').text() || '30', + 10, + ); + let novels = []; + + loadedCheerio('article.block.story.shortstory.mod-poster').each(function () { + const novelName = loadedCheerio(this).find('h2.title').text(); + let novelCover = loadedCheerio(this).find('figure').attr('style'); + let novelUrl = loadedCheerio(this).find('h2.title a').attr('href'); + + novelCover = novelCover + .match(/background-image: url(.*);/)[1] + .slice(1, -1) + .replace('/thumbs/', '/'); + + const novel = { + sourceId, + novelName, + novelCover, + novelUrl, + }; + + novels.push(novel); + }); + + return { totalPages, novels }; +}; + +const parseNovelAndChapters = async novelUrl => { + const result = await fetch(novelUrl); + const body = await result.text(); + + let loadedCheerio = cheerio.load(body); + + let novel = { + sourceId, + sourceName, + novelUrl, + url: novelUrl, + }; + + novel.novelName = loadedCheerio('meta[property="og:title"]').attr('content'); + + novel.novelCover = baseUrl + loadedCheerio('.poster > a > img').attr('src'); + + novel.summary = loadedCheerio('div[itemprop="description"]').text().trim(); + + novel.author = loadedCheerio('span[class="tag_list"] > a').text(); + + let chapterListUrl = loadedCheerio( + 'div.r-fullstory-btns a[title~=оглавление]', + ).attr('href'); + + if (!chapterListUrl?.startsWith('http')) { + chapterListUrl = baseUrl + chapterListUrl; + } + + const chaptersHtml = await fetch(chapterListUrl); + const chaptersHtmlToString = await chaptersHtml.text(); + + loadedCheerio = cheerio.load(chaptersHtmlToString); + let novelChapters = []; + + let all = loadedCheerio('div.pages a:last-child').text(); + all = parseInt(all || '1', 10); + + for (i = 0; i < all; i++) { + if (i > 0) { + let chapterHtml = await fetch(chapterListUrl + 'page/' + (i + 1)); + let chapterHtmlToString = await chapterHtml.text(); + loadedCheerio = cheerio.load(chapterHtmlToString); + } + + loadedCheerio('div[class="cat_block cat_line"] a').each(function () { + novelChapters.push({ + chapterName: loadedCheerio(this).attr('title'), + releaseDate: null, + chapterUrl: loadedCheerio(this).attr('href'), + }); + }); + } + novel.chapters = novelChapters.reverse(); + + return novel; +}; + +const parseChapter = async (novelUrl, chapterUrl) => { + const result = await fetch(chapterUrl); + const body = await result.text(); + const loadedCheerio = cheerio.load(body); + + let chapterName = loadedCheerio('h1.title').text(); + let chapterText = loadedCheerio('#arrticle').html(); + + const chapter = { + sourceId, + novelUrl, + chapterUrl, + chapterName, + chapterText, + }; + + return chapter; +}; + +const searchNovels = async searchTerm => { + let url = `${baseUrl}/index.php?story=${searchTerm}&do=search&subaction=search`; + + const result = await fetch(url); + const body = await result.text(); + + let loadedCheerio = cheerio.load(body); + let novels = []; + + loadedCheerio('article.block.story.shortstory.mod-poster').each(function () { + const novelName = loadedCheerio(this).find('h2.title').text(); + let novelCover = loadedCheerio(this).find('figure').attr('style'); + let novelUrl = loadedCheerio(this).find('h2.title a').attr('href'); + + novelCover = novelCover + .match(/background-image: url(.*);/)[1] + .slice(1, -1) + .replace('/thumbs/', '/'); + + const novel = { + sourceId, + novelName, + novelCover, + novelUrl, + }; + + novels.push(novel); + }); + + return novels; +}; + +const RanobesScraper = { + popularNovels, + parseNovelAndChapters, + parseChapter, + searchNovels, +}; + +export default RanobesScraper; diff --git a/src/sources/sourceManager.ts b/src/sources/sourceManager.ts index 80962e1c..11ffdb8f 100644 --- a/src/sources/sourceManager.ts +++ b/src/sources/sourceManager.ts @@ -78,6 +78,7 @@ import WuxiaBlogScraper from './en/wuxiablog'; import WuxiaCityScraper from './en/wuxiacity'; import NovelUpdatesScraper from './en/novelupdates'; import RanobesScraper from './en/ranobes'; +import RanobesruScraper from './ru/ranobes'; import YushuboScraper from './ch/yushubo'; import { KolNovelScraper, @@ -286,6 +287,7 @@ export const sourceManager = (sourceId: number): Scraper => { 129: LightNovelsBrasilScraper, 130: RiwyatScraper, 131: NovelsticScraper, + 132: RanobesruScraper, }; return scrapers[sourceId]; diff --git a/src/sources/sources.json b/src/sources/sources.json index d0880123..b046013d 100644 --- a/src/sources/sources.json +++ b/src/sources/sources.json @@ -880,5 +880,12 @@ "icon": "https://github.com/LNReader/lnreader-sources/blob/main/multisrc/madara/icons/novelstic.png?raw=true", "url": "https://novelstic.com/", "lang": "English" + }, + { + "sourceId": 132, + "url": "https://ranobes.com/", + "sourceName": "Ranobes", + "icon": "https://github.com/LNReader/lnreader-sources/blob/master/icons/src/en/ranobes/icon.png?raw=true", + "lang": "Russian" } ] diff --git a/src/utils/constants/dateFormats.ts b/src/utils/constants/dateFormats.ts deleted file mode 100644 index 3cb41832..00000000 --- a/src/utils/constants/dateFormats.ts +++ /dev/null @@ -1,8 +0,0 @@ -export const dateFormats = { - sameDay: '[Today]', - nextDay: '[Tomorrow]', - nextWeek: 'dddd', - lastDay: '[Yesterday]', - lastWeek: '[Last] dddd', - sameElse: 'L', -}; diff --git a/strings/languages/en/strings.json b/strings/languages/en/strings.json index 181bf03e..29a72b37 100644 --- a/strings/languages/en/strings.json +++ b/strings/languages/en/strings.json @@ -130,5 +130,15 @@ "advancedSettings": { "deleteReadChapters": "Delete read chapters", "deleteReadChaptersDialogTitle": "Are you sure? All chapters marked as read will be deleted." + }, + "date": { + "calendar": { + "sameDay": "[Today]", + "nextDay": "[Tomorrow]", + "nextWeek": "dddd", + "lastDay": "[Yesterday]", + "lastWeek": "[Last] dddd", + "sameElse": "LL" + } } } diff --git a/strings/translations.ts b/strings/translations.ts index 5830822c..406745bd 100644 --- a/strings/translations.ts +++ b/strings/translations.ts @@ -40,3 +40,12 @@ i18n.locale = Localization.locale; dayjs.locale(Localization.locale); export const getString = (stringKey: keyof StringMap) => i18n.t(stringKey); + +dayjs.Ls[dayjs.locale()].calendar = { + sameDay: getString('date.calendar.sameDay'), + nextDay: getString('date.calendar.nextDay'), + nextWeek: getString('date.calendar.nextWeek'), + lastDay: getString('date.calendar.lastDay'), + lastWeek: getString('date.calendar.lastWeek'), + sameElse: getString('date.calendar.sameElse'), +}; diff --git a/strings/types/index.ts b/strings/types/index.ts index 56c5b029..5312fea1 100644 --- a/strings/types/index.ts +++ b/strings/types/index.ts @@ -94,4 +94,10 @@ export interface StringMap { 'globalSearch.pinnedSources': 'string'; 'advancedSettings.deleteReadChapters': 'string'; 'advancedSettings.deleteReadChaptersDialogTitle': 'string'; + 'date.calendar.sameDay': 'string'; + 'date.calendar.nextDay': 'string'; + 'date.calendar.nextWeek': 'string'; + 'date.calendar.lastDay': 'string'; + 'date.calendar.lastWeek': 'string'; + 'date.calendar.sameElse': 'string'; }