open-lab/lnreader
2
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

feat: Source Updates (#350)

Browse Source 
- Fix ranobes.net (Closes LNReader/lnreader-sources#70)
- Add ranobes.com source
This commit is contained in:
Rider21
2022-08-05 18:52:24 +03:00
committed by GitHub
parent fa90d57eff
commit 0eec38f26f
10 changed files with 269 additions and 147 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/screens/history/HistoryScreen.tsx
+1 -2
View File
@@ -15,7 +15,6 @@ import HistoryCard from './components/HistoryCard/HistoryCard';
import { useSearch, useHistory, useBoolean } from '@hooks/index';
import { useTheme } from '../../redux/hooks';
import { dateFormats } from '../../utils/constants/dateFormats';
import { convertDateToISOString } from '../../database/utils/convertDateToISOString';
import { History } from '../../database/types';
@@ -150,7 +149,7 @@ const HistoryScreen = () => {
<Text
style={[styles.dateHeader, { color: theme.textColorSecondary }]}
>
{dayjs(date).calendar(null, dateFormats)}
{dayjs(date).calendar()}
</Text>
)}
renderItem={({ item }) => (
src/screens/updates/UpdatesScreen.tsx
+1 -2
View File
@@ -16,7 +16,6 @@ import { ChapterItem, Update } from '../../database/types';
import { useSearch, useUpdates } from '../../hooks';
import { useAppDispatch, useTheme } from '../../redux/hooks';
import { dateFormats } from '../../utils/constants/dateFormats';
import UpdateCard from './components/UpdateCard/UpdateCard';
import { updateLibraryAction } from '../../redux/updates/updates.actions';
import {
@@ -178,7 +177,7 @@ const UpdatesScreen = () => {
<Text
style={[styles.dateHeader, { color: theme.textColorSecondary }]}
>
{dayjs(date).calendar(null, dateFormats)}
{dayjs(date).calendar()}
</Text>
)}
renderItem={({ item }) => (
src/sources/en/ranobes.js
+73 -135
View File
@@ -1,40 +1,33 @@
import * as cheerio from 'cheerio';
import dayjs from 'dayjs';
import { showToast } from '../../hooks/showToast';
const sourceId = 51;
const sourceName = 'Ranobes';
const baseUrl = 'https://ranobes.net/';
const baseUrl = 'https://ranobes.net';
const popularNovels = async page => {
const totalPages = 152;
let url = `${baseUrl}novels/page/${page}`;
let url = `${baseUrl}/novels/page/${page}/`;
let headers = new Headers({
referer: 'https://ranobes.net/',
'user-agent':
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36',
});
const result = await fetch(url, { method: 'GET', headers: headers });
const result = await fetch(url);
const body = await result.text();
let loadedCheerio = cheerio.load(body);
const totalPages = parseInt(
loadedCheerio('.pages > a:last-child').text() || '200',
10,
);
let novels = [];
loadedCheerio('article.block.story.shortstory.mod-poster').each(function () {
const novelName = loadedCheerio(this).find('h2.title').text();
let novelCover = loadedCheerio(this).find('figure').attr('style');
let novelUrl = loadedCheerio(this).find('h2.title a').attr('href');
let novelUrl = loadedCheerio(this)
.find('a')
.attr('href')
.replace(baseUrl + 'novels/', '');
novelCover = novelCover.match(/background-image: url(.*);/)[1].slice(1, -1);
novelCover = novelCover
.match(/background-image: url(.*);/)[1]
.slice(1, -1)
.replace('/thumbs/', '/');
const novel = {
sourceId,
@@ -50,11 +43,7 @@ const popularNovels = async page => {
};
const parseNovelAndChapters = async novelUrl => {
showToast('Ranobes: Only has latest 25 chapters.');
const url = `${baseUrl}novels/${novelUrl}`;
const result = await fetch(url);
const result = await fetch(novelUrl);
const body = await result.text();
let loadedCheerio = cheerio.load(body);
@@ -62,141 +51,67 @@ const parseNovelAndChapters = async novelUrl => {
let novel = {
sourceId,
sourceName,
url,
novelUrl,
url: novelUrl,
};
novel.novelName = loadedCheerio('meta[property="og:title"]').attr('content');
novel.novelCover = baseUrl + loadedCheerio('.poster > a > img').attr('src');
novel.author = loadedCheerio('dl.author > dd').text();
novel.genre = loadedCheerio('.book-generes').text().trim();
novel.summary = loadedCheerio('div[itemprop="description"]').text().trim();
novel.author = loadedCheerio('span[itemprop="author"]').text();
novel.genre = loadedCheerio('span[itemprop="genre"]')
novel.summary = loadedCheerio('div[id="fs-info"] div[class="moreless__full"]')
.text()
.replace(', ', ',');
.trim();
let tempNovelUrl = loadedCheerio('.r-fullstory-chapters-foot')
.find('a')
.next()
.next()
.attr('href')
.split('/')[2];
novel.author = loadedCheerio('span[class="tag_list"] > a').text();
const chapterListUrl = `${baseUrl}up/${tempNovelUrl}/page/1`;
let chapterListUrl = loadedCheerio(
'div.r-fullstory-chapters-foot a[title~=contents]',
).attr('href');
if (!chapterListUrl?.startsWith('http')) {
chapterListUrl = baseUrl + chapterListUrl;
}
const chaptersHtml = await fetch(chapterListUrl);
const chapterHtmlToString = await chaptersHtml.text();
const chaptersHtmlToString = await chaptersHtml.text();
loadedCheerio = cheerio.load(chapterHtmlToString);
// let lastPage = $("div.pages > a").last().text();
loadedCheerio = cheerio.load(chaptersHtmlToString);
let json = loadedCheerio('#dle-content > script:nth-child(3)').html();
let data = JSON.parse(json.replace('window.__DATA__ =', ''));
let novelChapters = [];
loadedCheerio('#dle-content > div.cat_block.cat_line').each(function () {
const chapterName = loadedCheerio(this).find('.title').text();
let releaseDate = new Date();
let timeAgo = loadedCheerio(this).find('small').text();
timeAgo = timeAgo.match(/\d+/)[0];
if (timeAgo.includes('hours ago')) {
releaseDate.setTime(releaseDate.getTime() - timeAgo);
for (i = 0; i < data.pages_count; i++) {
if (i > 0) {
let chapterHtml = await fetch(chapterListUrl + 'page/' + (i + 1));
let chapterHtmlToString = await chapterHtml.text();
loadedCheerio = cheerio.load(chapterHtmlToString);
json = loadedCheerio('#dle-content > script:nth-child(3)').html();
data = JSON.parse(json.replace('window.__DATA__ =', ''));
}
if (timeAgo.includes('months ago')) {
releaseDate.setMonth(releaseDate.getMonth() - timeAgo);
}
if (timeAgo.includes('days ago')) {
releaseDate.setDate(releaseDate.getDate() - timeAgo);
}
releaseDate = dayjs(releaseDate).format('L');
const chapterUrl = loadedCheerio(this)
.find('a')
.attr('href')
.replace(baseUrl + 'up/', '');
const chapter = {
chapterName,
releaseDate,
chapterUrl,
};
novelChapters.push(chapter);
});
// for (i = 1; i <= lastPage; i++) {
// const chapterPageUrl = `${baseUrl}up/${tempNovelUrl}/page/${i}`;
// const chaptersHtml = await fetch(chapterPageUrl);
// const chapterHtmlToString = await chaptersHtml.text();
// $ = cheerio.load(chapterHtmlToString);
// $("#dle-content > div.cat_block.cat_line").each(function () {
// const chapterName = $(this).find(".title").text();
// let releaseDate = new Date();
// let timeAgo = $(this).find("small").text();
// timeAgo = timeAgo.match(/\d+/)[0];
// if (timeAgo.includes("hours ago")) {
// releaseDate.setTime(releaseDate.getTime() - timeAgo);
// }
// if (timeAgo.includes("months ago")) {
// releaseDate.setMonth(releaseDate.getMonth() - timeAgo);
// }
// if (timeAgo.includes("days ago")) {
// releaseDate.setDate(releaseDate.getDate() - timeAgo);
// }
// releaseDate = dayjs(releaseDate).format("L");
// const chapterUrl = $(this)
// .find("a")
// .attr("href")
// .replace(baseUrl + "up/", "");
// const chapter = {
// chapterName,
// releaseDate,
// chapterUrl,
// };
// novelChapters.push(chapter);
// });
// }
data.chapters.forEach(chapter => {
novelChapters.push({
chapterName: chapter.title,
releaseDate: dayjs(chapter.date).format('LLL'),
chapterUrl: chapter.link,
});
});
}
novel.chapters = novelChapters.reverse();
return novel;
};
const parseChapter = async (novelUrl, chapterUrl) => {
const url = `${baseUrl}up/${chapterUrl}`;
const result = await fetch(url);
const result = await fetch(chapterUrl);
const body = await result.text();
const loadedCheerio = cheerio.load(body);
let chapterName = loadedCheerio('h1.title').text();
let chapterText = loadedCheerio('#arrticle').html();
const chapter = {
sourceId,
novelUrl,
@@ -209,12 +124,35 @@ const parseChapter = async (novelUrl, chapterUrl) => {
};
const searchNovels = async searchTerm => {
/**
* TODO
*/
let url = `${baseUrl}/index.php?story=${searchTerm}&do=search&subaction=search`;
showToast('Ranobes: Search does not work');
return [];
const result = await fetch(url);
const body = await result.text();
let loadedCheerio = cheerio.load(body);
let novels = [];
loadedCheerio('article.block.story.shortstory.mod-poster').each(function () {
const novelName = loadedCheerio(this).find('h2.title').text();
let novelCover = loadedCheerio(this).find('figure').attr('style');
let novelUrl = loadedCheerio(this).find('h2.title a').attr('href');
novelCover = novelCover
.match(/background-image: url(.*);/)[1]
.slice(1, -1)
.replace('/thumbs/', '/');
const novel = {
sourceId,
novelName,
novelCover,
novelUrl,
};
novels.push(novel);
});
return novels;
};
const RanobesScraper = {
src/sources/ru/ranobes.js
+160
View File
@@ -0,0 +1,160 @@
import * as cheerio from 'cheerio';
const sourceId = 132;
const sourceName = 'Ranobes';
const baseUrl = 'https://ranobes.com';
const popularNovels = async page => {
let url = `${baseUrl}/ranobe/page/${page}/`;
const result = await fetch(url);
const body = await result.text();
let loadedCheerio = cheerio.load(body);
const totalPages = parseInt(
loadedCheerio('.pages > a:last-child').text() || '30',
10,
);
let novels = [];
loadedCheerio('article.block.story.shortstory.mod-poster').each(function () {
const novelName = loadedCheerio(this).find('h2.title').text();
let novelCover = loadedCheerio(this).find('figure').attr('style');
let novelUrl = loadedCheerio(this).find('h2.title a').attr('href');
novelCover = novelCover
.match(/background-image: url(.*);/)[1]
.slice(1, -1)
.replace('/thumbs/', '/');
const novel = {
sourceId,
novelName,
novelCover,
novelUrl,
};
novels.push(novel);
});
return { totalPages, novels };
};
const parseNovelAndChapters = async novelUrl => {
const result = await fetch(novelUrl);
const body = await result.text();
let loadedCheerio = cheerio.load(body);
let novel = {
sourceId,
sourceName,
novelUrl,
url: novelUrl,
};
novel.novelName = loadedCheerio('meta[property="og:title"]').attr('content');
novel.novelCover = baseUrl + loadedCheerio('.poster > a > img').attr('src');
novel.summary = loadedCheerio('div[itemprop="description"]').text().trim();
novel.author = loadedCheerio('span[class="tag_list"] > a').text();
let chapterListUrl = loadedCheerio(
'div.r-fullstory-btns a[title~=оглавление]',
).attr('href');
if (!chapterListUrl?.startsWith('http')) {
chapterListUrl = baseUrl + chapterListUrl;
}
const chaptersHtml = await fetch(chapterListUrl);
const chaptersHtmlToString = await chaptersHtml.text();
loadedCheerio = cheerio.load(chaptersHtmlToString);
let novelChapters = [];
let all = loadedCheerio('div.pages a:last-child').text();
all = parseInt(all || '1', 10);
for (i = 0; i < all; i++) {
if (i > 0) {
let chapterHtml = await fetch(chapterListUrl + 'page/' + (i + 1));
let chapterHtmlToString = await chapterHtml.text();
loadedCheerio = cheerio.load(chapterHtmlToString);
}
loadedCheerio('div[class="cat_block cat_line"] a').each(function () {
novelChapters.push({
chapterName: loadedCheerio(this).attr('title'),
releaseDate: null,
chapterUrl: loadedCheerio(this).attr('href'),
});
});
}
novel.chapters = novelChapters.reverse();
return novel;
};
const parseChapter = async (novelUrl, chapterUrl) => {
const result = await fetch(chapterUrl);
const body = await result.text();
const loadedCheerio = cheerio.load(body);
let chapterName = loadedCheerio('h1.title').text();
let chapterText = loadedCheerio('#arrticle').html();
const chapter = {
sourceId,
novelUrl,
chapterUrl,
chapterName,
chapterText,
};
return chapter;
};
const searchNovels = async searchTerm => {
let url = `${baseUrl}/index.php?story=${searchTerm}&do=search&subaction=search`;
const result = await fetch(url);
const body = await result.text();
let loadedCheerio = cheerio.load(body);
let novels = [];
loadedCheerio('article.block.story.shortstory.mod-poster').each(function () {
const novelName = loadedCheerio(this).find('h2.title').text();
let novelCover = loadedCheerio(this).find('figure').attr('style');
let novelUrl = loadedCheerio(this).find('h2.title a').attr('href');
novelCover = novelCover
.match(/background-image: url(.*);/)[1]
.slice(1, -1)
.replace('/thumbs/', '/');
const novel = {
sourceId,
novelName,
novelCover,
novelUrl,
};
novels.push(novel);
});
return novels;
};
const RanobesScraper = {
popularNovels,
parseNovelAndChapters,
parseChapter,
searchNovels,
};
export default RanobesScraper;
src/sources/sourceManager.ts
+2
View File
@@ -78,6 +78,7 @@ import WuxiaBlogScraper from './en/wuxiablog';
import WuxiaCityScraper from './en/wuxiacity';
import NovelUpdatesScraper from './en/novelupdates';
import RanobesScraper from './en/ranobes';
import RanobesruScraper from './ru/ranobes';
import YushuboScraper from './ch/yushubo';
import {
KolNovelScraper,
@@ -286,6 +287,7 @@ export const sourceManager = (sourceId: number): Scraper => {
129: LightNovelsBrasilScraper,
130: RiwyatScraper,
131: NovelsticScraper,
132: RanobesruScraper,
};
return scrapers[sourceId];
src/sources/sources.json
+7
View File
@@ -880,5 +880,12 @@
"icon": "https://github.com/LNReader/lnreader-sources/blob/main/multisrc/madara/icons/novelstic.png?raw=true",
"url": "https://novelstic.com/",
"lang": "English"
},
{
"sourceId": 132,
"url": "https://ranobes.com/",
"sourceName": "Ranobes",
"icon": "https://github.com/LNReader/lnreader-sources/blob/master/icons/src/en/ranobes/icon.png?raw=true",
"lang": "Russian"
}
]
src/utils/constants/dateFormats.ts
-8
View File
@@ -1,8 +0,0 @@
export const dateFormats = {
sameDay: '[Today]',
nextDay: '[Tomorrow]',
nextWeek: 'dddd',
lastDay: '[Yesterday]',
lastWeek: '[Last] dddd',
sameElse: 'L',
};
strings/languages/en/strings.json
+10
View File
@@ -130,5 +130,15 @@
"advancedSettings": {
"deleteReadChapters": "Delete read chapters",
"deleteReadChaptersDialogTitle": "Are you sure? All chapters marked as read will be deleted."
},
"date": {
"calendar": {
"sameDay": "[Today]",
"nextDay": "[Tomorrow]",
"nextWeek": "dddd",
"lastDay": "[Yesterday]",
"lastWeek": "[Last] dddd",
"sameElse": "LL"
}
}
}
strings/translations.ts
+9
View File
@@ -40,3 +40,12 @@ i18n.locale = Localization.locale;
dayjs.locale(Localization.locale);
export const getString = (stringKey: keyof StringMap) => i18n.t(stringKey);
dayjs.Ls[dayjs.locale()].calendar = {
sameDay: getString('date.calendar.sameDay'),
nextDay: getString('date.calendar.nextDay'),
nextWeek: getString('date.calendar.nextWeek'),
lastDay: getString('date.calendar.lastDay'),
lastWeek: getString('date.calendar.lastWeek'),
sameElse: getString('date.calendar.sameElse'),
};
strings/types/index.ts
+6
View File
@@ -94,4 +94,10 @@ export interface StringMap {
'globalSearch.pinnedSources': 'string';
'advancedSettings.deleteReadChapters': 'string';
'advancedSettings.deleteReadChaptersDialogTitle': 'string';
'date.calendar.sameDay': 'string';
'date.calendar.nextDay': 'string';
'date.calendar.nextWeek': 'string';
'date.calendar.lastDay': 'string';
'date.calendar.lastWeek': 'string';
'date.calendar.sameElse': 'string';
}