feat(source): Add Agitoon (#542)

* add: kr sources

* add: latest/popular novels

* bug: parseNovelAndChapters()

* fix: parse parseNovelAndChapters bug and add: parseChapter

* add: search feature

* add: popular gets the popular list of the current day; refactor

* fix: 'chapterId doesnt exist'

* fix: reverted back the package-lock.json file

---------

Co-authored-by: Vigneshwar M <vigneshwarm@Vigneshwars-MacBook-Pro.local>
This commit is contained in:
Vigneshwar M
2023-02-15 22:39:20 +05:30
committed by GitHub
parent a193656cb2
commit 8fe1dacab0
6 changed files with 224 additions and 24 deletions
+48 -24
View File
@@ -18421,7 +18421,8 @@
"@jsamr/react-native-li": {
"version": "2.3.1",
"resolved": "https://registry.npmjs.org/@jsamr/react-native-li/-/react-native-li-2.3.1.tgz",
"integrity": "sha512-Qbo4NEj48SQ4k8FZJHFE2fgZDKTWaUGmVxcIQh3msg5JezLdTMMHuRRDYctfdHI6L0FZGObmEv3haWbIvmol8w=="
"integrity": "sha512-Qbo4NEj48SQ4k8FZJHFE2fgZDKTWaUGmVxcIQh3msg5JezLdTMMHuRRDYctfdHI6L0FZGObmEv3haWbIvmol8w==",
"requires": {}
},
"@native-html/css-processor": {
"version": "1.11.0",
@@ -18862,7 +18863,8 @@
"@react-native-community/clipboard": {
"version": "1.5.1",
"resolved": "https://registry.npmjs.org/@react-native-community/clipboard/-/clipboard-1.5.1.tgz",
"integrity": "sha512-AHAmrkLEH5UtPaDiRqoULERHh3oNv7Dgs0bTC0hO5Z2GdNokAMPT5w8ci8aMcRemcwbtdHjxChgtjbeA38GBdA=="
"integrity": "sha512-AHAmrkLEH5UtPaDiRqoULERHh3oNv7Dgs0bTC0hO5Z2GdNokAMPT5w8ci8aMcRemcwbtdHjxChgtjbeA38GBdA==",
"requires": {}
},
"@react-native-community/eslint-config": {
"version": "3.0.1",
@@ -18933,12 +18935,14 @@
"@react-native-community/masked-view": {
"version": "0.1.11",
"resolved": "https://registry.npmjs.org/@react-native-community/masked-view/-/masked-view-0.1.11.tgz",
"integrity": "sha512-rQfMIGSR/1r/SyN87+VD8xHHzDYeHaJq6elOSCAD+0iLagXkSI2pfA0LmSXP21uw5i3em7GkkRjfJ8wpqWXZNw=="
"integrity": "sha512-rQfMIGSR/1r/SyN87+VD8xHHzDYeHaJq6elOSCAD+0iLagXkSI2pfA0LmSXP21uw5i3em7GkkRjfJ8wpqWXZNw==",
"requires": {}
},
"@react-native-community/slider": {
"version": "4.1.12",
"resolved": "https://registry.npmjs.org/@react-native-community/slider/-/slider-4.1.12.tgz",
"integrity": "sha512-CiuLZ2orueBiWHYxfaJF57jQY6HY2Q3z5pdAE4MKH8EqIImr/jgDJrJ/UxOVZHK1Ng9P+XlGIKfVIcuWZ6guuA=="
"integrity": "sha512-CiuLZ2orueBiWHYxfaJF57jQY6HY2Q3z5pdAE4MKH8EqIImr/jgDJrJ/UxOVZHK1Ng9P+XlGIKfVIcuWZ6guuA==",
"requires": {}
},
"@react-native-cookies/cookies": {
"version": "6.2.1",
@@ -18951,7 +18955,8 @@
"@react-native-picker/picker": {
"version": "2.4.1",
"resolved": "https://registry.npmjs.org/@react-native-picker/picker/-/picker-2.4.1.tgz",
"integrity": "sha512-1XWy3IQgwr7MWd30KdY1iUh2gQZD+JiotN1ifj/ptFUYKon/0UFwngKQaWCO/CP/FdLl20/huSSLwKedYrdMMA=="
"integrity": "sha512-1XWy3IQgwr7MWd30KdY1iUh2gQZD+JiotN1ifj/ptFUYKon/0UFwngKQaWCO/CP/FdLl20/huSSLwKedYrdMMA==",
"requires": {}
},
"@react-native/assets": {
"version": "1.0.0",
@@ -19004,7 +19009,8 @@
"@react-navigation/elements": {
"version": "1.2.1",
"resolved": "https://registry.npmjs.org/@react-navigation/elements/-/elements-1.2.1.tgz",
"integrity": "sha512-EnmAbKMsptrliRKf95rdgS6BhMjML+mIns06+G1Vdih6BrEo7/0iytThUv3WBf99AI76dyEq/cqLUwHPiFzXWg=="
"integrity": "sha512-EnmAbKMsptrliRKf95rdgS6BhMjML+mIns06+G1Vdih6BrEo7/0iytThUv3WBf99AI76dyEq/cqLUwHPiFzXWg==",
"requires": {}
},
"@react-navigation/material-bottom-tabs": {
"version": "6.0.9",
@@ -19647,7 +19653,8 @@
"version": "5.3.2",
"resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz",
"integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==",
"dev": true
"dev": true,
"requires": {}
},
"aggregate-error": {
"version": "3.1.0",
@@ -19892,7 +19899,8 @@
"babel-core": {
"version": "7.0.0-bridge.0",
"resolved": "https://registry.npmjs.org/babel-core/-/babel-core-7.0.0-bridge.0.tgz",
"integrity": "sha512-poPX9mZH/5CSanm50Q+1toVci6pv5KSRv/5TWCwtzQS5XEwn40BcCrgIeMFWP9CKKIniKXNxoIOnOq4VVlGXhg=="
"integrity": "sha512-poPX9mZH/5CSanm50Q+1toVci6pv5KSRv/5TWCwtzQS5XEwn40BcCrgIeMFWP9CKKIniKXNxoIOnOq4VVlGXhg==",
"requires": {}
},
"babel-eslint": {
"version": "10.1.0",
@@ -21480,7 +21488,8 @@
"version": "22.4.1",
"resolved": "https://registry.npmjs.org/eslint-plugin-jest/-/eslint-plugin-jest-22.4.1.tgz",
"integrity": "sha512-gcLfn6P2PrFAVx3AobaOzlIEevpAEf9chTpFZz7bYfc7pz8XRv7vuKTIE4hxPKZSha6XWKKplDQ0x9Pq8xX2mg==",
"dev": true
"dev": true,
"requires": {}
},
"eslint-plugin-prettier": {
"version": "3.1.2",
@@ -21544,7 +21553,8 @@
"version": "4.3.0",
"resolved": "https://registry.npmjs.org/eslint-plugin-react-hooks/-/eslint-plugin-react-hooks-4.3.0.tgz",
"integrity": "sha512-XslZy0LnMn+84NEG9jSGR6eGqaZB3133L8xewQo3fQagbQuGt7a63gf+P1NGKZavEYEC3UXaWEAA/AqDkuN6xA==",
"dev": true
"dev": true,
"requires": {}
},
"eslint-plugin-react-native": {
"version": "3.11.0",
@@ -25782,7 +25792,8 @@
"ws": {
"version": "7.4.6",
"resolved": "https://registry.npmjs.org/ws/-/ws-7.4.6.tgz",
"integrity": "sha512-YmhHDO4MzaDLB+M9ym/mDA5z0naX8j7SIlT8f8z+I0VtzsRbekxEutHSme7NPS2qE8StCYQNUnfWdXta/Yu85A=="
"integrity": "sha512-YmhHDO4MzaDLB+M9ym/mDA5z0naX8j7SIlT8f8z+I0VtzsRbekxEutHSme7NPS2qE8StCYQNUnfWdXta/Yu85A==",
"requires": {}
}
}
},
@@ -26141,22 +26152,26 @@
"react-native-device-info": {
"version": "8.4.7",
"resolved": "https://registry.npmjs.org/react-native-device-info/-/react-native-device-info-8.4.7.tgz",
"integrity": "sha512-I0z7FfbtxKW5+1wWE7L1bzy0iuiQzgUNVUrvpQoVQSz/BpC9kNIwZokC8O3HpUJDkFVr2Bhd5WO3tISzS7OXxw=="
"integrity": "sha512-I0z7FfbtxKW5+1wWE7L1bzy0iuiQzgUNVUrvpQoVQSz/BpC9kNIwZokC8O3HpUJDkFVr2Bhd5WO3tISzS7OXxw==",
"requires": {}
},
"react-native-easing-gradient": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/react-native-easing-gradient/-/react-native-easing-gradient-1.0.0.tgz",
"integrity": "sha512-wkFPbjDLvySHKzqkCYxD8vTgKLdlgch9AL8wkO+liJuukwVXeD2sDqtPPsMKqX6hWK+Ebg8shjIeavNfoOLy7w=="
"integrity": "sha512-wkFPbjDLvySHKzqkCYxD8vTgKLdlgch9AL8wkO+liJuukwVXeD2sDqtPPsMKqX6hWK+Ebg8shjIeavNfoOLy7w==",
"requires": {}
},
"react-native-error-boundary": {
"version": "1.1.16",
"resolved": "https://registry.npmjs.org/react-native-error-boundary/-/react-native-error-boundary-1.1.16.tgz",
"integrity": "sha512-yCbKhjGo2kMJBoUdy503L3oaInrLo1gychGr5e/IrezfPhfUGhfr8EtG2EKgZDcSfmvodBXvARq942hWis5sUA=="
"integrity": "sha512-yCbKhjGo2kMJBoUdy503L3oaInrLo1gychGr5e/IrezfPhfUGhfr8EtG2EKgZDcSfmvodBXvARq942hWis5sUA==",
"requires": {}
},
"react-native-fast-image": {
"version": "8.5.11",
"resolved": "https://registry.npmjs.org/react-native-fast-image/-/react-native-fast-image-8.5.11.tgz",
"integrity": "sha512-cNW4bIJg3nvKaheG8vGMfqCt5LMWX9MS5+wMudgKIHbGO51spRr4sgnlhVgwHLcZ5aeNOVJ8CPRxDIWKRq/0QA=="
"integrity": "sha512-cNW4bIJg3nvKaheG8vGMfqCt5LMWX9MS5+wMudgKIHbGO51spRr4sgnlhVgwHLcZ5aeNOVJ8CPRxDIWKRq/0QA==",
"requires": {}
},
"react-native-fit-image": {
"version": "1.5.5",
@@ -26201,12 +26216,14 @@
"react-native-mmkv": {
"version": "2.4.3",
"resolved": "https://registry.npmjs.org/react-native-mmkv/-/react-native-mmkv-2.4.3.tgz",
"integrity": "sha512-0hYNOTbsjJ5j5cpX+qjwReGIxuYE9MWRsyPQ72fTOvzmEjYXCF4Wr2VQoqx8R6W1/Uinekvln7dGvJ4RExmHhQ=="
"integrity": "sha512-0hYNOTbsjJ5j5cpX+qjwReGIxuYE9MWRsyPQ72fTOvzmEjYXCF4Wr2VQoqx8R6W1/Uinekvln7dGvJ4RExmHhQ==",
"requires": {}
},
"react-native-pager-view": {
"version": "5.4.9",
"resolved": "https://registry.npmjs.org/react-native-pager-view/-/react-native-pager-view-5.4.9.tgz",
"integrity": "sha512-D6tzxpwMGdl6CXgtskGWhKRc5cJakCazESRGt7PkqnpyiH3N35ft1KmR82pCSQetAFlytFiToeu3a/dG5CELvA=="
"integrity": "sha512-D6tzxpwMGdl6CXgtskGWhKRc5cJakCazESRGt7PkqnpyiH3N35ft1KmR82pCSQetAFlytFiToeu3a/dG5CELvA==",
"requires": {}
},
"react-native-paper": {
"version": "5.1.2",
@@ -26274,7 +26291,8 @@
"react-native-safe-area-context": {
"version": "3.2.0",
"resolved": "https://registry.npmjs.org/react-native-safe-area-context/-/react-native-safe-area-context-3.2.0.tgz",
"integrity": "sha512-k2Nty4PwSnrg9HwrYeeE+EYqViYJoOFwEy9LxL5RIRfoqxAq/uQXNGwpUg2/u4gnKpBbEPa9eRh15KKMe/VHkA=="
"integrity": "sha512-k2Nty4PwSnrg9HwrYeeE+EYqViYJoOFwEy9LxL5RIRfoqxAq/uQXNGwpUg2/u4gnKpBbEPa9eRh15KKMe/VHkA==",
"requires": {}
},
"react-native-safe-modules": {
"version": "1.0.3",
@@ -26296,14 +26314,16 @@
"react-freeze": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/react-freeze/-/react-freeze-1.0.0.tgz",
"integrity": "sha512-yQaiOqDmoKqks56LN9MTgY06O0qQHgV4FUrikH357DydArSZHQhl0BJFqGKIZoTqi8JizF9Dxhuk1FIZD6qCaw=="
"integrity": "sha512-yQaiOqDmoKqks56LN9MTgY06O0qQHgV4FUrikH357DydArSZHQhl0BJFqGKIZoTqi8JizF9Dxhuk1FIZD6qCaw==",
"requires": {}
}
}
},
"react-native-shimmer-placeholder": {
"version": "2.0.9",
"resolved": "https://registry.npmjs.org/react-native-shimmer-placeholder/-/react-native-shimmer-placeholder-2.0.9.tgz",
"integrity": "sha512-s2pfAAO6uaybREYM2KcaxP3c2XlZvLfHfzmhMlGEoOhSKaoM92KpA1Hx0BeC75yilkliJuowqO/hFzvLB0h7hg=="
"integrity": "sha512-s2pfAAO6uaybREYM2KcaxP3c2XlZvLfHfzmhMlGEoOhSKaoM92KpA1Hx0BeC75yilkliJuowqO/hFzvLB0h7hg==",
"requires": {}
},
"react-native-swipe-gestures": {
"version": "1.0.5",
@@ -26313,7 +26333,8 @@
"react-native-tab-view": {
"version": "3.1.1",
"resolved": "https://registry.npmjs.org/react-native-tab-view/-/react-native-tab-view-3.1.1.tgz",
"integrity": "sha512-M5pRN6utQfytKWoKlKVzg5NbkYu308qNoW1khGTtEOTs1k14p2dHJ/BWOJoJYHKbPVUyZldbG9MFT7gUl4YHnw=="
"integrity": "sha512-M5pRN6utQfytKWoKlKVzg5NbkYu308qNoW1khGTtEOTs1k14p2dHJ/BWOJoJYHKbPVUyZldbG9MFT7gUl4YHnw==",
"requires": {}
},
"react-native-tts": {
"version": "4.1.0",
@@ -26563,12 +26584,14 @@
"redux-persist": {
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/redux-persist/-/redux-persist-6.0.0.tgz",
"integrity": "sha512-71LLMbUq2r02ng2We9S215LtPu3fY0KgaGE0k8WRgl6RkqxtGfl7HUozz1Dftwsb0D/5mZ8dwAaPbtnzfvbEwQ=="
"integrity": "sha512-71LLMbUq2r02ng2We9S215LtPu3fY0KgaGE0k8WRgl6RkqxtGfl7HUozz1Dftwsb0D/5mZ8dwAaPbtnzfvbEwQ==",
"requires": {}
},
"redux-thunk": {
"version": "2.4.1",
"resolved": "https://registry.npmjs.org/redux-thunk/-/redux-thunk-2.4.1.tgz",
"integrity": "sha512-OOYGNY5Jy2TWvTL1KgAlVy6dcx3siPJ1wTq741EPyUKfn6W6nChdICjZwCd0p8AZBs5kWpZlbkXW2nE/zjUa+Q=="
"integrity": "sha512-OOYGNY5Jy2TWvTL1KgAlVy6dcx3siPJ1wTq741EPyUKfn6W6nChdICjZwCd0p8AZBs5kWpZlbkXW2nE/zjUa+Q==",
"requires": {}
},
"regenerate": {
"version": "1.4.2",
@@ -28105,7 +28128,8 @@
"use-event-callback": {
"version": "0.1.0",
"resolved": "https://registry.npmjs.org/use-event-callback/-/use-event-callback-0.1.0.tgz",
"integrity": "sha512-5fTzY5UEXHMK5UR0NRkUz6TPfWmmX9fO8Tx3SnHrfMPdrQ7Rna0gDBy0r56SP68TwsP9DgwSBzeysCu3A/Z2NA=="
"integrity": "sha512-5fTzY5UEXHMK5UR0NRkUz6TPfWmmX9fO8Tx3SnHrfMPdrQ7Rna0gDBy0r56SP68TwsP9DgwSBzeysCu3A/Z2NA==",
"requires": {}
},
"use-subscription": {
"version": "1.5.1",
+1
View File
@@ -18,6 +18,7 @@ const getLanguageCode = language => {
Chinese: 'cn',
Japanese: 'jp',
Vietnamese: 'vi',
Korean: 'kr',
};
return codes[language];
+165
View File
@@ -0,0 +1,165 @@
import * as cheerio from 'cheerio';
const sourceId = 145;
const sourceName = 'Agitoon';
const baseUrl = 'https://agit501.xyz/';
const popularNovels = async page => {
const totalPages = 20;
const list_limit = 20 * (page - 1);
const day = new Date().getDay();
const res = await fetch(baseUrl + 'novel/index.update.php', {
headers: {
'content-type': 'application/x-www-form-urlencoded; charset=UTF-8',
},
body: `mode=get_data_novel_list_p&novel_menu=3&np_day=${day}&np_rank=1&np_distributor=0&np_genre=00&np_order=1&np_genre_ex_1=00&np_genre_ex_2=00&list_limit=${list_limit}&is_query_first=true`,
method: 'POST',
});
const resJson = await res.json();
const novels = resJson.list.map(novel => {
return {
sourceId,
novelUrl: baseUrl + 'novel/list/' + novel.wr_id,
novelName: novel.wr_subject,
novelCover: baseUrl + novel.np_dir + '/thumbnail/' + novel.np_thumbnail,
};
});
return { totalPages, novels };
};
const parseNovelAndChapters = async novelUrl => {
const novelId = novelUrl.split('/').reverse()[0];
// cheerio
const result = await fetch(novelUrl);
const body = await result.text();
const loadedCheerio = cheerio.load(body, { decodeEntities: false });
const novelName = loadedCheerio('h5.pt-2').text();
const summary = loadedCheerio('.pt-1.mt-1.pb-1.mb-1').text();
const author = loadedCheerio('.post-item-list-cate-v')
.first()
.text()
.split(' : ')
.reverse()[0];
const novelCover =
baseUrl.slice(0, baseUrl.length - 1) +
loadedCheerio('div.col-5.pr-0.pl-0 img').attr('src');
const genresTag = loadedCheerio('.col-7 > .post-item-list-cate > span');
let genres = '';
genresTag.each((_, element) => {
genres += loadedCheerio(element).text();
genres += ', ';
});
genres = genres.slice(0, genres.length - 2);
// normal REST HTTP requests
let chapters = [];
const res = await fetch(baseUrl + 'novel/list.update.php', {
headers: {
'content-type': 'application/x-www-form-urlencoded; charset=UTF-8',
},
body: `mode=get_data_novel_list_c&wr_id_p=${novelId}&page_no=1&cnt_list=10000&order_type=Asc`,
method: 'POST',
});
const resJson = await res.json();
chapters = resJson.list.map(chapter => {
return {
chapterId: chapter.wr_id,
chapterName: chapter.wr_subject,
chapterUrl: baseUrl + `novel/view/${chapter.wr_id}/2`,
releaseDate: chapter.wr_datetime,
};
});
const novel = {
sourceId,
sourceName,
novelUrl,
novelName,
novelCover,
summary,
author,
status: '',
genre: genres,
url: novelUrl,
chapters,
};
return novel;
};
const parseChapter = async (novelUrl, chapterUrl) => {
const result = await fetch(chapterUrl);
const body = await result.text();
const loadedCheerio = cheerio.load(body);
const title = loadedCheerio('div > div.col-12 > h2').text();
const contentTag = loadedCheerio('#id_wr_content > p');
let content = '';
contentTag.each((_, element) => {
content += loadedCheerio(element).text();
content += '<br />';
});
// gets rid of the popup thingy
content = content.replace(
'팝업메뉴는 빈공간을 더치하거나 스크룰시 사라집니다',
'',
);
const chapter = {
sourceId,
novelUrl,
chapterUrl,
chapterName: title,
chapterText: content,
};
return chapter;
};
const searchNovels = async searchTerm => {
const rawResults = await fetch('https://agit501.xyz/novel/search.php', {
headers: {
'content-type': 'application/x-www-form-urlencoded; charset=UTF-8',
},
body: `mode=get_data_novel_list_p_sch&search_novel=${searchTerm}&list_limit=0`,
method: 'POST',
});
const results = await rawResults.json();
if (!results.list && results.list.length() === 0) {
return [];
}
const novels = results.list.map(result => {
return {
sourceId,
novelUrl: baseUrl + 'novel/list/' + result.wr_id,
novelName: result.wr_subject,
novelCover: baseUrl + result.np_dir + '/thumbnail/' + result.np_thumbnail,
};
});
return novels;
};
const AgitoonScraper = {
popularNovels,
parseNovelAndChapters,
parseChapter,
searchNovels,
};
export default AgitoonScraper;
+2
View File
@@ -145,6 +145,7 @@ import MTNovelScraper from './en/mtnovel';
import NovelTop1Scraper from './en/noveltop1';
import { SelectedFilter, SourceFilter } from './types/filterTypes';
import KolNovelScraper from './ar/KolNovel';
import AgitoonScraper from './kr/Agitoon';
interface PopularNovelsResponse {
totalPages: number;
@@ -310,6 +311,7 @@ export const sourceManager = (sourceId: number): Scraper => {
142: AuthorTodayScraper,
143: WebNovelOkuScraper,
144: NobleMtlScraper,
145: AgitoonScraper,
};
return scrapers[sourceId];
+7
View File
@@ -964,5 +964,12 @@
"icon": "https://i1.wp.com/noblemtl.com/wp-content/uploads/2022/07/cropped-Noble-192x192.png",
"url": "https://noblemtl.com/",
"lang": "English"
},
{
"sourceId": 145,
"sourceName": "Agitoon",
"icon": "https://agit501.xyz/asset/img/agit2.png?v=2",
"url": "https://agit501.xyz/",
"lang": "Korean"
}
]
+1
View File
@@ -5,6 +5,7 @@ export const languages = [
'French',
'Indonesian',
'Japanese',
'Korean',
'Portuguese (Brazil)',
'Russian',
'Spanish',