Create v2
This commit is contained in:
Rajarshee Chatterjee
@@ -0,0 +1,261 @@
|
||||
import cheerio from "react-native-cheerio";
|
||||
|
||||
import moment from "moment";
|
||||
|
||||
const baseUrl = "https://syosetu.com"; // base url for syosetu.com
|
||||
|
||||
// get given page of search (if pagenum is 0 or >100 (max possible on site) see first page)
|
||||
const searchUrl = (pagenum, order) => {
|
||||
return `https://yomou.syosetu.com/search.php?order=${order || "hyoka"}${
|
||||
!isNaN((pagenum = parseInt(pagenum))) // check if pagenum is a number
|
||||
? `&p=${pagenum <= 1 || pagenum > 100 ? "1" : pagenum}` // check if pagenum is between 1 and 100
|
||||
: "" // if isn't don't set ?p
|
||||
}`;
|
||||
};
|
||||
|
||||
// get the Syosetu ID of the chapter
|
||||
const getLastPartOfUrl = (url) =>
|
||||
(ex = /.*(?=\/)(.{2,})$/.exec(url)) ? ex[1].replace(/\//g, "") : null;
|
||||
|
||||
// get novelUrl from Syosetu ID
|
||||
const getNovelUrl = (id) => `https://ncode.syosetu.com/${id}`;
|
||||
|
||||
// get chapterUrl from Syosetu ID and chapter ID
|
||||
const getChapterUrl = (id, chn) =>
|
||||
`https://ncode.syosetu.com/${id}${chn === "oneshot" ? "" : `/${chn}`}`;
|
||||
|
||||
// Because syosetu does not have any covers, I did a replacement for them
|
||||
const novelCover =
|
||||
"https://raw.githubusercontent.com/skillgg/lnreader-sources/main/src/jp/syosetu/not_found.png";
|
||||
|
||||
// ID of this extension
|
||||
const sourceId = 36;
|
||||
|
||||
// Name of this extension
|
||||
const sourceName = "Syosetu";
|
||||
|
||||
// there are 20 mangas per page
|
||||
// this is number of pages loaded in the first "batch" of loads
|
||||
// TOOD: in-app dynamic loading on scroll
|
||||
const maxPageLoad = 3;
|
||||
|
||||
const popularNovels = async (page) => {
|
||||
const totalPages = 100;
|
||||
// array of all the novels
|
||||
let novels = [];
|
||||
// returns list of novels from given page
|
||||
let getNovelsFromPage = async (pagenumber) => {
|
||||
// load page
|
||||
const result = await fetch(searchUrl(pagenumber || null));
|
||||
const body = await result.text();
|
||||
// Cheerio it!
|
||||
const cheerioQuery = cheerio.load(body, { decodeEntities: false });
|
||||
|
||||
let pageNovels = [];
|
||||
// find class=searchkekka_box
|
||||
cheerioQuery(".searchkekka_box").each(function (i, e) {
|
||||
// get div with link and name
|
||||
const novelDIV = cheerioQuery(this).find(".novel_h");
|
||||
// get link element
|
||||
const novelA = novelDIV.children()[0];
|
||||
// add new novel to array
|
||||
pageNovels.push({
|
||||
novelName: novelDIV.text(), // get the name
|
||||
novelUrl: getLastPartOfUrl(novelA.attribs.href), // get last part of the link
|
||||
sourceId,
|
||||
novelCover, // TODO: IDK what to do about covers... On Syo they don't have them
|
||||
});
|
||||
});
|
||||
// return all novels from this page
|
||||
return pageNovels;
|
||||
};
|
||||
|
||||
novels = await getNovelsFromPage(page);
|
||||
/** Use
|
||||
* novels.push(...(await getNovelsFromPage(pageNumber)))
|
||||
* if you want to load more
|
||||
*/
|
||||
|
||||
// respond with novels!
|
||||
return { totalPages, novels };
|
||||
};
|
||||
|
||||
const parseNovelAndChapters = async (novelUrl) => {
|
||||
const url = getNovelUrl(novelUrl);
|
||||
|
||||
let chapters = [];
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
const cheerioQuery = cheerio.load(body, { decodeEntities: false });
|
||||
|
||||
// create novel object
|
||||
let novel = {
|
||||
sourceId,
|
||||
sourceName,
|
||||
url: url,
|
||||
novelUrl,
|
||||
novelName: cheerioQuery(".novel_title").text(),
|
||||
author: cheerioQuery(".novel_writername").text().replace("作者：", ""),
|
||||
novelCover,
|
||||
};
|
||||
|
||||
// Get all the chapters
|
||||
const cqGetChapters = cheerioQuery(".novel_sublist2");
|
||||
if (cqGetChapters.length !== 0) {
|
||||
// has more than 1 chapter
|
||||
novel.summary = cheerioQuery("#novel_ex")
|
||||
.text()
|
||||
.replace(/<\s*br.*?>/g, "\n");
|
||||
cqGetChapters.each(function (i, e) {
|
||||
const chapterA = cheerioQuery(this).find("a");
|
||||
const [chapterName, releaseDate, chapterUrl] = [
|
||||
// set the variables
|
||||
chapterA.text(),
|
||||
cheerioQuery(this)
|
||||
.find("dt") // get title
|
||||
.text() // get text
|
||||
.replace(/（.）/g, "") // remove "(edited)" mark
|
||||
.trim(), // trim spaces
|
||||
getLastPartOfUrl(chapterA.attr("href")),
|
||||
];
|
||||
chapters.push({ chapterName, releaseDate, chapterUrl });
|
||||
});
|
||||
} else {
|
||||
/**
|
||||
* Because there are oneshots on the site, they have to be treated with special care
|
||||
* that's what pisses me off in Shosetsu app. They have this extension,
|
||||
* but every oneshot is set as "there are no chapters" and all contents are thrown into the description!!
|
||||
*/
|
||||
// get summary for oneshot chapter
|
||||
|
||||
const result = await fetch(searchUrl() + `&word=${novel.novelName}`);
|
||||
const body = await result.text();
|
||||
const summaryQuery = cheerio.load(body, { decodeEntities: false });
|
||||
const foundText = summaryQuery(".searchkekka_box")
|
||||
.first()
|
||||
.find(".ex")
|
||||
.text()
|
||||
.replace(/\s{2,}/g, "\n");
|
||||
novel.summary = foundText;
|
||||
|
||||
// add single chapter
|
||||
chapters.push({
|
||||
chapterName: "Oneshot",
|
||||
releaseDate: cheerioQuery("head")
|
||||
.find("meta[name='WWWC']")
|
||||
.attr("content"), // get date from metadata
|
||||
chapterUrl: "oneshot", // set chapterUrl to oneshot so that chapterScraper knows it's a one-shot
|
||||
});
|
||||
}
|
||||
novel.chapters = chapters;
|
||||
|
||||
return novel;
|
||||
};
|
||||
|
||||
let parseChapter = async (novelUrl, chapterUrl) => {
|
||||
const url = getChapterUrl(novelUrl, chapterUrl); // get Url
|
||||
|
||||
const result = await fetch(url);
|
||||
const body = await result.text();
|
||||
|
||||
// create cheerioQuery
|
||||
const cheerioQuery = cheerio.load(body, {
|
||||
decodeEntities: false,
|
||||
});
|
||||
|
||||
let chapterText = cheerioQuery("#novel_honbun") // get chapter text
|
||||
.html();
|
||||
// create chapter data structure
|
||||
let chapter = {
|
||||
sourceId: 36,
|
||||
novelUrl,
|
||||
chapterUrl,
|
||||
chapterName: "",
|
||||
chapterText,
|
||||
};
|
||||
|
||||
if (chapterUrl === "oneshot")
|
||||
// oneshot get name
|
||||
chapter.chapterName = cheerioQuery("#novel_title").text();
|
||||
else {
|
||||
// single chapter
|
||||
|
||||
// get name
|
||||
chapter.chapterName = cheerioQuery(".novel_subtitle").first().text();
|
||||
|
||||
// get next/prev buttons
|
||||
const chapterButtons = cheerioQuery(
|
||||
"#novel_contents .novel_bn"
|
||||
).first();
|
||||
}
|
||||
|
||||
return chapter;
|
||||
};
|
||||
|
||||
let searchNovels = async (searchTerm) => {
|
||||
// const orderBy = req.query.o;
|
||||
|
||||
// array of all the novels
|
||||
let novels = [];
|
||||
|
||||
let isNext = true;
|
||||
|
||||
// returns list of novels from given page
|
||||
let getNovelsFromPage = async (pagenumber) => {
|
||||
// load page
|
||||
const result = await fetch(
|
||||
searchUrl(pagenumber || null, null) + `&word=${searchTerm}`
|
||||
);
|
||||
const body = await result.text();
|
||||
// Cheerio it!
|
||||
const cheerioQuery = cheerio.load(body, { decodeEntities: false });
|
||||
|
||||
if (cheerioQuery(".nextlink").length === 0) isNext = false;
|
||||
|
||||
let pageNovels = [];
|
||||
// find class=searchkekka_box
|
||||
cheerioQuery(".searchkekka_box").each(function (i, e) {
|
||||
// get div with link and name
|
||||
const novelDIV = cheerioQuery(this).find(".novel_h");
|
||||
// get link element
|
||||
const novelA = novelDIV.children()[0];
|
||||
// add new novel to array
|
||||
pageNovels.push({
|
||||
novelName: novelDIV.text(), // get the name
|
||||
novelUrl: getLastPartOfUrl(novelA.attribs.href), // get last part of the link
|
||||
sourceId,
|
||||
novelCover, // TODO: IDK what to do about covers... On Syo they don't have them
|
||||
});
|
||||
});
|
||||
// return all novels from this page
|
||||
return pageNovels;
|
||||
};
|
||||
|
||||
// counter of loaded pages
|
||||
// let pagesLoaded = 0;
|
||||
// do {
|
||||
// // always load first one
|
||||
// novels.push(...(await getNovelsFromPage(pagesLoaded + 1)));
|
||||
// pagesLoaded++;
|
||||
// } while (pagesLoaded < maxPageLoad && isNext); // check if we should load more
|
||||
|
||||
novels = await getNovelsFromPage(1);
|
||||
|
||||
/** Use
|
||||
* novels.push(...(await getNovelsFromPage(pageNumber)))
|
||||
* if you want to load more
|
||||
*/
|
||||
|
||||
// respond with novels!
|
||||
return novels;
|
||||
};
|
||||
|
||||
const SyosetuScraper = {
|
||||
popularNovels,
|
||||
parseNovelAndChapters,
|
||||
parseChapter,
|
||||
searchNovels,
|
||||
};
|
||||
|
||||
export default SyosetuScraper;
|
||||
Reference in New Issue
Block a user