lnreader-plugins/src/jp/syosetu.js

import cheerio from "react-native-cheerio";

import moment from "moment";

const baseUrl = "https://syosetu.com"; // base url for syosetu.com

// get given page of search (if pagenum is 0 or >100 (max possible on site) see first page)
const searchUrl = (pagenum, order) => {
    return `https://yomou.syosetu.com/search.php?order=${order || "hyoka"}${
        !isNaN((pagenum = parseInt(pagenum))) // check if pagenum is a number
            ? `&p=${pagenum <= 1 || pagenum > 100 ? "1" : pagenum}` // check if pagenum is between 1 and 100
            : "" // if isn't don't set ?p
    }`;
};

// get the Syosetu ID of the chapter
const getLastPartOfUrl = (url) =>
    (ex = /.*(?=\/)(.{2,})$/.exec(url)) ? ex[1].replace(/\//g, "") : null;

// get novelUrl from Syosetu ID
const getNovelUrl = (id) => `https://ncode.syosetu.com/${id}`;

// get chapterUrl from Syosetu ID and chapter ID
const getChapterUrl = (id, chn) =>
    `https://ncode.syosetu.com/${id}${chn === "oneshot" ? "" : `/${chn}`}`;

// Because syosetu does not have any covers, I did a replacement for them
const novelCover =
    "https://raw.githubusercontent.com/skillgg/lnreader-sources/main/src/jp/syosetu/not_found.png";

// ID of this extension
const sourceId = 36;

// Name of this extension
const sourceName = "Syosetu";

// there are 20 mangas per page
// this is number of pages loaded in the first "batch" of loads
// TOOD: in-app dynamic loading on scroll
const maxPageLoad = 3;

const popularNovels = async (page) => {
    const totalPages = 100;
    // array of all the novels
    let novels = [];
    // returns list of novels from given page
    let getNovelsFromPage = async (pagenumber) => {
        // load page
        const result = await fetch(searchUrl(pagenumber || null));
        const body = await result.text();
        // Cheerio it!
        const cheerioQuery = cheerio.load(body, { decodeEntities: false });

        let pageNovels = [];
        // find class=searchkekka_box
        cheerioQuery(".searchkekka_box").each(function (i, e) {
            // get div with link and name
            const novelDIV = cheerioQuery(this).find(".novel_h");
            // get link element
            const novelA = novelDIV.children()[0];
            // add new novel to array
            pageNovels.push({
                novelName: novelDIV.text(), // get the name
                novelUrl: getLastPartOfUrl(novelA.attribs.href), // get last part of the link
                sourceId,
                novelCover, // TODO: IDK what to do about covers... On Syo they don't have them
            });
        });
        // return all novels from this page
        return pageNovels;
    };

    novels = await getNovelsFromPage(page);
    /** Use
     * novels.push(...(await getNovelsFromPage(pageNumber)))
     * if you want to load more
     */

    // respond with novels!
    return { totalPages, novels };
};

const parseNovelAndChapters = async (novelUrl) => {
    const url = getNovelUrl(novelUrl);

    let chapters = [];

    const result = await fetch(url);
    const body = await result.text();
    const cheerioQuery = cheerio.load(body, { decodeEntities: false });

    // create novel object
    let novel = {
        sourceId,
        sourceName,
        url: url,
        novelUrl,
        novelName: cheerioQuery(".novel_title").text(),
        author: cheerioQuery(".novel_writername").text().replace("作者：", ""),
        novelCover,
    };

    // Get all the chapters
    const cqGetChapters = cheerioQuery(".novel_sublist2");
    if (cqGetChapters.length !== 0) {
        // has more than 1 chapter
        novel.summary = cheerioQuery("#novel_ex")
            .text()
            .replace(/<\s*br.*?>/g, "\n");
        cqGetChapters.each(function (i, e) {
            const chapterA = cheerioQuery(this).find("a");
            const [chapterName, releaseDate, chapterUrl] = [
                // set the variables
                chapterA.text(),
                cheerioQuery(this)
                    .find("dt") // get title
                    .text() // get text
                    .replace(/（.）/g, "") // remove "(edited)" mark
                    .trim(), // trim spaces
                getLastPartOfUrl(chapterA.attr("href")),
            ];
            chapters.push({ chapterName, releaseDate, chapterUrl });
        });
    } else {
        /**
         * Because there are oneshots on the site, they have to be treated with special care
         * that's what pisses me off in Shosetsu app. They have this extension,
         * but every oneshot is set as "there are no chapters" and all contents are thrown into the description!!
         */
        // get summary for oneshot chapter

        const result = await fetch(searchUrl() + `&word=${novel.novelName}`);
        const body = await result.text();
        const summaryQuery = cheerio.load(body, { decodeEntities: false });
        const foundText = summaryQuery(".searchkekka_box")
            .first()
            .find(".ex")
            .text()
            .replace(/\s{2,}/g, "\n");
        novel.summary = foundText;

        // add single chapter
        chapters.push({
            chapterName: "Oneshot",
            releaseDate: cheerioQuery("head")
                .find("meta[name='WWWC']")
                .attr("content"), // get date from metadata
            chapterUrl: "oneshot", // set chapterUrl to oneshot so that chapterScraper knows it's a one-shot
        });
    }
    novel.chapters = chapters;

    return novel;
};

let parseChapter = async (novelUrl, chapterUrl) => {
    const url = getChapterUrl(novelUrl, chapterUrl); // get Url

    const result = await fetch(url);
    const body = await result.text();

    // create cheerioQuery
    const cheerioQuery = cheerio.load(body, {
        decodeEntities: false,
    });

    let chapterText = cheerioQuery("#novel_honbun") // get chapter text
        .html();
    // create chapter data structure
    let chapter = {
        sourceId: 36,
        novelUrl,
        chapterUrl,
        chapterName: "",
        chapterText,
    };

    if (chapterUrl === "oneshot")
        // oneshot get name
        chapter.chapterName = cheerioQuery("#novel_title").text();
    else {
        // single chapter

        // get name
        chapter.chapterName = cheerioQuery(".novel_subtitle").first().text();

        // get next/prev buttons
        const chapterButtons = cheerioQuery(
            "#novel_contents .novel_bn"
        ).first();
    }

    return chapter;
};

let searchNovels = async (searchTerm) => {
    // const orderBy = req.query.o;

    // array of all the novels
    let novels = [];

    let isNext = true;

    // returns list of novels from given page
    let getNovelsFromPage = async (pagenumber) => {
        // load page
        const result = await fetch(
            searchUrl(pagenumber || null, null) + `&word=${searchTerm}`
        );
        const body = await result.text();
        // Cheerio it!
        const cheerioQuery = cheerio.load(body, { decodeEntities: false });

        if (cheerioQuery(".nextlink").length === 0) isNext = false;

        let pageNovels = [];
        // find class=searchkekka_box
        cheerioQuery(".searchkekka_box").each(function (i, e) {
            // get div with link and name
            const novelDIV = cheerioQuery(this).find(".novel_h");
            // get link element
            const novelA = novelDIV.children()[0];
            // add new novel to array
            pageNovels.push({
                novelName: novelDIV.text(), // get the name
                novelUrl: getLastPartOfUrl(novelA.attribs.href), // get last part of the link
                sourceId,
                novelCover, // TODO: IDK what to do about covers... On Syo they don't have them
            });
        });
        // return all novels from this page
        return pageNovels;
    };

    // counter of loaded pages
    // let pagesLoaded = 0;
    // do {
    //     // always load first one
    //     novels.push(...(await getNovelsFromPage(pagesLoaded + 1)));
    //     pagesLoaded++;
    // } while (pagesLoaded < maxPageLoad && isNext); // check if we should load more

    novels = await getNovelsFromPage(1);

    /** Use
     * novels.push(...(await getNovelsFromPage(pageNumber)))
     * if you want to load more
     */

    // respond with novels!
    return novels;
};

const SyosetuScraper = {
    popularNovels,
    parseNovelAndChapters,
    parseChapter,
    searchNovels,
};

export default SyosetuScraper;