From e1207c172a89f6578411b39813575d26ce90ef27 Mon Sep 17 00:00:00 2001 From: Marcin Majewski Date: Sun, 4 Feb 2024 17:39:07 +0100 Subject: [PATCH] Add accordion and fix minor issues (#742) * Add docs v0.0.1 * Add icon size warning * Fix sourcenover caution * Fix semver descriptions * Filters fix * Quickfix quickstart * Add icon warning v2 * Fix typos * Add NovelItem * Update plugins host * Update plugins host * Remove unnecesarry .js and .dist * Update plugins host * Add host.ps1 * Fix readme.txt * Fix READNE.txt, again * Remove .js and .dist * Add accordion * Working pagination * readme update with ost * Add linux/windows hosting functions * Add tswatch * Fix fetch function * Change plugin versions * Fix chrome * Fix workflows * Fix search not showing results * remove unnecesarry code * Revert novelupdates changes * Reverted NobleMTL instead of novelupdates, fixing that * Revert nobleMTL to e346c7d0 * WTF NobleMTL? --- index.ts | 85 +- libs/fetch.ts | 39 +- package.json | 2 +- plugins/english/nobleMTL.ts | 26 +- plugins/japanese/Syosetu.ts | 165 +- test_web/api/plugins.ts | 17 +- test_web/index.html | 478 ++--- test_web/static/css/accordion.less | 72 + test_web/static/css/darkmode.less | 55 +- test_web/static/css/index.less | 602 +++--- test_web/static/css/index_old.less | 445 +++++ test_web/static/js/accordion.js | 92 + test_web/static/js/index.js | 2799 +++++++++++++++------------- test_web/static/js/index_old.js | 1315 +++++++++++++ test_web/static/js/types.d.ts | 1 + tswatch.ts | 18 + 16 files changed, 4339 insertions(+), 1872 deletions(-) create mode 100644 test_web/static/css/accordion.less create mode 100644 test_web/static/css/index_old.less create mode 100644 test_web/static/js/accordion.js create mode 100644 test_web/static/js/index_old.js create mode 100644 tswatch.ts diff --git a/index.ts b/index.ts index 29d14e1..89f47b6 100644 --- a/index.ts +++ b/index.ts @@ -9,7 +9,7 @@ import localizedFormat from "dayjs/plugin/localizedFormat"; const app = express(); const port = Number(process.argv[2]); -if(!port){ +if (!port) { process.exit(0); } const host = "localhost"; @@ -23,7 +23,9 @@ export const getHeaders = () => { app.use((req, res, next) => { const xHeaders = req.headers["x-custom-headers"]; if (xHeaders && typeof xHeaders === "string") { - for (const [headerName, headerValue] of Object.entries(pluginApi.fetchHeaders)) { + for (const [headerName, headerValue] of Object.entries( + pluginApi.fetchHeaders + )) { if ( typeof headerName === "string" && typeof headerValue === "string" @@ -56,41 +58,68 @@ app.post("/filters", async (req, res) => { res.json(filters || []); }); app.post("/popularNovels/", async (req, res) => { - const filters = req.body["filters"] || {}; + const requirePath = req.body["pluginRequirePath"]; const showLatestNovels = req.body["showLatestNovels"] || false; - const novels = await pluginApi.popularNovels( - req.body["pluginRequirePath"], - { showLatestNovels, filters } - ); - res.json(novels); + const defaultFilters = await pluginApi.getFilter(requirePath); + const filters = showLatestNovels + ? defaultFilters + : req.body["filters"] || defaultFilters; + const page = parseInt(req.body["page"]); + try { + const novels = await pluginApi.popularNovels(requirePath, page || 1, { + showLatestNovels, + filters, + }); + res.json(novels); + } catch (err) { + res.json({ error: err }); + } }); app.post("/searchNovels/", async (req, res) => { - const novels = await pluginApi.searchNovels( - req.body["pluginRequirePath"], - req.body["searchTerm"] - ); - res.json(novels); + const page = parseInt(req.body["page"]) || 1; + try { + const novels = await pluginApi.searchNovels( + req.body["pluginRequirePath"], + page, + req.body["searchTerm"] + ); + res.json(novels); + } catch (err) { + res.json({ error: err }); + } }); app.post("/parseNovelAndChapters/", async (req, res) => { - const sourceNovel = await pluginApi.parseNovelAndChapters( - req.body["pluginRequirePath"], - req.body["novelUrl"] - ); - res.json(sourceNovel); + try { + const sourceNovel = await pluginApi.parseNovelAndChapters( + req.body["pluginRequirePath"], + req.body["novelUrl"] + ); + res.json(sourceNovel); + } catch (err) { + res.json({ error: err }); + } }); app.post("/parseChapter/", async (req, res) => { - const chapterText = await pluginApi.parseChapter( - req.body["pluginRequirePath"], - req.body["chapterUrl"] - ); - res.send(chapterText); + try { + const chapterText = await pluginApi.parseChapter( + req.body["pluginRequirePath"], + req.body["chapterUrl"] + ); + res.send(chapterText); + } catch (err) { + res.json({ error: err }); + } }); app.post("/fetchImage/", async (req, res) => { - const base64 = await pluginApi.fetchImage( - req.body["pluginRequirePath"], - req.body["url"] - ); - res.send(base64); + try { + const base64 = await pluginApi.fetchImage( + req.body["pluginRequirePath"], + req.body["url"] + ); + res.send(base64); + } catch (err) { + res.json({ error: err }); + } }); app.get("/", (req, res) => { diff --git a/libs/fetch.ts b/libs/fetch.ts index 4a58abb..d829019 100644 --- a/libs/fetch.ts +++ b/libs/fetch.ts @@ -1,8 +1,13 @@ type FetchInit = { - headers?: Record | Headers; + headers?: Record | Headers; method?: string; body?: FormData | string; - [x: string]: string | Record | undefined | FormData | Headers; + [x: string]: + | string + | Record + | undefined + | FormData + | Headers; }; const makeInit = async (init?: FetchInit) => { @@ -31,21 +36,19 @@ const makeInit = async (init?: FetchInit) => { }; } return init; -} +}; /** * Fetch with (Android) User Agent - * @param url - * @param init + * @param url + * @param init * @returns response as normal fetch */ -export async function fetchApi( - url: string, - init?: FetchInit -) { +export async function fetchApi(url: string, init?: FetchInit) { init = await makeInit(init); - console.log(url, init); - return await fetch(url, init); + // console.log(init.headers); + // console.log(url, init); + return await fetch(url, init as RequestInit); } /** @@ -59,12 +62,12 @@ export const fetchFile = async function (url: string, init?: FetchInit) { init = await makeInit(init); console.log(url, init); try { - const res = await fetch(url, init); - if (!res.ok) return ''; + const res = await fetch(url, init as RequestInit); + if (!res.ok) return ""; const arrayBuffer = await res.arrayBuffer(); return Buffer.from(arrayBuffer).toString("base64"); } catch (e) { - return ''; + return ""; } }; @@ -79,17 +82,17 @@ export const fetchFile = async function (url: string, init?: FetchInit) { export const fetchText = async function ( url: string, init?: FetchInit, - encoding?: string, + encoding?: string ): Promise { init = await makeInit(init); console.log(url, init); try { - const res = await fetch(url, init); - if (!res.ok) return ''; + const res = await fetch(url, init as RequestInit); + if (!res.ok) return ""; const arrayBuffer = await res.arrayBuffer(); const decoder = new TextDecoder(encoding); return decoder.decode(arrayBuffer); } catch (e) { - return ''; + return ""; } }; diff --git a/package.json b/package.json index 001b213..11c79c3 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,7 @@ "description": "Plugins repo for LNReader", "main": "index.js", "scripts": { - "start": "tsc-watch --onSuccess \"node ./.js/index.js 3000\"", + "start": "ts-node ./tswatch.ts", "test": "exit 0", "json": "tsc && node ./.js/scripts/json_plugins.js", "generate": "ts-node ./scripts/multisrc/generate.ts", diff --git a/plugins/english/nobleMTL.ts b/plugins/english/nobleMTL.ts index bfdc506..bb7c295 100644 --- a/plugins/english/nobleMTL.ts +++ b/plugins/english/nobleMTL.ts @@ -124,10 +124,34 @@ class NobleMTL implements Plugin.PluginBase { .trim(); // Feb 25, 2024 if (!releaseDate) return; + const months = [ + "jan", + "feb", + "mar", + "apr", + "may", + "jun", + "jul", + "aug", + "sep", + "oct", + "nov", + "dec", + ]; + const rx = new RegExp( + `(${months.join("|")}).*? (\\d{2}).*?(\\d{4})`, + "i" + ).exec(releaseDate); + if (!rx) return; + const year = +rx[3]; + const month = months.indexOf(rx[1].toLowerCase()); + const day = +rx[2]; + + if (month < 0) return; chapter.push({ name: chapterName, - releaseTime: new Date(releaseDate).toISOString(), + releaseTime: new Date(year, month, day).toISOString(), url: chapterUrl, }); }); diff --git a/plugins/japanese/Syosetu.ts b/plugins/japanese/Syosetu.ts index 0ba973a..5bf1abc 100644 --- a/plugins/japanese/Syosetu.ts +++ b/plugins/japanese/Syosetu.ts @@ -1,8 +1,9 @@ -import { load as loadCheerio } from "cheerio"; -import { fetchFile } from "@libs/fetch"; +import { load, load as loadCheerio } from "cheerio"; +import { fetchApi, fetchFile } from "@libs/fetch"; import { Plugin } from "@typings/plugin"; import { defaultCover } from "@libs/defaultCover"; -import { Filters } from "@libs/filterInputs"; +import { FilterTypes, Filters } from "@libs/filterInputs"; +import { NovelItem } from "../../test_web/static/js"; // const novelStatus = require('@libs/novelStatus'); // const isUrlAbsolute = require('@libs/isAbsoluteUrl'); // const parseDate = require('@libs/parseDate'); @@ -12,11 +13,11 @@ class Syosetu implements Plugin.PluginBase { name = "Syosetu"; icon = "src/jp/syosetu/icon.png"; site = "https://yomou.syosetu.com/"; - filters?: Filters | undefined; - version = "1.0.0"; + version = "1.1.0"; headers = { - "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" - } + "User-Agent": + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", + }; searchUrl = (pagenum?: number, order?: string) => { return `https://yomou.syosetu.com/search.php?order=${order || "hyoka"}${ pagenum !== undefined @@ -24,37 +25,62 @@ class Syosetu implements Plugin.PluginBase { : "" // if isn't don't set ?p }`; }; - async popularNovels(pageNo: number, options: Plugin.PopularNovelsOptions): Promise { + async popularNovels( + pageNo: number, + { filters }: Plugin.PopularNovelsOptions + ): Promise { const getNovelsFromPage = async (pagenumber: number) => { // load page + let url = this.site; - const result = await fetch(this.searchUrl(pagenumber), {headers: this.headers}); - const body = await result.text(); - // Cheerio it! - const cheerioQuery = loadCheerio(body, { decodeEntities: false }); - let pageNovels: Plugin.NovelItem[] = []; - // find class=searchkekka_box - cheerioQuery(".searchkekka_box").each(function (i, e) { - // get div with link and name - const novelDIV = cheerioQuery(this).find(".novel_h"); - // get link element - const novelA = novelDIV.children()[0]; - // add new novel to array - pageNovels.push({ - name: novelDIV.text(), // get the name - url: novelA.attribs.href, // get last part of the link - cover: defaultCover, - }); + if (!filters.genre.value) { + url += `rank/list/type/${filters.ranking.value}_${filters.modifier.value}/?p=${pagenumber}`; + } else { + url += `rank/${ + filters.genre.value.length === 1 + ? "isekailist" + : "genrelist" + }/type/${filters.ranking.value}_${filters.genre.value}${ + filters.modifier.value === "total" + ? "" + : `_${filters.modifier.value}` + }/?p=${pagenumber}`; + } + const html = await (await fetchApi(url)).text(); + + const loadedCheerio = loadCheerio(html, { + decodeEntities: false, }); - // return all novels from this page - return pageNovels; - } + + if ( + parseInt(loadedCheerio(".is-current").html() || "1") !== + pagenumber + ) + return []; + + const novels: NovelItem[] = []; + loadedCheerio(".c-card").each((_, e) => { + const anchor = loadedCheerio(e).find( + ".p-ranklist-item__title a" + ); + const url = anchor.attr("href"); + if (!url) return; + const name = anchor.text(); + const novel: NovelItem = { + url, + name, + cover: defaultCover, + }; + novels.push(novel); + }); + return novels; + }; const novels = await getNovelsFromPage(pageNo); return novels; } async parseNovelAndChapters(novelUrl: string): Promise { let chapters: Plugin.ChapterItem[] = []; - const result = await fetch(novelUrl, {headers: this.headers}); + const result = await fetch(novelUrl, { headers: this.headers }); const body = await result.text(); const loadedCheerio = loadCheerio(body, { decodeEntities: false }); @@ -103,7 +129,7 @@ class Syosetu implements Plugin.PluginBase { const nameResult = await fetch( this.searchUrl() + `&word=${novel.name}`, - {headers: this.headers} + { headers: this.headers } ); const nameBody = await nameResult.text(); const summaryQuery = loadCheerio(nameBody, { @@ -129,7 +155,7 @@ class Syosetu implements Plugin.PluginBase { return novel; } async parseChapter(chapterUrl: string): Promise { - const result = await fetch(chapterUrl, {headers: this.headers}); + const result = await fetch(chapterUrl, { headers: this.headers }); const body = await result.text(); // create cheerioQuery @@ -137,20 +163,23 @@ class Syosetu implements Plugin.PluginBase { decodeEntities: false, }); - let chapterText = cheerioQuery("#novel_honbun") // get chapter text - .html() || ''; + let chapterText = + cheerioQuery("#novel_honbun") // get chapter text + .html() || ""; return chapterText; } - async searchNovels(searchTerm: string, pageNo: number): Promise { + async searchNovels( + searchTerm: string, + pageNo: number + ): Promise { let novels = []; // returns list of novels from given page const getNovelsFromPage = async (pagenumber: number) => { // load page - const result = await fetch( - this.searchUrl(pagenumber) + `&word=${searchTerm}`, - {headers: this.headers} - ); + const url = this.searchUrl(pagenumber) + `&word=${searchTerm}`; + console.log(url); + const result = await fetch(url, { headers: this.headers }); const body = await result.text(); // Cheerio it! const cheerioQuery = loadCheerio(body, { decodeEntities: false }); @@ -171,7 +200,7 @@ class Syosetu implements Plugin.PluginBase { }); // return all novels from this page return pageNovels; - } + }; // counter of loaded pages // let pagesLoaded = 0; @@ -181,7 +210,7 @@ class Syosetu implements Plugin.PluginBase { // pagesLoaded++; // } while (pagesLoaded < maxPageLoad && isNext); // check if we should load more - novels = await getNovelsFromPage(1); + novels = await getNovelsFromPage(pageNo); /** Use * novels.push(...(await getNovelsFromPage(pageNumber))) @@ -194,6 +223,62 @@ class Syosetu implements Plugin.PluginBase { async fetchImage(url: string): Promise { return fetchFile(url); } + filters = { + ranking: { + type: FilterTypes.Picker, + label: "Ranked by", + options: [ + { label: "日間", value: "daily" }, + { label: "週間", value: "weekly" }, + { label: "月間", value: "monthly" }, + { label: "四半期", value: "quarter" }, + { label: "年間", value: "yearly" }, + { label: "累計", value: "total" }, + ], + value: "total", + }, + genre: { + type: FilterTypes.Picker, + label: "Ranking Genre", + options: [ + { label: "総ジャンル", value: "" }, + { label: "異世界転生/転移〔恋愛〕〕", value: "1" }, + { label: "異世界転生/転移〔ファンタジー〕", value: "2" }, + { label: "異世界転生/転移〔文芸・SF・その他〕", value: "o" }, + { label: "異世界〔恋愛〕", value: "101" }, + { label: "現実世界〔恋愛〕", value: "102" }, + { label: "ハイファンタジー〔ファンタジー〕", value: "201" }, + { label: "ローファンタジー〔ファンタジー〕", value: "202" }, + { label: "純文学〔文芸〕", value: "301" }, + { label: "ヒューマンドラマ〔文芸〕", value: "302" }, + { label: "歴史〔文芸〕", value: "303" }, + { label: "推理〔文芸〕", value: "304" }, + { label: "ホラー〔文芸〕", value: "305" }, + { label: "アクション〔文芸〕", value: "306" }, + { label: "コメディー〔文芸〕", value: "307" }, + { label: "VRゲーム〔SF〕", value: "401" }, + { label: "宇宙〔SF〕", value: "402" }, + { label: "空想科学〔SF〕", value: "403" }, + { label: "パニック〔SF〕", value: "404" }, + { label: "童話〔その他〕", value: "9901" }, + { label: "詩〔その他〕", value: "9902" }, + { label: "エッセイ〔その他〕", value: "9903" }, + { label: "その他〔その他〕", value: "9999" }, + ], + value: "", + }, + modifier: { + type: FilterTypes.Picker, + label: "Modifier", + options: [ + { label: "すべて", value: "total" }, + { label: "連載中", value: "r" }, + { label: "完結済", value: "er" }, + { label: "短編", value: "t" }, + ], + value: "total", + }, + } satisfies Filters; } export default new Syosetu(); diff --git a/test_web/api/plugins.ts b/test_web/api/plugins.ts index 49d4520..3976d09 100644 --- a/test_web/api/plugins.ts +++ b/test_web/api/plugins.ts @@ -6,7 +6,7 @@ import { Plugin, isPlugin } from "@typings/plugin"; import { Filters } from "@libs/filterInputs"; const root = path.dirname(require?.main?.filename || ""); -export const fetchHeaders: Record = {} +export const fetchHeaders: Record = {}; export const all_plugins = (): PluginList => { const res: PluginList = {}; @@ -48,9 +48,12 @@ const getPlugin = async ( ): Promise => { console.log("loading plugin", requirePath); const plugin = await require(requirePath).default; - console.log(plugin); - if (isPlugin(plugin)) return plugin; - console.log("Not a plugin!"); + // console.log(plugin); + if (isPlugin(plugin)) { + console.log("loaded plugin", plugin.name); + return plugin; + } + console.error("Not a plugin!"); return null; }; @@ -59,13 +62,15 @@ export const getFilter = async (pluginRequirePath: string) => export const popularNovels = async ( pluginRequirePath: string, + page: number, options: Plugin.PopularNovelsOptions -) => (await getPlugin(pluginRequirePath))?.popularNovels(1, options); +) => (await getPlugin(pluginRequirePath))?.popularNovels(page, options); export const searchNovels = async ( pluginRequirePath: string, + page: number, searchTerm: string -) => (await getPlugin(pluginRequirePath))?.searchNovels(searchTerm, 1); +) => (await getPlugin(pluginRequirePath))?.searchNovels(searchTerm, page); export const parseNovelAndChapters = async ( pluginRequirePath: string, diff --git a/test_web/index.html b/test_web/index.html index 2f97299..a14d646 100644 --- a/test_web/index.html +++ b/test_web/index.html @@ -9,12 +9,14 @@ + + Testing plugin @@ -27,6 +29,7 @@
+
@@ -36,7 +39,8 @@
-
+ - -
Remember to copy the headers into the plugin file after changing them!
-
Copied
- - -
-
- popularNovels -
-