diff --git a/public/static/multisrc/madara/boxnovel/icon.png b/public/static/multisrc/readnovelfull/boxnovel/icon.png similarity index 100% rename from public/static/multisrc/madara/boxnovel/icon.png rename to public/static/multisrc/readnovelfull/boxnovel/icon.png diff --git a/scripts/multisrc/madara/filters/boxnovel.json b/scripts/multisrc/madara/filters/boxnovel.json deleted file mode 100644 index 2d0c7ef..0000000 --- a/scripts/multisrc/madara/filters/boxnovel.json +++ /dev/null @@ -1,240 +0,0 @@ -{ - "filters": { - "genre[]": { - "type": "Checkbox", - "label": "Genre", - "value": [], - "options": [ - { - "label": "Action", - "value": "action" - }, - { - "label": "Adventure", - "value": "adventure" - }, - { - "label": "Comedy", - "value": "comedy" - }, - { - "label": "Drama", - "value": "drama" - }, - { - "label": "Eastern", - "value": "eastern" - }, - { - "label": "Ecchi", - "value": "ecchi" - }, - { - "label": "Fantasy", - "value": "fantasy" - }, - { - "label": "Gender Bender", - "value": "gender-bender" - }, - { - "label": "Harem", - "value": "harem" - }, - { - "label": "Historical", - "value": "historical" - }, - { - "label": "Horror", - "value": "horror" - }, - { - "label": "Josei", - "value": "josei" - }, - { - "label": "Martial Arts", - "value": "martial-arts" - }, - { - "label": "Mature", - "value": "mature" - }, - { - "label": "Mecha", - "value": "mecha" - }, - { - "label": "Mystery", - "value": "mystery" - }, - { - "label": "Psychological", - "value": "psychological" - }, - { - "label": "Romance", - "value": "romance" - }, - { - "label": "School Life", - "value": "school-life" - }, - { - "label": "Sci-fi", - "value": "sci-fi" - }, - { - "label": "Seinen", - "value": "seinen" - }, - { - "label": "Shoujo", - "value": "shoujo" - }, - { - "label": "Shounen", - "value": "shounen" - }, - { - "label": "Slice of Life", - "value": "slice-of-life" - }, - { - "label": "Smut", - "value": "smut" - }, - { - "label": "Sports", - "value": "sports" - }, - { - "label": "Supernatural", - "value": "supernatural" - }, - { - "label": "Tragedy", - "value": "tragedy" - }, - { - "label": "Wuxia", - "value": "wuxia" - }, - { - "label": "Xianxia", - "value": "xianxia" - }, - { - "label": "Xuanhuan", - "value": "xuanhuan" - }, - { - "label": "Yaoi", - "value": "yaoi" - } - ] - }, - "op": { - "type": "Switch", - "label": "having all selected genres", - "value": false - }, - "author": { - "type": "Text", - "label": "Author", - "value": "" - }, - "artist": { - "type": "Text", - "label": "Artist", - "value": "" - }, - "release": { - "type": "Text", - "label": "Year of Released", - "value": "" - }, - "adult": { - "type": "Picker", - "label": "Adult content", - "value": "", - "options": [ - { - "label": "All", - "value": "" - }, - { - "label": "None adult content", - "value": "0" - }, - { - "label": "Only adult content", - "value": "1" - } - ] - }, - "status[]": { - "type": "Checkbox", - "label": "Status", - "value": [], - "options": [ - { - "label": "OnGoing", - "value": "on-going" - }, - { - "label": "Completed", - "value": "end" - }, - { - "label": "Canceled", - "value": "canceled" - }, - { - "label": "On Hold", - "value": "on-hold" - }, - { - "label": "Upcoming", - "value": "upcoming" - } - ] - }, - "m_orderby": { - "type": "Picker", - "label": "Order by", - "value": "", - "options": [ - { - "label": "Relevance", - "value": "" - }, - { - "label": "Latest", - "value": "latest" - }, - { - "label": "A-Z", - "value": "alphabet" - }, - { - "label": "Rating", - "value": "rating" - }, - { - "label": "Trending", - "value": "trending" - }, - { - "label": "Most Views", - "value": "views" - }, - { - "label": "New", - "value": "new-manga" - } - ] - } - } -} \ No newline at end of file diff --git a/scripts/multisrc/madara/sources.json b/scripts/multisrc/madara/sources.json index ece7100..9c73ab3 100644 --- a/scripts/multisrc/madara/sources.json +++ b/scripts/multisrc/madara/sources.json @@ -1,14 +1,4 @@ [ - { - "id": "boxnovel", - "sourceSite": "https://novgo.co/", - "sourceName": "BoxNovel", - "options": { - "useNewChapterEndpoint": true, - "customJs": "chapterText.find('div:has(a)').remove()", - "versionIncrements": 6 - } - }, { "id": "novelTL", "sourceSite": "https://noveltranslate.com/", diff --git a/scripts/multisrc/readnovelfull/README.md b/scripts/multisrc/readnovelfull/README.md new file mode 100644 index 0000000..3c8c8f4 --- /dev/null +++ b/scripts/multisrc/readnovelfull/README.md @@ -0,0 +1,31 @@ +# Readnovelfull multisrc generator + +## Compatiblity + +### This generator is brought over from the madara generator one folder away + +it should(!) work for all sites that uses this theme + +## Add a new source + +### sources.json + +To add a new source you need to add it to sources.json: + +- id: the id of the source (something unique) +- sourceName: the name of the source (you can use the value of "name" in "https://site.com/wp-json/" if it exists) +- sourceSite: the site url +- options: the options of the source + - lang: the language of the source (default: "English") (check that the language + exists in the languages (check folder names in "plugins/")) + - versionIncrements: needs to be updated everytime the site url is updated + - popularPage: the href value using this CSS selector `a[title="Latest Release"]` + - ajax-chapter-list: if chapters are obtained from an ajax endpoint + - pageParam: if a site uses something other than page for accessing pages + +### filters + +To add filters to a source you need to run the script "get_filters.js" \ +(`npx node scripts/multisrc/readnovelfull/get_filters.js` +(if you are at the root of the project) (and you have ran "npm install" before)) +and follow the instructions (url is easier and faster but sometimes it doesn't work) diff --git a/scripts/multisrc/readnovelfull/filters/allnovel.json b/scripts/multisrc/readnovelfull/filters/allnovel.json new file mode 100644 index 0000000..a1c640b --- /dev/null +++ b/scripts/multisrc/readnovelfull/filters/allnovel.json @@ -0,0 +1,174 @@ +{ + "filters": { + "type": { + "type": "Picker", + "label": "Novel Listing", + "value": "most-popular", + "options": [ + { + "label": "Hot Novel", + "value": "hot-novel" + }, + { + "label": "Completed Novel", + "value": "completed-novel" + }, + { + "label": "Most Popular", + "value": "most-popular" + } + ] + }, + "genres": { + "type": "Picker", + "label": "Genre", + "value": "", + "options": [ + { + "label": "Shounen", + "value": "genre/Shounen" + }, + { + "label": "Harem", + "value": "genre/Harem" + }, + { + "label": "Comedy", + "value": "genre/Comedy" + }, + { + "label": "Martial Arts", + "value": "genre/Martial+Arts" + }, + { + "label": "School Life", + "value": "genre/School+Life" + }, + { + "label": "Mystery", + "value": "genre/Mystery" + }, + { + "label": "Shoujo", + "value": "genre/Shoujo" + }, + { + "label": "Romance", + "value": "genre/Romance" + }, + { + "label": "Sci-fi", + "value": "genre/Sci-fi" + }, + { + "label": "Gender Bender", + "value": "genre/Gender+Bender" + }, + { + "label": "Mature", + "value": "genre/Mature" + }, + { + "label": "Fantasy", + "value": "genre/Fantasy" + }, + { + "label": "Horror", + "value": "genre/Horror" + }, + { + "label": "Drama", + "value": "genre/Drama" + }, + { + "label": "Tragedy", + "value": "genre/Tragedy" + }, + { + "label": "Supernatural", + "value": "genre/Supernatural" + }, + { + "label": "Ecchi", + "value": "genre/Ecchi" + }, + { + "label": "Xuanhuan", + "value": "genre/Xuanhuan" + }, + { + "label": "Adventure", + "value": "genre/Adventure" + }, + { + "label": "Action", + "value": "genre/Action" + }, + { + "label": "Psychological", + "value": "genre/Psychological" + }, + { + "label": "Xianxia", + "value": "genre/Xianxia" + }, + { + "label": "Wuxia", + "value": "genre/Wuxia" + }, + { + "label": "Historical", + "value": "genre/Historical" + }, + { + "label": "Slice of Life", + "value": "genre/Slice+of+Life" + }, + { + "label": "Seinen", + "value": "genre/Seinen" + }, + { + "label": "Lolicon", + "value": "genre/Lolicon" + }, + { + "label": "Adult", + "value": "genre/Adult" + }, + { + "label": "Josei", + "value": "genre/Josei" + }, + { + "label": "Sports", + "value": "genre/Sports" + }, + { + "label": "Smut", + "value": "genre/Smut" + }, + { + "label": "Mecha", + "value": "genre/Mecha" + }, + { + "label": "Yaoi", + "value": "genre/Yaoi" + }, + { + "label": "Shounen Ai", + "value": "genre/Shounen+Ai" + }, + { + "label": "History", + "value": "genre/History" + }, + { + "label": "Reincarnation", + "value": "genre/Reincarnation" + } + ] + } + } +} \ No newline at end of file diff --git a/scripts/multisrc/readnovelfull/filters/boxnovel.json b/scripts/multisrc/readnovelfull/filters/boxnovel.json new file mode 100644 index 0000000..c655d31 --- /dev/null +++ b/scripts/multisrc/readnovelfull/filters/boxnovel.json @@ -0,0 +1,254 @@ +{ + "filters": { + "type": { + "type": "Picker", + "label": "Novel Listing", + "value": "sort/nov-love-popular", + "options": [ + { + "label": "Hot Novel", + "value": "sort/nov-love-hot" + }, + { + "label": "Completed Novel", + "value": "sort/nov-love-complete" + }, + { + "label": "Most Popular", + "value": "sort/nov-love-popular" + } + ] + }, + "genres": { + "type": "Picker", + "label": "Genre", + "value": "", + "options": [ + { + "label": "Action", + "value": "nov-love-genres/action" + }, + { + "label": "Adventure", + "value": "nov-love-genres/adventure" + }, + { + "label": "Anime & comics", + "value": "nov-love-genres/anime-&-comics" + }, + { + "label": "Comedy", + "value": "nov-love-genres/comedy" + }, + { + "label": "Drama", + "value": "nov-love-genres/drama" + }, + { + "label": "Eastern", + "value": "nov-love-genres/eastern" + }, + { + "label": "Fan-fiction", + "value": "nov-love-genres/fan-fiction" + }, + { + "label": "Fanfiction", + "value": "nov-love-genres/fanfiction" + }, + { + "label": "Fantasy", + "value": "nov-love-genres/fantasy" + }, + { + "label": "Game", + "value": "nov-love-genres/game" + }, + { + "label": "Games", + "value": "nov-love-genres/games" + }, + { + "label": "Gender bender", + "value": "nov-love-genres/gender-bender" + }, + { + "label": "General", + "value": "nov-love-genres/general" + }, + { + "label": "Harem", + "value": "nov-love-genres/harem" + }, + { + "label": "Historical", + "value": "nov-love-genres/historical" + }, + { + "label": "Horror", + "value": "nov-love-genres/horror" + }, + { + "label": "Isekai", + "value": "nov-love-genres/isekai" + }, + { + "label": "Josei", + "value": "nov-love-genres/josei" + }, + { + "label": "Litrpg", + "value": "nov-love-genres/litrpg" + }, + { + "label": "Magic", + "value": "nov-love-genres/magic" + }, + { + "label": "Magical realism", + "value": "nov-love-genres/magical-realism" + }, + { + "label": "Martial arts", + "value": "nov-love-genres/martial-arts" + }, + { + "label": "Mature", + "value": "nov-love-genres/mature" + }, + { + "label": "Mecha", + "value": "nov-love-genres/mecha" + }, + { + "label": "Modern life", + "value": "nov-love-genres/modern-life" + }, + { + "label": "Myster", + "value": "nov-love-genres/myster" + }, + { + "label": "Mystery", + "value": "nov-love-genres/mystery" + }, + { + "label": "Other", + "value": "nov-love-genres/other" + }, + { + "label": "Psychological", + "value": "nov-love-genres/psychological" + }, + { + "label": "Reincarnation", + "value": "nov-love-genres/reincarnation" + }, + { + "label": "Report story", + "value": "nov-love-genres/report-story" + }, + { + "label": "Romance", + "value": "nov-love-genres/romance" + }, + { + "label": "Romance.smut", + "value": "nov-love-genres/romance.smut" + }, + { + "label": "School life", + "value": "nov-love-genres/school-life" + }, + { + "label": "Sci-fi", + "value": "nov-love-genres/sci-fi" + }, + { + "label": "Seinen", + "value": "nov-love-genres/seinen" + }, + { + "label": "Shoujo", + "value": "nov-love-genres/shoujo" + }, + { + "label": "Shoujo ai", + "value": "nov-love-genres/shoujo-ai" + }, + { + "label": "Shounen", + "value": "nov-love-genres/shounen" + }, + { + "label": "Shounen ai", + "value": "nov-love-genres/shounen-ai" + }, + { + "label": "Slice of life", + "value": "nov-love-genres/slice-of-life" + }, + { + "label": "Smut", + "value": "nov-love-genres/smut" + }, + { + "label": "Sports", + "value": "nov-love-genres/sports" + }, + { + "label": "Supernatural", + "value": "nov-love-genres/supernatural" + }, + { + "label": "System", + "value": "nov-love-genres/system" + }, + { + "label": "Thriller", + "value": "nov-love-genres/thriller" + }, + { + "label": "Tragedy", + "value": "nov-love-genres/tragedy" + }, + { + "label": "Urban", + "value": "nov-love-genres/urban" + }, + { + "label": "Urban life", + "value": "nov-love-genres/urban-life" + }, + { + "label": "Video games", + "value": "nov-love-genres/video-games" + }, + { + "label": "War", + "value": "nov-love-genres/war" + }, + { + "label": "Wuxia", + "value": "nov-love-genres/wuxia" + }, + { + "label": "Xianxia", + "value": "nov-love-genres/xianxia" + }, + { + "label": "Xuanhuan", + "value": "nov-love-genres/xuanhuan" + }, + { + "label": "Yaoi", + "value": "nov-love-genres/yaoi" + }, + { + "label": "Yuri", + "value": "nov-love-genres/yuri" + } + ] + } + } +} \ No newline at end of file diff --git a/scripts/multisrc/readnovelfull/filters/lightnovelplus.json b/scripts/multisrc/readnovelfull/filters/lightnovelplus.json new file mode 100644 index 0000000..54795a6 --- /dev/null +++ b/scripts/multisrc/readnovelfull/filters/lightnovelplus.json @@ -0,0 +1,130 @@ +{ + "filters": { + "type": { + "type": "Picker", + "label": "Novel Listing", + "value": "hot_novel", + "options": [ + { + "label": "Hot Novel", + "value": "hot_novel" + }, + { + "label": "Novel Completed", + "value": "completed_novel" + } + ] + }, + "genres": { + "type": "Picker", + "label": "Genre", + "value": "", + "options": [ + { + "label": "Fantasy", + "value": "60" + }, + { + "label": "Action", + "value": "132" + }, + { + "label": "Sci-fi", + "value": "61" + }, + { + "label": "Romance", + "value": "59" + }, + { + "label": "Adventure", + "value": "62" + }, + { + "label": "Xuanhuan", + "value": "64" + }, + { + "label": "Modern", + "value": "66" + }, + { + "label": "Mystery", + "value": "63" + }, + { + "label": "Romance", + "value": "68" + }, + { + "label": "Fantasy", + "value": "70" + }, + { + "label": "Historical", + "value": "74" + }, + { + "label": "Sci-fi", + "value": "75" + }, + { + "label": "Xuanhuan", + "value": "76" + }, + { + "label": "Mystery", + "value": "77" + }, + { + "label": "Adventure", + "value": "116" + }, + { + "label": "LGBT+", + "value": "182" + }, + { + "label": "Fantasy Romance", + "value": "134" + }, + { + "label": "Video Games", + "value": "243" + }, + { + "label": "Sci-fi Romance", + "value": "252" + }, + { + "label": "Historical Romance", + "value": "256" + }, + { + "label": "Magical Realism", + "value": "331" + }, + { + "label": "Eastern Fantasy", + "value": "334" + }, + { + "label": "Contemporary Romance", + "value": "344" + }, + { + "label": "Games", + "value": "503" + }, + { + "label": "Urban", + "value": "504" + }, + { + "label": "Harem", + "value": "517" + } + ] + } + } +} \ No newline at end of file diff --git a/scripts/multisrc/readnovelfull/filters/readnovelfull.json b/scripts/multisrc/readnovelfull/filters/readnovelfull.json new file mode 100644 index 0000000..f2687f3 --- /dev/null +++ b/scripts/multisrc/readnovelfull/filters/readnovelfull.json @@ -0,0 +1,174 @@ +{ + "filters": { + "type": { + "type": "Picker", + "label": "Novel Listing", + "value": "novel-list/most-popular-novel", + "options": [ + { + "label": "Hot Novel", + "value": "novel-list/hot-novel" + }, + { + "label": "Completed Novel", + "value": "novel-list/completed-novel" + }, + { + "label": "Most Popular", + "value": "novel-list/most-popular-novel" + } + ] + }, + "genres": { + "type": "Picker", + "label": "Genre", + "value": "", + "options": [ + { + "label": "Action", + "value": "genres/action" + }, + { + "label": "Adult", + "value": "genres/adult" + }, + { + "label": "Adventure", + "value": "genres/adventure" + }, + { + "label": "Comedy", + "value": "genres/comedy" + }, + { + "label": "Drama", + "value": "genres/drama" + }, + { + "label": "Eastern", + "value": "genres/eastern" + }, + { + "label": "Ecchi", + "value": "genres/ecchi" + }, + { + "label": "Fantasy", + "value": "genres/fantasy" + }, + { + "label": "Game", + "value": "genres/game" + }, + { + "label": "Gender Bender", + "value": "genres/gender+bender" + }, + { + "label": "Harem", + "value": "genres/harem" + }, + { + "label": "Historical", + "value": "genres/historical" + }, + { + "label": "Horror", + "value": "genres/horror" + }, + { + "label": "Josei", + "value": "genres/josei" + }, + { + "label": "Lolicon", + "value": "genres/lolicon" + }, + { + "label": "Martial Arts", + "value": "genres/martial+arts" + }, + { + "label": "Mature", + "value": "genres/mature" + }, + { + "label": "Mecha", + "value": "genres/mecha" + }, + { + "label": "Modern Life", + "value": "genres/modern+life" + }, + { + "label": "Mystery", + "value": "genres/mystery" + }, + { + "label": "Psychological", + "value": "genres/psychological" + }, + { + "label": "Reincarnation", + "value": "genres/reincarnation" + }, + { + "label": "Romance", + "value": "genres/romance" + }, + { + "label": "School life", + "value": "genres/school+life" + }, + { + "label": "Sci-fi", + "value": "genres/sci-fi" + }, + { + "label": "Seinen", + "value": "genres/seinen" + }, + { + "label": "Shoujo", + "value": "genres/shoujo" + }, + { + "label": "Shounen", + "value": "genres/shounen" + }, + { + "label": "Slice of Life", + "value": "genres/slice+of+life" + }, + { + "label": "Smut", + "value": "genres/smut" + }, + { + "label": "Sports", + "value": "genres/sports" + }, + { + "label": "Supernatural", + "value": "genres/supernatural" + }, + { + "label": "System", + "value": "genres/system" + }, + { + "label": "Thriller", + "value": "genres/thriller" + }, + { + "label": "Tragedy", + "value": "genres/tragedy" + }, + { + "label": "Transmigration", + "value": "genres/transmigration" + } + ] + } + } +} \ No newline at end of file diff --git a/scripts/multisrc/readnovelfull/generator.js b/scripts/multisrc/readnovelfull/generator.js index 349713d..18d96b4 100644 --- a/scripts/multisrc/readnovelfull/generator.js +++ b/scripts/multisrc/readnovelfull/generator.js @@ -1,15 +1,21 @@ -import { readFileSync } from 'fs'; +import list from './sources.json' with { type: 'json' }; +import { existsSync, readFileSync } from 'fs'; import { fileURLToPath } from 'url'; import { dirname, join } from 'path'; const folder = dirname(fileURLToPath(import.meta.url)); -import list from './sources.json' with { type: 'json' }; - export const generateAll = function () { return list.map(source => { + const exist = existsSync(join(folder, 'filters', source.id + '.json')); + if (exist) { + const filters = readFileSync( + join(folder, 'filters', source.id + '.json'), + ); + source.filters = JSON.parse(filters).filters; + } console.log( - `[readnovelfull] Generating: ${source.id}${' '.repeat(20 - source.id.length)}`, + `[readnovelfull] Generating: ${source.id}${' '.repeat(20 - source.id.length)} ${source.filters ? '🔎with filters🔍' : '🚫no filters🚫'}`, ); return generator(source); }); diff --git a/scripts/multisrc/readnovelfull/get_filters.js b/scripts/multisrc/readnovelfull/get_filters.js new file mode 100644 index 0000000..6fe8293 --- /dev/null +++ b/scripts/multisrc/readnovelfull/get_filters.js @@ -0,0 +1,216 @@ +import * as fs from 'fs'; +import * as cheerio from 'cheerio'; +import * as path from 'path'; +import * as readline from 'readline'; +import process from 'process'; +import { fileURLToPath } from 'url'; +import { dirname } from 'path'; +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); + +function getFilters(name, html, baseUrl) { + const $ = cheerio.load(html); + const filters = { + filters: { + 'type': { + type: 'Picker', + label: 'Novel Listing', + value: '', + options: [], + }, + 'genres': { + type: 'Picker', + label: 'Genre', + value: '', + options: [], + }, + }, + }; + + const baseSelector = 'div.navbar-collapse li.dropdown'; + + // --- Type Filters --- + let defaultValue = null; + const typeOptions = []; + $(`${baseSelector}:eq(0) li a`).each((_, el) => { + const $el = $(el); + const title = $el.attr('title'); + const label = $el.text().trim(); + const href = $el.attr('href'); + + if (!href || !label || title === 'Latest Release') return; + + try { + const fullUrl = new URL(decodeURI(href), baseUrl); + const params = fullUrl.searchParams; + + const value = params.has('type') + ? params.get('type') + : (fullUrl.pathname.startsWith('/') + ? fullUrl.pathname.substring(1) + : fullUrl.pathname) || ''; + + if (value) { + typeOptions.push({ label, value }); + if (title === 'Most Popular') { + defaultValue = value; + } + } else { + console.warn( + `Skipping type option "${label}" for ${name}: Could not determine value from href "${href}"`, + ); + } + } catch (error) { + console.error( + `Error processing type URL (${href}) for ${name}: ${error.message}`, + ); + } + }); + filters.filters.type.options = typeOptions; + filters.filters.type.value = defaultValue ?? (typeOptions[0]?.value || ''); // Set default: Popular > First > Empty + + // --- Genres Filters --- + const genreOptions = []; + $(`${baseSelector}:eq(1) li a`).each((_, el) => { + const $el = $(el); + const label = $el.text().trim(); + const href = $el.attr('href'); + + if (!href || !label) return; + + try { + const fullUrl = new URL(decodeURI(href), baseUrl); + const params = fullUrl.searchParams; + + const value = + params.get('type') === 'category_novel' && params.has('id') + ? params.get('id') + : (fullUrl.pathname.startsWith('/') + ? fullUrl.pathname.substring(1) + : fullUrl.pathname) || ''; + + if (value) { + genreOptions.push({ label, value }); + } else { + console.warn( + `Skipping genre option "${label}" for ${name}: Could not determine value from href "${href}"`, + ); + } + } catch (error) { + console.error( + `Error processing genre URL (${href}) for ${name}: ${error.message}`, + ); + } + }); + filters.filters.genre.options = genreOptions; // Genre has no default + + // --- Validation and Saving --- + if ( + filters.filters.type.options.length === 0 || + filters.filters.genre.options.length === 0 + ) { + console.warn( + `🚨Warning for ${name}: Type or Genre options might be incomplete or empty. (${path.join(__dirname, 'filters', name + '.json')})🚨`, + ); + } + try { + const filtersDir = path.join(__dirname, 'filters'); + if (!fs.existsSync(filtersDir)) + fs.mkdirSync(filtersDir, { recursive: true }); + fs.writeFileSync( + path.join(filtersDir, name + '.json'), + JSON.stringify(filters, null, 2), + ); + console.log(`✅Filters created successfully for ${name}✅`); + } catch (error) { + console.error( + `🚨Error writing filters file for ${name}: ${error.message}🚨`, + ); + } +} + +async function getFiltersFromURL(name, url) { + const response = await fetch(url); + if (!response.ok) { + throw new Error( + `HTTP error! status: ${response.status}, while fetching ${response.url}`, + ); + } + const html = await response.text(); + try { + getFilters(name, html, url); + } catch (e) { + console.error('Error while getting filters from', url, e); + } +} + +async function askGetFilter() { + const rl = readline.createInterface({ + input: process.stdin, + output: process.stdout, + }); + + const EREASE_PREV_LINE = '\x1b[1A\r\x1b[2K'; + rl.question( + 'Enter the id of the site (same one as in sources.json): ', + async name => { + rl.question( + EREASE_PREV_LINE + + "Do you want to get the filters from a URL or the html text? (if url dosen't work try html) (url/html): ", + async method => { + if (method.toLowerCase() === 'url') { + const sources = JSON.parse( + fs.readFileSync(path.join(__dirname, 'sources.json'), 'utf-8'), + ); + const source = sources.find(s => s.id === name); + if (source && source.sourceSite) { + console.log('Getting filters from', source.sourceSite); + try { + await getFiltersFromURL(name, source.sourceSite); + } catch (e) { + console.error( + 'Error while getting filters from', + source.sourceSite, + ); + console.log(e.message || e); + } + rl.close(); + } else { + rl.question( + EREASE_PREV_LINE + + 'Enter the URL (same one as in sources.json): ', + async url => { + rl.close(); + try { + await getFiltersFromURL(name, url); + } catch (e) { + console.error('Error while getting filters from', url); + console.log(e.message || e); + } + }, + ); + } + } else { + process.stdout.write( + EREASE_PREV_LINE + + `Enter the html text from the page at {sourceSite}/?s=&post_type=wp-manga (at the end press ENTER then press CTRL+C): `, + ); + let html = ''; + rl.on('SIGINT', () => { + console.log('Stopeed reading input, creating filters file'); + getFilters(name, html, url); + rl.close(); + }); + rl.on('line', line => { + html += line + '\n'; + }); + } + }, + ); + }, + ); +} + +askGetFilter(); + +export { getFiltersFromURL }; diff --git a/scripts/multisrc/readnovelfull/sources.json b/scripts/multisrc/readnovelfull/sources.json index cfd80c1..82d41f6 100644 --- a/scripts/multisrc/readnovelfull/sources.json +++ b/scripts/multisrc/readnovelfull/sources.json @@ -1,34 +1,46 @@ [ { "id": "readnovelfull", - "sourceSite": "https://readnovelfull.com", + "sourceSite": "https://readnovelfull.com/", "sourceName": "ReadNovelFull", "options": { - "versionIncrements": 1, - "popularPage": "novel-list/most-popular-novel?page=%%PAGE%%", - "latestPage": "novel-list/latest-release-novel?page=%%PAGE%%", - "searchPage": "novel-list/search?keyword=%%SEARCH%%&page=%%PAGE%%", - "ajax-chapter-list": true + "latestPage": "novel-list/latest-release-novel", + "searchPage": "novel-list/search" } }, { "id": "allnovel", - "sourceSite": "https://allnovel.org", + "sourceSite": "https://allnovel.org/", "sourceName": "AllNovel", "options": { - "popularPage": "most-popular?page=%%PAGE%%", - "latestPage": "latest-release-novel?page=%%PAGE%%", - "searchPage": "search?keyword=%%SEARCH%%&page=%%PAGE%%" + "latestPage": "latest-release-novel", + "searchPage": "search", + "chapterListing": "ajax-chapter-option" } }, { "id": "lightnovelplus", - "sourceSite": "https://lightnovelplus.com", + "sourceSite": "https://lightnovelplus.com/", "sourceName": "LightNovelPlus", "options": { - "popularPage": "book/bookclass.html?type=hot_novel&language=en&page_num=%%PAGE%%", - "latestPage": "book/bookclass.html?type=last_release&language=en&page_num=%%PAGE%%", - "searchPage": "book/search.html?keyword=%%SEARCH%%&page_num=%%PAGE%%&language=en" + "novelListing": "book/bookclass.html", + "chapterListing": "get_chapter_list", + "searchPage": "book/search.html", + "latestPage": "last_release", + "genreParam": "category_novel", + "pageParam": "page_num", + "langParam": "language", + "urlLangCode": "en", + "ajaxChapterParam": "bookId" + } + }, + { + "id": "boxnovel", + "sourceSite": "https://novlove.com/", + "sourceName": "BoxNovel", + "options": { + "latestPage": "sort/nov-love-daily-update", + "searchPage": "search" } } ] diff --git a/scripts/multisrc/readnovelfull/template.ts b/scripts/multisrc/readnovelfull/template.ts index 4cf80bc..e819e1a 100644 --- a/scripts/multisrc/readnovelfull/template.ts +++ b/scripts/multisrc/readnovelfull/template.ts @@ -1,15 +1,23 @@ -import { CheerioAPI, load as parseHTML, load } from 'cheerio'; +import { CheerioAPI, load } from 'cheerio'; import { fetchApi } from '@libs/fetch'; import { Plugin } from '@typings/plugin'; import { NovelStatus } from '@libs/novelStatus'; +import { Filters } from '@libs/filterInputs'; type ReadNovelFullOptions = { lang?: string; versionIncrements?: number; - popularPage: string; latestPage: string; searchPage: string; - ajaxChapterList?: boolean; + novelListing?: string; + chapterListing?: string; + TypeParam?: string; + genreParam?: string; + genreKey?: string; + pageParam?: string; + langParam?: string; + urlLangCode?: string; + ajaxChapterParam?: string; }; export type ReadNovelFullMetadata = { @@ -27,6 +35,7 @@ class ReadNovelFullPlugin implements Plugin.PluginBase { site: string; version: string; options: ReadNovelFullOptions; + filters?: Filters | undefined; constructor(metadata: ReadNovelFullMetadata) { this.id = metadata.id; @@ -34,8 +43,9 @@ class ReadNovelFullPlugin implements Plugin.PluginBase { this.icon = `multisrc/readnovelfull/${metadata.id.toLowerCase()}/icon.png`; this.site = metadata.sourceSite; const versionIncrements = metadata.options?.versionIncrements || 0; - this.version = `1.0.${1 + versionIncrements}`; + this.version = `2.0.${0 + versionIncrements}`; this.options = metadata.options; + this.filters = metadata.filters; } async getCheerio(url: string, search: boolean): Promise { @@ -50,28 +60,39 @@ class ReadNovelFullPlugin implements Plugin.PluginBase { } parseNovels($: CheerioAPI): Plugin.NovelItem[] { - const novels: Plugin.NovelItem[] = []; + const baseUrl = this.site; - $('.list-novel > .row, .list-truyen > .row').each((i, elem) => { - const novelName = $(elem).find('h3').text(); - let coverUrl = $(elem).find('img').attr('src'); - const novelUrl = this.site + $(elem).find('a').attr('href'); + const novels = $('.list-novel .row, .list-truyen .row') + .toArray() + .map(el => { + const $el = $(el); - if ( - coverUrl && - !(coverUrl.includes('https://') || coverUrl.includes('http://')) - ) { - coverUrl = this.site + coverUrl; - } + const novelName = $el.find('h3').text()?.trim(); + const rawHref = $el.find('a').attr('href'); + const rawCover = + $el.find('img').attr('data-src') || $el.find('img').attr('src'); - if (novelUrl && novelName.trim()) { - novels.push({ + if (!novelName || !rawHref) { + return null; + } + + let novelPath: string; + let novelCover: string | undefined; + + const novelUrlObject = new URL(rawHref, baseUrl); + novelPath = novelUrlObject.pathname.slice(1); + + if (rawCover) { + novelCover = new URL(rawCover, baseUrl).href; + } + + return { name: novelName, - cover: coverUrl, - path: novelUrl.replace(this.site, ''), - }); - } - }); + path: novelPath, + cover: novelCover, + }; + }) + .filter(Boolean); return novels; } @@ -80,51 +101,76 @@ class ReadNovelFullPlugin implements Plugin.PluginBase { pageNo: number, { filters, showLatestNovels }: Plugin.PopularNovelsOptions, ): Promise { - let url = - this.site + - '/' + - this.options.popularPage.replace('%%PAGE%%', pageNo.toString()); + let url = ''; + const pageParam = this.options.pageParam || 'page'; - if (!filters) filters = {}; + if (this.options.novelListing) { + // URL structure with parameters + const params = new URLSearchParams(); + const typeParam = this.options.TypeParam || 'type'; // Default to 'type' if not specified - if (showLatestNovels) { - url = - this.site + - '/' + - this.options.latestPage.replace('%%PAGE%%', pageNo.toString()); + if (showLatestNovels) { + params.append(typeParam, this.options.latestPage); + } else if (filters.genres.value.length) { + params.append(typeParam, this.options.genreParam!); + const genreKey = this.options.genreKey || 'id'; // Default to 'id' if not specified + params.append(genreKey, filters.genres.value); + } else { + params.append(typeParam, filters.type.value); + } + + // Add language parameter if specified + if (this.options.langParam && this.options.urlLangCode) { + params.append(this.options.langParam, this.options.urlLangCode); + } + + params.append(pageParam, pageNo.toString()); + + url = `${this.site}${this.options.novelListing}?${params.toString()}`; + } else { + // URL structure with path segments + let basePage = ''; + const params = new URLSearchParams({ + [pageParam]: pageNo.toString(), + }); + + if (showLatestNovels) { + basePage = this.options.latestPage; + } else if (filters?.genres?.value?.length) { + basePage = filters.genres.value; + } else { + basePage = filters.type.value; + } + + url = `${this.site}${basePage}?${params.toString()}`; } const $ = await this.getCheerio(url, false); - return this.parseNovels($); } async parseNovel(novelPath: string): Promise { let $ = await this.getCheerio(this.site + novelPath, false); + const baseUrl = this.site; - const novel: Plugin.SourceNovel & { totalPages?: number } = { - path: novelPath.replace(this.site, ''), - name: 'Untitled', + const novel: Partial = { + path: novelPath, }; novel.name = $('h3.title').first().text().trim(); - novel.cover = $('.book > img').attr('src'); - if ( - novel.cover && - !(novel.cover.includes('https://') || novel.cover.includes('http://')) - ) { - novel.cover = this.site + novel.cover; + const coverUrl = + $('.book img').attr('data-src') || $('.book img').attr('src'); + if (coverUrl) { + novel.cover = new URL(coverUrl, baseUrl).href; } - $('ul.info.info-meta > li, .info > div').each(function () { - const detailName = $(this) - .find('h3') - .first() - .text() - .toLowerCase() - .replace(':', ''); - - const detail = $(this).text().split(':')[1].trim().toLowerCase(); + $('ul.info.info-meta li, .info div').each(function () { + const detailText = $(this).text().split(':'); + const detailName = detailText[0].trim().toLowerCase(); + const detail = detailText[1] + ?.split(',') + .map(g => g.trim()) + .join(', '); switch (detailName) { case 'author': @@ -134,85 +180,63 @@ class ReadNovelFullPlugin implements Plugin.PluginBase { novel.genres = detail; break; case 'status': - switch (detail) { - case 'completed': - novel.status = NovelStatus.Completed; - break; - case 'ongoing': - novel.status = NovelStatus.Ongoing; - break; - case 'hiatus': - novel.status = NovelStatus.OnHiatus; - break; - default: - novel.status = NovelStatus.Unknown; - break; - } - break; + const map: Record = { + completed: NovelStatus.Completed, + ongoing: NovelStatus.Ongoing, + hiatus: NovelStatus.OnHiatus, + }; + novel.status = map[detail.toLowerCase()] ?? NovelStatus.Unknown; } }); - novel.summary = $('.desc-text').text(); + const summaryDiv = $('.desc-text'); + const pTagSummary = summaryDiv.find('p'); + novel.summary = + pTagSummary.length > 0 + ? pTagSummary + .map((_, el) => $(el).text().trim()) + .get() + .join('\n\n') + : summaryDiv.text().trim(); // --- Else (no

tags, assume
) --- + const chapters: Plugin.ChapterItem[] = []; - const novelId = $('#rating').attr('data-novel-id'); - if (this.options?.ajaxChapterList) { - const chaptersUrl = - this.site + '/ajax/chapter-archive?novelId=' + novelId; + const dataNovelId = $('#rating').attr('data-novel-id'); + const idMatch = novelPath.match(/\d+/); + const novelId = dataNovelId || (idMatch ? idMatch[0] : null); + + if (novelId) { + const ajaxParam = this.options.ajaxChapterParam || 'novelId'; + const chapterListing = + this.options.chapterListing || 'ajax/chapter-archive'; + + const params = new URLSearchParams({ + [ajaxParam]: novelId.toString(), + }); + const chaptersUrl = `${this.site}${chapterListing}?${params.toString()}`; $ = await this.getCheerio(chaptersUrl, false); - $('.panel-body') - .find('li') - .each(function () { - const chapterName = $(this).find('a').attr('title') || ''; - const chapterUrl = $(this).find('a').attr('href') || ''; + const listing = $('.panel-body li a, select option'); - chapters.push({ - name: chapterName, - path: chapterUrl, - }); - }); - } else { - chapters.push(...this.parsePageChapters($)); - novel.totalPages = parseInt( - ($('ul.pagination > li.last > a').attr('data-page') ?? - $('ul.pagination > li.last > a') - .attr('href') - ?.match(/\?page_num=(\d+)/)?.[1])!, - ); - } + listing.each((_, el) => { + const $el = $(el); + const chapterName = $el.attr('title') || $el.text() || ''; + const chapterHref = $el.attr('href') || $el.attr('value') || ''; - novel.chapters = chapters; - - return novel; - } - - parsePageChapters($: CheerioAPI) { - const chapters: Plugin.ChapterItem[] = []; - $('.list-chapter') - .find('li') - .each(function () { - const chapterName = $(this).find('a').attr('title') || ''; - const chapterUrl = $(this).find('a').attr('href') || ''; + const chapterUrlObject = new URL(chapterHref, this.site); + const chapterUrl = chapterUrlObject.pathname.slice(1); chapters.push({ name: chapterName, path: chapterUrl, + releaseTime: null, }); }); - return chapters; - } + } + novel.chapters = chapters; - async parsePage(novelPath: string, page: string): Promise { - const $ = await this.getCheerio( - this.site + novelPath + '?page=' + page, - false, - ); - const chapters = this.parsePageChapters($); - return { - chapters, - }; + return novel as Plugin.SourceNovel; } async parseChapter(chapterPath: string): Promise { @@ -225,12 +249,18 @@ class ReadNovelFullPlugin implements Plugin.PluginBase { searchTerm: string, page: number, ): Promise { - const url = - this.site + - '/' + - this.options?.searchPage - .replace('%%SEARCH%%', encodeURIComponent(searchTerm)) - .replace('%%PAGE%%', page.toString()); + const pageParam = this.options.pageParam || 'page'; + + const params = new URLSearchParams({ + keyword: searchTerm, + [pageParam]: page.toString(), + }); + + if (this.options.langParam) { + params.append(this.options.langParam, this.options.urlLangCode!); + } + + const url = `${this.site}${this.options.searchPage}?${params.toString()}`; const $ = await this.getCheerio(url, true); return this.parseNovels($); diff --git a/src/plugins/english/freewebnovel.ts b/src/plugins/english/freewebnovel.ts index b74e70e..4e48450 100644 --- a/src/plugins/english/freewebnovel.ts +++ b/src/plugins/english/freewebnovel.ts @@ -7,7 +7,7 @@ class FreeWebNovel implements Plugin.PluginBase { id = 'FWN.com'; name = 'Free Web Novel'; site = 'https://freewebnovel.com/'; - version = '2.0.0'; + version = '2.0.1'; icon = 'src/en/freewebnovel/icon.png'; lastSearch: number | null = null; @@ -219,9 +219,9 @@ class FreeWebNovel implements Plugin.PluginBase { onend() { novel.chapters = chapter; - novel.genres = genreArray.join(''); - novel.author = authorParts.join(''); - novel.summary = summaryParts.join('\n'); + novel.genres = genreArray.join('').trim(); + novel.author = authorParts.join('').trim(); + novel.summary = summaryParts.join('\n\n'); novel.status = statusParts .join('') .toLowerCase()