refactor: Overhaul Plugin Architecture & Playground UI (#1846)
* refactor: Move Plugins Out and Consolidate with Multisrc * feat: Overhaul Playground UI * feat: Add Filters to New UI * feat: Persist Selected Plugin * feat: Add Direct Parse Novel/Chapter Options and Feedback Toasts [skip ci] * feat: UX Improvements * chore: Update Plugin Path in Publish Workflow * fix: Add Backward Compatibility * chore: Update Plugin Publishing Script
This commit is contained in:
Rajarshee Chatterjee
committed by
GitHub
GitHub
parent
252beff0bf
commit
20c2b75b01
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,40 @@
|
||||
import list from './sources.json' with { type: 'json' };
|
||||
import { existsSync, readFileSync } from 'fs';
|
||||
import { fileURLToPath } from 'url';
|
||||
import { dirname, join } from 'path';
|
||||
|
||||
const folder = dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
export const generateAll = function () {
|
||||
return list.map(source => {
|
||||
const exist = existsSync(join(folder, 'filters', source.id + '.json'));
|
||||
if (exist) {
|
||||
const filters = readFileSync(
|
||||
join(folder, 'filters', source.id + '.json'),
|
||||
);
|
||||
source.filters = JSON.parse(filters).filters;
|
||||
}
|
||||
console.log(
|
||||
`[readwn] Generating: ${source.id}${' '.repeat(20 - source.id.length)} ${source.filters ? '🔎with filters🔍' : '🚫no filters🚫'}`,
|
||||
);
|
||||
return generator(source);
|
||||
});
|
||||
};
|
||||
|
||||
const generator = function generator(source) {
|
||||
const readwnTemplate = readFileSync(join(folder, 'template.ts'), {
|
||||
encoding: 'utf-8',
|
||||
});
|
||||
|
||||
const pluginScript = `
|
||||
${readwnTemplate}
|
||||
const plugin = new ReadwnPlugin(${JSON.stringify(source)});
|
||||
export default plugin;
|
||||
`.trim();
|
||||
|
||||
return {
|
||||
lang: 'english',
|
||||
filename: source.sourceName,
|
||||
pluginScript,
|
||||
};
|
||||
};
|
||||
@@ -0,0 +1,155 @@
|
||||
require('module-alias/register');
|
||||
import * as fs from 'fs';
|
||||
import * as cheerio from 'cheerio';
|
||||
import * as path from 'path';
|
||||
import { FilterTypes, FilterOption } from '@libs/filterInputs';
|
||||
const type: string[] = ['genres', 'status', 'sort'];
|
||||
|
||||
async function getFilters(name: string, url: string) {
|
||||
const html = await fetch(url + '/list/all/all-newstime-0.html').then(res =>
|
||||
res.text(),
|
||||
);
|
||||
const $: cheerio.CheerioAPI = cheerio.load(html);
|
||||
const filters: any = {
|
||||
'sort': {
|
||||
type: FilterTypes.Picker,
|
||||
label: 'Sort By',
|
||||
value: 'onclick',
|
||||
options: [],
|
||||
},
|
||||
'status': {
|
||||
type: FilterTypes.Picker,
|
||||
label: 'Status',
|
||||
value: 'all',
|
||||
options: [],
|
||||
},
|
||||
'genres': {
|
||||
type: FilterTypes.Picker,
|
||||
label: 'Genre / Category',
|
||||
value: '',
|
||||
options: [],
|
||||
},
|
||||
'tags': {
|
||||
type: FilterTypes.Picker,
|
||||
label: 'Tags',
|
||||
value: '',
|
||||
options: [{ label: 'NONE', value: '' }],
|
||||
},
|
||||
};
|
||||
|
||||
$('ul.proplist')?.each?.(function (index, ulElement) {
|
||||
if (!type[index]) return;
|
||||
$(ulElement)
|
||||
.find('li > a')
|
||||
.each((indx, liElement) =>
|
||||
filters[type[index]].options.push({
|
||||
label: $(liElement).text(),
|
||||
value: $(liElement).attr('href'),
|
||||
}),
|
||||
);
|
||||
filters[type[index]].options = filters[type[index]].options.map(
|
||||
(item: FilterOption) => {
|
||||
let res = item.value;
|
||||
if (index == 0) res = item.value.split('/')[2];
|
||||
if (index == 1)
|
||||
res = item.value.replace(/\/list\/all\/(.*?)-.*$/, '$1');
|
||||
if (index == 2) res = item.value.replace(/.*\/all-(.*?)-.*/, '$1');
|
||||
return {
|
||||
label: item.label,
|
||||
value: res,
|
||||
};
|
||||
},
|
||||
) as FilterOption[];
|
||||
filters[type[index]].options.sort((a: FilterOption, b: FilterOption) => {
|
||||
if (a.label === 'All') {
|
||||
return -1;
|
||||
} else if (b.label === 'All') {
|
||||
return 1;
|
||||
}
|
||||
return a.label.localeCompare(b.label);
|
||||
});
|
||||
});
|
||||
|
||||
const response = await fetch(url + '/browsetags/').then(res => res.text());
|
||||
const loadedCheerio: cheerio.CheerioAPI = cheerio.load(response);
|
||||
const allPage = loadedCheerio('.tag-letters > a')
|
||||
.map((index, element) => loadedCheerio(element).attr('href'))
|
||||
.get();
|
||||
// ===================== tags ======================
|
||||
for (let i = 0; i < allPage.length; i++) {
|
||||
console.log('fetch', url + allPage[i]);
|
||||
const resTags = await fetch(url + allPage[i]).then(res => res.text());
|
||||
const $: cheerio.CheerioAPI = cheerio.load(resTags);
|
||||
$('.tag-items > li > a').each((index, element) =>
|
||||
filters['tags'].options.push({
|
||||
label: $(element).text()?.trim(),
|
||||
value: $(element)
|
||||
.attr('href')
|
||||
?.split('/')
|
||||
?.pop()
|
||||
?.replace('-0.html', ''),
|
||||
}),
|
||||
);
|
||||
const nextPage = $('.pagination > li:last-child > a').attr('href');
|
||||
if (nextPage) {
|
||||
const allpage = parseInt(nextPage.replace(/[^0-9]/g, '') || '0', 10);
|
||||
for (let pageNo = 0; pageNo < allpage; pageNo++) {
|
||||
await sleep(3000);
|
||||
console.log(
|
||||
'fetch',
|
||||
url + allPage[i].replace('-0.html', `-${pageNo + 1}.html`),
|
||||
);
|
||||
const resTags = await fetch(
|
||||
url + allPage[i].replace('-0.html', `-${pageNo + 1}.html`),
|
||||
).then(res => res.text());
|
||||
const $: cheerio.CheerioAPI = cheerio.load(resTags);
|
||||
|
||||
$('.tag-items > li > a').each((index, element) =>
|
||||
filters['tags'].options.push({
|
||||
label: $(element).text()?.trim(),
|
||||
value: $(element).attr('href')?.split('/')?.pop()?.split('-')?.[0],
|
||||
}),
|
||||
);
|
||||
}
|
||||
}
|
||||
await sleep(3000);
|
||||
}
|
||||
|
||||
fs.writeFileSync(
|
||||
path.join(__dirname, 'filters', name + '.json'),
|
||||
JSON.stringify({ filters }, null, 2),
|
||||
);
|
||||
console.log(`✅Filters created successfully for ${name}✅`);
|
||||
}
|
||||
|
||||
async function sleep(ms: number) {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async function askGetFilter() {
|
||||
const readline = require('readline').createInterface({
|
||||
input: process.stdin,
|
||||
output: process.stdout,
|
||||
});
|
||||
|
||||
const EREASE_PREV_LINE = '\x1b[1A\r\x1b[2K';
|
||||
await readline.question(
|
||||
'Enter the id of the site (same one as in sources.json): ',
|
||||
async (name: string) => {
|
||||
await readline.question(
|
||||
EREASE_PREV_LINE + 'Enter the URL (same one as in sources.json): ',
|
||||
async (url: string) => {
|
||||
try {
|
||||
await getFilters(name, url);
|
||||
} catch (e: any) {
|
||||
console.error('Error while getting filters from', url);
|
||||
console.log(e.message || e);
|
||||
}
|
||||
readline.close();
|
||||
},
|
||||
);
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
askGetFilter();
|
||||
@@ -0,0 +1,193 @@
|
||||
{
|
||||
"filters": {
|
||||
"sort": {
|
||||
"label": "Sort By",
|
||||
"options": [
|
||||
{
|
||||
"label": "New",
|
||||
"value": "newstime"
|
||||
},
|
||||
{
|
||||
"label": "Popular",
|
||||
"value": "onclick"
|
||||
},
|
||||
{
|
||||
"label": "Updates",
|
||||
"value": "lastdotime"
|
||||
}
|
||||
],
|
||||
"value": "newstime",
|
||||
"type": "Picker"
|
||||
},
|
||||
"status": {
|
||||
"label": "Status",
|
||||
"options": [
|
||||
{
|
||||
"label": "All",
|
||||
"value": "all"
|
||||
},
|
||||
{
|
||||
"label": "Completed",
|
||||
"value": "Completed"
|
||||
},
|
||||
{
|
||||
"label": "Ongoing",
|
||||
"value": "Ongoing"
|
||||
}
|
||||
],
|
||||
"value": "all",
|
||||
"type": "Picker"
|
||||
},
|
||||
"genres": {
|
||||
"options": [
|
||||
{
|
||||
"label": "All",
|
||||
"value": "all"
|
||||
},
|
||||
{
|
||||
"label": "Action",
|
||||
"value": "action"
|
||||
},
|
||||
{
|
||||
"label": "Adventure",
|
||||
"value": "adventure"
|
||||
},
|
||||
{
|
||||
"label": "Comedy",
|
||||
"value": "comedy"
|
||||
},
|
||||
{
|
||||
"label": "Contemporary Romance",
|
||||
"value": "contemporary-romance"
|
||||
},
|
||||
{
|
||||
"label": "Drama",
|
||||
"value": "drama"
|
||||
},
|
||||
{
|
||||
"label": "Eastern Fantasy",
|
||||
"value": "eastern-fantasy"
|
||||
},
|
||||
{
|
||||
"label": "Fantasy",
|
||||
"value": "fantasy"
|
||||
},
|
||||
{
|
||||
"label": "Fantasy Romance",
|
||||
"value": "fantasy-romance"
|
||||
},
|
||||
{
|
||||
"label": "Gender Bender",
|
||||
"value": "gender-bender"
|
||||
},
|
||||
{
|
||||
"label": "Harem",
|
||||
"value": "harem"
|
||||
},
|
||||
{
|
||||
"label": "Historical",
|
||||
"value": "historical"
|
||||
},
|
||||
{
|
||||
"label": "Horror",
|
||||
"value": "horror"
|
||||
},
|
||||
{
|
||||
"label": "Josei",
|
||||
"value": "josei"
|
||||
},
|
||||
{
|
||||
"label": "Lolicon",
|
||||
"value": "lolicon"
|
||||
},
|
||||
{
|
||||
"label": "Magical Realism",
|
||||
"value": "magical-realism"
|
||||
},
|
||||
{
|
||||
"label": "Martial Arts",
|
||||
"value": "martial-arts"
|
||||
},
|
||||
{
|
||||
"label": "Mecha",
|
||||
"value": "mecha"
|
||||
},
|
||||
{
|
||||
"label": "Mystery",
|
||||
"value": "mystery"
|
||||
},
|
||||
{
|
||||
"label": "Psychological",
|
||||
"value": "psychological"
|
||||
},
|
||||
{
|
||||
"label": "Romance",
|
||||
"value": "romance"
|
||||
},
|
||||
{
|
||||
"label": "School Life",
|
||||
"value": "school-life"
|
||||
},
|
||||
{
|
||||
"label": "Sci-fi",
|
||||
"value": "sci-fi"
|
||||
},
|
||||
{
|
||||
"label": "Seinen",
|
||||
"value": "seinen"
|
||||
},
|
||||
{
|
||||
"label": "Shoujo",
|
||||
"value": "shoujo"
|
||||
},
|
||||
{
|
||||
"label": "Shounen",
|
||||
"value": "shounen"
|
||||
},
|
||||
{
|
||||
"label": "Shounen Ai",
|
||||
"value": "shounen-ai"
|
||||
},
|
||||
{
|
||||
"label": "Slice of Life",
|
||||
"value": "slice-of-life"
|
||||
},
|
||||
{
|
||||
"label": "Sports",
|
||||
"value": "sports"
|
||||
},
|
||||
{
|
||||
"label": "Supernatural",
|
||||
"value": "supernatural"
|
||||
},
|
||||
{
|
||||
"label": "Tragedy",
|
||||
"value": "tragedy"
|
||||
},
|
||||
{
|
||||
"label": "Video Games",
|
||||
"value": "video-games"
|
||||
},
|
||||
{
|
||||
"label": "Wuxia",
|
||||
"value": "wuxia"
|
||||
},
|
||||
{
|
||||
"label": "Xianxia",
|
||||
"value": "xianxia"
|
||||
},
|
||||
{
|
||||
"label": "Xuanhuan",
|
||||
"value": "xuanhuan"
|
||||
},
|
||||
{
|
||||
"label": "Yaoi",
|
||||
"value": "yaoi"
|
||||
}
|
||||
]
|
||||
},
|
||||
"tags": {
|
||||
"options": [{ "label": "Default", "value": "" }]
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
[
|
||||
{
|
||||
"id": "wuxiap",
|
||||
"sourceSite": "https://www.wuxiabox.com",
|
||||
"sourceName": "Wuxiabox"
|
||||
},
|
||||
{
|
||||
"id": "wuxiacity",
|
||||
"sourceSite": "https://www.wuxiafox.com",
|
||||
"sourceName": "Wuxiafox"
|
||||
},
|
||||
{
|
||||
"id": "ltnovel",
|
||||
"sourceSite": "https://www.ltnovels.com",
|
||||
"sourceName": "Ltnovel",
|
||||
"options": {
|
||||
"versionIncrements": 1
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "wuxiamtl",
|
||||
"sourceSite": "https://www.fanmtl.com",
|
||||
"sourceName": "Fans MTL",
|
||||
"options": {
|
||||
"versionIncrements": 2
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "fannovel",
|
||||
"sourceSite": "https://www.fanmtl.com",
|
||||
"sourceName": "FanNovel",
|
||||
"options": {
|
||||
"versionIncrements": 2
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "wuxiaspace",
|
||||
"sourceSite": "https://www.wuxiaspot.com",
|
||||
"sourceName": "Wuxia Space",
|
||||
"options": {
|
||||
"versionIncrements": 1
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "wuxiav",
|
||||
"sourceSite": "https://www.wuxiav.com",
|
||||
"sourceName": "WuxiaV"
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,225 @@
|
||||
import { fetchApi } from '@libs/fetch';
|
||||
import { Filters } from '@libs/filterInputs';
|
||||
import { Plugin } from '@/types/plugin';
|
||||
import { NovelStatus } from '@libs/novelStatus';
|
||||
import { load as parseHTML } from 'cheerio';
|
||||
import dayjs from 'dayjs';
|
||||
|
||||
type ReadwnOptions = {
|
||||
versionIncrements?: number;
|
||||
};
|
||||
|
||||
export type ReadwnMetadata = {
|
||||
id: string;
|
||||
sourceSite: string;
|
||||
sourceName: string;
|
||||
filters?: any;
|
||||
options?: ReadwnOptions;
|
||||
};
|
||||
|
||||
class ReadwnPlugin implements Plugin.PluginBase {
|
||||
id: string;
|
||||
name: string;
|
||||
icon: string;
|
||||
site: string;
|
||||
version: string;
|
||||
filters?: Filters;
|
||||
|
||||
constructor(metadata: ReadwnMetadata) {
|
||||
this.id = metadata.id;
|
||||
this.name = metadata.sourceName;
|
||||
this.icon = `multisrc/readwn/${metadata.id.toLowerCase()}/icon.png`;
|
||||
this.site = metadata.sourceSite;
|
||||
const versionIncrements = metadata.options?.versionIncrements || 0;
|
||||
this.version = `1.0.${2 + versionIncrements}`;
|
||||
this.filters = metadata.filters;
|
||||
}
|
||||
|
||||
async popularNovels(
|
||||
pageNo: number,
|
||||
{ filters, showLatestNovels }: Plugin.PopularNovelsOptions,
|
||||
): Promise<Plugin.NovelItem[]> {
|
||||
let url = this.site + '/list/';
|
||||
url += (filters?.genres?.value || 'all') + '/';
|
||||
url += (filters?.status?.value || 'all') + '-';
|
||||
url += showLatestNovels ? 'lastdotime' : filters?.sort?.value || 'newstime';
|
||||
url += '-' + (pageNo - 1) + '.html';
|
||||
|
||||
if (filters?.tags?.value) {
|
||||
//only 1 page
|
||||
url = this.site + '/tags/' + filters.tags.value + '-0.html';
|
||||
}
|
||||
|
||||
const body = await fetchApi(url).then(res => res.text());
|
||||
const loadedCheerio = parseHTML(body);
|
||||
|
||||
const novels: Plugin.NovelItem[] = loadedCheerio('li.novel-item')
|
||||
.map((index, element) => ({
|
||||
name: loadedCheerio(element).find('h4').text() || '',
|
||||
cover:
|
||||
this.site +
|
||||
loadedCheerio(element).find('.novel-cover > img').attr('data-src'),
|
||||
path: loadedCheerio(element).find('a').attr('href') || '',
|
||||
}))
|
||||
.get()
|
||||
.filter(novel => novel.name && novel.path);
|
||||
|
||||
return novels;
|
||||
}
|
||||
|
||||
async parseNovel(novelPath: string): Promise<Plugin.SourceNovel> {
|
||||
const body = await fetchApi(this.site + novelPath).then(res => res.text());
|
||||
const loadedCheerio = parseHTML(body);
|
||||
|
||||
const novel: Plugin.SourceNovel = {
|
||||
path: novelPath,
|
||||
name: loadedCheerio('h1.novel-title').text() || '',
|
||||
};
|
||||
|
||||
novel.author = loadedCheerio('span[itemprop=author]').text();
|
||||
novel.cover =
|
||||
this.site + loadedCheerio('figure.cover > img').attr('data-src');
|
||||
|
||||
novel.summary = loadedCheerio('.summary')
|
||||
.text()
|
||||
.replace('Summary', '')
|
||||
.trim();
|
||||
|
||||
novel.genres = loadedCheerio('div.categories > ul > li')
|
||||
.map((index, element) => loadedCheerio(element).text()?.trim())
|
||||
.get()
|
||||
.join(',');
|
||||
|
||||
loadedCheerio('div.header-stats > span').each(function () {
|
||||
if (loadedCheerio(this).find('small').text() === 'Status') {
|
||||
novel.status =
|
||||
loadedCheerio(this).find('strong').text() === 'Ongoing'
|
||||
? NovelStatus.Ongoing
|
||||
: NovelStatus.Completed;
|
||||
}
|
||||
});
|
||||
|
||||
const latestChapterNo = parseInt(
|
||||
loadedCheerio('.header-stats')
|
||||
.find('span > strong')
|
||||
.first()
|
||||
.text()
|
||||
.trim(),
|
||||
);
|
||||
|
||||
const chapters: Plugin.ChapterItem[] = loadedCheerio('.chapter-list li')
|
||||
.map((chapterIndex, element) => {
|
||||
const name = loadedCheerio(element)
|
||||
.find('a .chapter-title')
|
||||
.text()
|
||||
.trim();
|
||||
const path = loadedCheerio(element).find('a').attr('href')?.trim();
|
||||
if (!name || !path) return null;
|
||||
|
||||
let releaseTime = loadedCheerio(element)
|
||||
.find('a .chapter-update')
|
||||
.text()
|
||||
.trim();
|
||||
if (releaseTime?.includes?.('ago')) {
|
||||
const timeAgo = releaseTime.match(/\d+/)?.[0] || '0';
|
||||
const timeAgoInt = parseInt(timeAgo, 10);
|
||||
|
||||
if (timeAgoInt) {
|
||||
const dayJSDate = dayjs(); // today
|
||||
if (
|
||||
releaseTime.includes('hours ago') ||
|
||||
releaseTime.includes('hour ago')
|
||||
) {
|
||||
dayJSDate.subtract(timeAgoInt, 'hours'); // go back N hours
|
||||
}
|
||||
|
||||
if (
|
||||
releaseTime.includes('days ago') ||
|
||||
releaseTime.includes('day ago')
|
||||
) {
|
||||
dayJSDate.subtract(timeAgoInt, 'days'); // go back N days
|
||||
}
|
||||
|
||||
if (
|
||||
releaseTime.includes('months ago') ||
|
||||
releaseTime.includes('month ago')
|
||||
) {
|
||||
dayJSDate.subtract(timeAgoInt, 'months'); // go back N months
|
||||
}
|
||||
|
||||
releaseTime = dayJSDate.format('LL');
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
name,
|
||||
path,
|
||||
releaseTime,
|
||||
chapterNumber: chapterIndex + 1,
|
||||
};
|
||||
})
|
||||
.get()
|
||||
.filter(chapter => chapter);
|
||||
|
||||
if (latestChapterNo > chapters.length) {
|
||||
const lastChapterNo = parseInt(
|
||||
chapters[chapters.length - 1].path.match(/_(\d+)\.html/)?.[1] || '',
|
||||
10,
|
||||
);
|
||||
|
||||
for (
|
||||
let i = (lastChapterNo || chapters.length) + 1;
|
||||
i <= latestChapterNo;
|
||||
i++
|
||||
) {
|
||||
chapters.push({
|
||||
name: 'Chapter ' + i,
|
||||
path: novelPath.replace('.html', '_' + i + '.html'),
|
||||
releaseTime: null,
|
||||
chapterNumber: i,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
novel.chapters = chapters;
|
||||
return novel;
|
||||
}
|
||||
|
||||
async parseChapter(chapterPath: string): Promise<string> {
|
||||
const body = await fetchApi(this.site + chapterPath).then(res =>
|
||||
res.text(),
|
||||
);
|
||||
const loadedCheerio = parseHTML(body);
|
||||
|
||||
const chapterText = loadedCheerio('.chapter-content').html() || '';
|
||||
return chapterText;
|
||||
}
|
||||
|
||||
async searchNovels(searchTerm: string): Promise<Plugin.NovelItem[]> {
|
||||
const result = await fetchApi(this.site + '/e/search/index.php', {
|
||||
headers: {
|
||||
'Content-Type': 'application/x-www-form-urlencoded',
|
||||
Referer: this.site + '/search.html',
|
||||
Origin: this.site,
|
||||
},
|
||||
method: 'POST',
|
||||
body: new URLSearchParams({
|
||||
show: 'title',
|
||||
tempid: 1,
|
||||
tbname: 'news',
|
||||
keyboard: searchTerm,
|
||||
}).toString(),
|
||||
}).then(res => res.text());
|
||||
const loadedCheerio = parseHTML(result);
|
||||
|
||||
const novels: Plugin.NovelItem[] = loadedCheerio('li.novel-item')
|
||||
.map((index, element) => ({
|
||||
name: loadedCheerio(element).find('h4').text() || '',
|
||||
cover: this.site + loadedCheerio(element).find('img').attr('data-src'),
|
||||
path: loadedCheerio(element).find('a').attr('href') || '',
|
||||
}))
|
||||
.get()
|
||||
.filter(novel => novel.name && novel.path);
|
||||
return novels;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user