plugin refactoring (#639)

* fix

* fix

* madara plugins cleaned

Error: net::ERR_NAME_NOT_RESOLVED at https://cratenovel.com/novel/page/1?m_orderby=latest
Error: net::ERR_SSL_PROTOCOL_ERROR at https://clicknovel.net/novel/page/1?m_orderby=latest
Error: net::ERR_NAME_NOT_RESOLVED at https://www.onlymtl.com/novel/page/1?m_orderby=latest
Error: net::ERR_TOO_MANY_REDIRECTS at https://mtlnovel.club/novel/page/1?m_orderby=latest
Error: net::ERR_NAME_NOT_RESOLVED at https://noobchan.xyz/novel/page/1?m_orderby=latest

* readwn

* .
This commit is contained in:
Rider21
2023-11-27 17:21:58 +03:00
committed by GitHub
parent 86171ab1cc
commit c118cc2702
142 changed files with 74807 additions and 10912 deletions
+8 -2
View File
@@ -1,7 +1,8 @@
import { FilterInputs } from "../../../libs/filterInputs";
import { ScrpitGeneratorFunction } from "../generate";
import list from "./sources.json";
import { MadaraMetadata } from "./template";
import { readFileSync } from 'fs';
import { readFileSync } from "fs";
import path from "path";
@@ -14,7 +15,12 @@ export const generateAll: ScrpitGeneratorFunction = function () {
const generator = function generator(metadata: MadaraMetadata) {
const lang = metadata.options?.lang || "English";
const madaraTemplate = readFileSync(path.join(__dirname, 'template.ts'), {encoding: 'utf-8'});
const madaraTemplate = readFileSync(path.join(__dirname, "template.ts"), {encoding: "utf-8"});
metadata.filters = metadata.filters?.map((filter: any) => ({
...filter,
inputType: FilterInputs[filter.inputType]
}));
const pluginScript = `
${madaraTemplate}
const plugin = new MadaraPlugin(${JSON.stringify(metadata)});
-50
View File
@@ -2555,25 +2555,6 @@
"lang": "English"
}
},
{
"id": "cratenovel",
"sourceSite": "https://cratenovel.com/",
"sourceName": "CrateNovel",
"filters": [],
"options": {
"useNewChapterEndpoint": true,
"lang": "English"
}
},
{
"id": "clicknovel",
"sourceSite": "https://clicknovel.net/",
"sourceName": "ClickNovel",
"filters": [],
"options": {
"lang": "English"
}
},
{
"id": "readwebnovels",
"sourceSite": "https://readwebnovels.net/",
@@ -3233,16 +3214,6 @@
"lang": "Indonesian"
}
},
{
"id": "onlymtl",
"sourceSite": "https://www.onlymtl.com/",
"sourceName": "OnlyMTL",
"filters": [],
"options": {
"useNewChapterEndpoint": true,
"lang": "English"
}
},
{
"id": "hizomanga",
"sourceSite": "https://hizomanga.com/",
@@ -3889,26 +3860,6 @@
"lang": "English"
}
},
{
"id": "mtlnovel.club",
"sourceSite": "https://mtlnovel.club/",
"sourceName": "MTLNovel.Club",
"filters": [],
"options": {
"useNewChapterEndpoint": true,
"lang": "English"
}
},
{
"id": "noobchanTL",
"sourceSite": "https://noobchan.xyz/",
"sourceName": "Noobchan Translation",
"filters": [],
"options": {
"useNewChapterEndpoint": true,
"lang": "English"
}
},
{
"id": "guavaread",
"sourceSite": "https://guavaread.com/",
@@ -5314,7 +5265,6 @@
],
"options": {
"path": {
"novels": "novel",
"novel": "manga",
"chapter": "manga"
},
+3 -3
View File
@@ -43,7 +43,7 @@ class MadaraPlugin implements Plugin.PluginBase {
userAgent: string;
cookieString: string;
options?: MadaraOptions;
filter?: Filter[] | undefined;
filters?: Filter[] | undefined;
constructor(metadata: MadaraMetadata){
this.id = metadata.id;
@@ -55,14 +55,14 @@ class MadaraPlugin implements Plugin.PluginBase {
this.userAgent = "";
this.cookieString = "";
this.options = metadata.options;
this.filter = metadata.filters;
this.filters = metadata.filters;
}
async popularNovels(pageNo: number, {filters, showLatestNovels}: Plugin.PopularNovelsOptions): Promise<Plugin.NovelItem[]> {
const novels: Plugin.NovelItem[] = [];
let url = this.site;
if (filters?.genres && this.options?.path?.genres) {
url += this.options?.path?.genres + filters.genres + '/';
url += this.options?.path?.genres + filters.genres;
} else {
url += this.options?.path?.novels ? this.options.path.novels : MadaraDefaultPath.novels;
}
+34
View File
@@ -0,0 +1,34 @@
import { FilterInputs } from "../../../libs/filterInputs";
import { ScrpitGeneratorFunction } from "../generate";
import list from "./sources.json";
import { ReadwnMetadata } from "./template";
import { readFileSync } from "fs";
import path from "path";
export const generateAll: ScrpitGeneratorFunction = function () {
return list.map((metadata: ReadwnMetadata) => {
return generator(metadata);
});
};
const generator = function generator(metadata: ReadwnMetadata) {
const rulateTemplate = readFileSync(path.join(__dirname, "template.ts"), {
encoding: "utf-8",
});
metadata.filters = metadata.filters?.map((filter: any) => ({
...filter,
inputType: FilterInputs[filter.inputType],
}));
const pluginScript = `
${rulateTemplate}
const plugin = new ReadwnPlugin(${JSON.stringify(metadata)});
export default plugin;
`.trim();
return {
lang: "english",
filename: metadata.sourceName,
pluginScript,
};
};
File diff suppressed because it is too large Load Diff
+199
View File
@@ -0,0 +1,199 @@
import { fetchFile, fetchApi } from "@libs/fetch";
import { Filter } from "@libs/filterInputs";
import { Plugin } from "@typings/plugin";
import { NovelStatus } from "@libs/novelStatus";
import { load as parseHTML } from "cheerio";
import { parseMadaraDate } from "@libs/parseMadaraDate";
// import { defaultCover } from "@libs/defaultCover";
export interface ReadwnMetadata {
id: string;
sourceSite: string;
sourceName: string;
filters?: Filter[];
}
class ReadwnPlugin implements Plugin.PluginBase {
id: string;
name: string;
icon: string;
site: string;
version: string;
userAgent: string;
cookieString: string;
filters?: Filter[] | undefined;
constructor(metadata: ReadwnMetadata) {
this.id = metadata.id;
this.name = metadata.sourceName + "[readwn]";
const iconFileName = metadata.sourceName.replace(/\s+/g, "").toLowerCase();
this.icon = `multisrc/readwn/icons/${iconFileName}.png`;
this.site = metadata.sourceSite;
this.version = "1.0.0";
this.userAgent = "";
this.cookieString = "";
this.filters = metadata.filters;
}
async popularNovels(
pageNo: number,
{ filters, showLatestNovels }: Plugin.PopularNovelsOptions,
): Promise<Plugin.NovelItem[]> {
const novels: Plugin.NovelItem[] = [];
const baseUrl = this.site;
let url = baseUrl + "/list/";
url += (filters?.genres || "all") + "/";
url += (filters?.status || "all") + "-";
url +=
(showLatestNovels ? "lastdotime" : filters?.sort || "newstime") + "-";
url += pageNo - 1 + ".html";
if (filters?.tags) {
url = baseUrl + "/tags/" + filters.tags + "-0.html";
}
const result = await fetchApi(url);
const body = await result.text();
const loadedCheerio = parseHTML(body);
loadedCheerio("li.novel-item").each(function () {
const name = loadedCheerio(this).find("h4").text();
const url = loadedCheerio(this).find("a").attr("href");
const cover = loadedCheerio(this).find(".novel-cover > img").attr("data-src");
if (!name || !url) return;
novels.push({ name: name, cover: baseUrl + cover, url: baseUrl + url });
});
return novels;
}
async parseNovelAndChapters(novelUrl: string): Promise<Plugin.SourceNovel> {
const baseUrl = this.site;
const result = await fetchApi(novelUrl);
const body = await result.text();
let loadedCheerio = parseHTML(body);
const novel: Plugin.SourceNovel = {
url: novelUrl,
};
novel.name = loadedCheerio("h1.novel-title").text();
novel.author = loadedCheerio("span[itemprop=author]").text();
novel.cover =
baseUrl + loadedCheerio("figure.cover > img").attr("data-src");
novel.summary = loadedCheerio(".summary")
.text()
.replace("Summary", "")
.trim();
novel.genres = loadedCheerio("div.categories > ul > li")
.map((index, element) => loadedCheerio(element).text()?.trim())
.get()
.join(",");
loadedCheerio("div.header-stats > span").each(function () {
if (loadedCheerio(this).find("small").text() === "Status") {
novel.status = loadedCheerio(this).find("strong").text();
}
});
const chapters: Plugin.ChapterItem[] = [];
const latestChapterNo = parseInt(
loadedCheerio(".header-stats")
.find("span > strong")
.first()
.text()
.trim(),
);
let lastChapterNo = 1;
loadedCheerio(".chapter-list li").each(function () {
const name = loadedCheerio(this).find("a .chapter-title").text().trim();
const url = loadedCheerio(this).find("a").attr("href")?.trim();
const releaseTime = loadedCheerio(this)
.find("a .chapter-update")
.text()
.trim();
lastChapterNo = parseInt(
loadedCheerio(this).find("a .chapter-no").text().trim(),
);
if (!name || !url) return;
chapters.push({
name,
releaseTime: parseMadaraDate(releaseTime),
url: baseUrl + url,
});
});
const novelId =
"/" + novelUrl.replace(".html", "").replace(this.site, "") + "_";
// Itterate once more before loop to finish off
lastChapterNo++;
for (let i = lastChapterNo; i <= latestChapterNo; i++) {
const name = "Chapter " + i;
const releaseTime = null;
const url = baseUrl + novelId + i + ".html";
chapters.push({ name, releaseTime, url });
}
novel.chapters = chapters;
return novel;
}
async parseChapter(chapterUrl: string): Promise<string> {
const result = await fetchApi(chapterUrl);
const body = await result.text();
const loadedCheerio = parseHTML(body);
const chapterText = loadedCheerio(".chapter-content").html();
return chapterText || "";
}
async searchNovels(
searchTerm: string,
//page: number | undefined = 1,
): Promise<Plugin.NovelItem[]> {
const novels: Plugin.NovelItem[] = [];
const baseUrl = this.site;
const searchUrl = baseUrl + "/e/search/index.php";
const result = await fetchApi(searchUrl, {
headers: {
"Content-Type": "application/x-www-form-urlencoded",
Referer: baseUrl + "search.html",
Origin: baseUrl,
"user-agent":
this.userAgent ||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.131 Safari/537.36",
},
method: "POST",
body: JSON.stringify({
show: "title",
tempid: 1,
tbname: "news",
keyboard: searchTerm,
}),
});
const body = await result.text();
const loadedCheerio = parseHTML(body);
loadedCheerio("li.novel-item").each(function () {
const name = loadedCheerio(this).find("h4").text();
const url = loadedCheerio(this).find("a").attr("href");
const cover = loadedCheerio(this).find("img").attr("data-src");
if (!name || !url) return;
novels.push({ name, cover: baseUrl + cover, url: baseUrl + url });
});
return novels;
}
fetchImage = fetchFile;
}
+34
View File
@@ -0,0 +1,34 @@
import { FilterInputs } from "../../../libs/filterInputs";
import { ScrpitGeneratorFunction } from "../generate";
import list from "./sources.json";
import { RulateMetadata } from "./template";
import { readFileSync } from "fs";
import path from "path";
export const generateAll: ScrpitGeneratorFunction = function () {
return list.map((metadata: RulateMetadata) => {
return generator(metadata);
});
};
const generator = function generator(metadata: RulateMetadata) {
const rulateTemplate = readFileSync(path.join(__dirname, "template.ts"), {
encoding: "utf-8",
});
metadata.filters = metadata.filters?.map((filter: any) => ({
...filter,
inputType: FilterInputs[filter.inputType],
}));
const pluginScript = `
${rulateTemplate}
const plugin = new RulatePlugin(${JSON.stringify(metadata)});
export default plugin;
`.trim();
return {
lang: "russian",
filename: metadata.sourceName,
pluginScript,
};
};
File diff suppressed because it is too large Load Diff
+262
View File
@@ -0,0 +1,262 @@
import { fetchFile, fetchApi } from "@libs/fetch";
import { Filter } from "@libs/filterInputs";
import { Plugin } from "@typings/plugin";
import { NovelStatus } from "@libs/novelStatus";
import { load as parseHTML } from "cheerio";
// import { defaultCover } from "@libs/defaultCover";
export interface RulateMetadata {
id: string;
sourceSite: string;
sourceName: string;
filters?: Filter[];
}
class RulatePlugin implements Plugin.PluginBase {
id: string;
name: string;
icon: string;
site: string;
version: string;
userAgent: string;
cookieString: string;
filters?: Filter[] | undefined;
constructor(metadata: RulateMetadata) {
this.id = metadata.id;
this.name = metadata.sourceName + "[rulate]";
const iconFileName = metadata.sourceName.replace(/\s+/g, "").toLowerCase();
this.icon = `multisrc/rulate/icons/${iconFileName}.png`;
this.site = metadata.sourceSite;
this.version = "1.0.0";
this.userAgent = "";
this.cookieString = "";
this.filters = metadata.filters;
}
async popularNovels(
pageNo: number,
{ filters, showLatestNovels }: Plugin.PopularNovelsOptions,
): Promise<Plugin.NovelItem[]> {
const novels: Plugin.NovelItem[] = [];
const baseUrl = this.site;
let url = baseUrl + "/search?t=";
url += "&cat=" + (filters?.cat || "0");
url += "&s_lang=" + (filters?.s_lang || "0");
url += "&t_lang=" + (filters?.t_lang || "0");
url += "&type=" + (filters?.type || "0");
url += "&sort=" + (showLatestNovels ? "4" : filters?.sort || "6");
url += "&atmosphere=" + (filters?.atmosphere || "0");
url += "&adult=" + (filters?.adult || "0");
if (filters?.genres instanceof Array) {
url += filters.genres.map((i) => "&genres[]=" + i).join("");
}
if (filters?.genres_ex instanceof Array) {
url += filters.genres_ex.map((i) => "&genres_ex[]=" + i).join("");
}
if (filters?.tags instanceof Array) {
url += filters.tags.map((i) => "&tags[]=" + i).join("");
}
if (filters?.tags_ex instanceof Array) {
url += filters.tags_ex.map((i) => "&tags_ex[]=" + i).join("");
}
if (filters?.fandoms instanceof Array) {
url += filters.fandoms.map((i) => "&fandoms[]=" + i).join("");
}
if (filters?.fandoms_ex instanceof Array) {
url += filters.fandoms_ex.map((i) => "&fandoms_ex[]=" + i).join("");
}
if (filters?.trash instanceof Array) {
url += filters.trash.map((i) => "&" + i + "=1").join("");
}
url += "&Book_page=" + pageNo;
const result = await fetchApi(url);
const body = await result.text();
const loadedCheerio = parseHTML(body);
loadedCheerio(
'ul[class="search-results"] > li:not([class="ad_type_catalog"])',
).each(function () {
loadedCheerio(this).find("p > a").text();
const name = loadedCheerio(this).find("p > a").text();
const cover = loadedCheerio(this).find("img").attr("src");
const url = loadedCheerio(this).find("p > a").attr("href");
if (!name || !url) return;
novels.push({ name, cover: baseUrl + cover, url: baseUrl + url });
});
return novels;
}
async parseNovelAndChapters(novelUrl: string): Promise<Plugin.SourceNovel> {
const baseUrl = this.site;
const novel: Plugin.SourceNovel = {
url: novelUrl,
};
let result = await fetchApi(novelUrl);
if (result.url.includes("mature?path=")) {
const formData = new FormData();
formData.append("path", novelUrl);
formData.append("ok", "Да");
await fetchApi(result.url, {
method: "POST",
body: formData,
});
result = await fetchApi(novelUrl);
}
const body = await result.text();
const loadedCheerio = parseHTML(body);
novel.name = loadedCheerio(
'div[class="container"] > div > div > h1, div.span8:nth-child(1) > h1:nth-child(2)',
)
.text()
.trim();
novel.cover =
baseUrl + loadedCheerio('div[class="images"] > div img').attr("src");
novel.summary = loadedCheerio(
"#Info > div:nth-child(3), .book-description",
).text();
novel.author = loadedCheerio(
".book-stats-icons_author > span:nth-child(2) > a:nth-child(1)",
).text();
const genres: string[] = [];
loadedCheerio("div.span5 > p, .span5 > div:nth-child(2) > p").each(
function () {
switch (loadedCheerio(this).find("strong").text()) {
case "Автор:":
novel.author = loadedCheerio(this).find("em > a").text().trim();
break;
case "Выпуск:":
novel.status =
loadedCheerio(this).find("em").text().trim() === "продолжается"
? NovelStatus.Ongoing
: NovelStatus.Completed;
break;
case "Тэги:":
loadedCheerio(this)
.find("em > a")
.each(function () {
genres.push(loadedCheerio(this).text());
});
break;
case "Жанры:":
loadedCheerio(this)
.find("em > a")
.each(function () {
genres.push(loadedCheerio(this).text());
});
break;
}
},
);
if (genres.length) {
novel.genres = genres.reverse().join(",");
}
const chapters: Plugin.ChapterItem[] = [];
loadedCheerio("table > tbody > tr.chapter_row").each(function () {
const chapterName = loadedCheerio(this)
.find('td[class="t"] > a')
.text()
.trim();
const releaseDate = loadedCheerio(this)
.find("td > span")
.attr("title")
?.trim();
const chapterUrl = loadedCheerio(this)
.find('td[class="t"] > a')
.attr("href");
if (
loadedCheerio(this).find('td > span[class="disabled"]').length < 1 &&
releaseDate
) {
chapters.push({
name: chapterName,
releaseTime: releaseDate,
url: baseUrl + chapterUrl,
});
}
});
novel.chapters = chapters;
return novel;
}
async parseChapter(chapterUrl: string): Promise<string> {
const baseUrl = this.site;
let result = await fetchApi(chapterUrl);
if (result.url.includes("mature?path=")) {
const formData = new FormData();
formData.append("ok", "Да");
await fetchApi(result.url, {
method: "POST",
body: formData,
});
result = await fetchApi(chapterUrl);
}
const body = await result.text();
const loadedCheerio = parseHTML(body);
loadedCheerio(".content-text img").each(function () {
if (!loadedCheerio(this).attr("src")?.startsWith("http")) {
const src = loadedCheerio(this).attr("src");
loadedCheerio(this).attr("src", baseUrl + src);
}
});
const chapterText = loadedCheerio(".content-text").html();
return chapterText || "";
}
async searchNovels(
searchTerm: string,
//page: number | undefined = 1,
): Promise<Plugin.NovelItem[]> {
const novels: Plugin.NovelItem[] = [];
const result = await fetchApi(
this.site + "/search/autocomplete?query=" + searchTerm,
);
const json = (await result.json()) as response[];
json.forEach((novel) => {
const name = novel.title_one + " / " + novel.title_two;
if (!novel.url) return;
novels.push({
name,
cover: this.site + novel.img,
url: this.site + novel.url,
});
});
return novels;
}
fetchImage = fetchFile;
}
interface response {
id: number;
title_one: string;
title_two: string;
url: string;
img: string;
}